Monitoring & Observability untuk Developer

# R = Rate — Jumlah request per detik
rate(http_requests_total[5m])

# E = Errors — Jumlah error per detik
rate(http_requests_total{status=~"5.."}[5m])

# D = Duration — Latency request
histogram_quantile(0.99, rate(http_request_duration_seconds_bucket[5m]))

# Error ratio = errors / total requests
rate(http_requests_total{status=~"5.."}[5m])
  /
rate(http_requests_total[5m])

# U = Utilization — Persentase penggunaan resource
node_cpu_seconds_total
node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes

# S = Saturation — Overload / queue yang menunggu
node_load1  # Load average 1 menit
node_disk_io_time_weighted_seconds_total

# E = Errors — Error yang terjadi
node_disk_read_errors_total
node_network_receive_errs_total

{
  "timestamp": "2026-06-25T14:30:22.123Z",
  "level": "error",
  "service": "payment-api",
  "trace_id": "abc123def456",
  "span_id": "789ghi012",
  "method": "POST",
  "path": "/api/v1/payments",
  "status_code": 500,
  "duration_ms": 3420,
  "error": "Connection refused to payment-gateway:5432",
  "user_id": "u_12345",
  "request_id": "req_abc789",
  "host": "payment-api-3a7f9c",
  "message": "Payment processing failed"
}

// Instalasi: npm install @opentelemetry/sdk-node @opentelemetry/api

const { NodeTracerProvider } = require('@opentelemetry/sdk-trace-node');
const { JaegerExporter } = require('@opentelemetry/exporter-jaeger');
const { Resource } = require('@opentelemetry/resources');
const { SemanticResourceAttributes } = require('@opentelemetry/semantic-conventions');
const { trace } = require('@opentelemetry/api');

// Setup tracer provider
const provider = new NodeTracerProvider({
  resource: new Resource({
    [SemanticResourceAttributes.SERVICE_NAME]: 'payment-service',
    [SemanticResourceAttributes.SERVICE_VERSION]: '1.0.0',
  }),
});

// Export ke Jaeger
const exporter = new JaegerExporter({
  endpoint: 'http://jaeger:14268/api/traces',
});
provider.addSpanProcessor(new BatchSpanProcessor(exporter));
provider.register();

// Menggunakan tracing di kode aplikasi
const tracer = trace.getTracer('payment-service');

async function processPayment(orderId, amount) {
  return tracer.startActiveSpan('processPayment', async (span) => {
    try {
      span.setAttribute('order.id', orderId);
      span.setAttribute('payment.amount', amount);

      // Span untuk validasi
      await tracer.startActiveSpan('validateOrder', async (childSpan) => {
        await validateOrder(orderId);
        childSpan.end();
      });

      // Span untuk koneksi payment gateway
      await tracer.startActiveSpan('callPaymentGateway', async (childSpan) => {
        await callGateway(amount);
        childSpan.end();
      });

      span.setStatus({ code: 0 }); // OK
    } catch (error) {
      span.setStatus({ code: 2, message: error.message }); // ERROR
      span.recordException(error);
      throw error;
    } finally {
      span.end();
    }
  });
}

# Download Prometheus
wget https://github.com/prometheus/prometheus/releases/download/v2.53.0/prometheus-2.53.0.linux-amd64.tar.gz
tar xzf prometheus-2.53.0.linux-amd64.tar.gz
cd prometheus-2.53.0.linux-amd64

# Konfigurasi prometheus.yml
cat > prometheus.yml << 'EOF'
global:
  scrape_interval: 15s
  evaluation_interval: 15s

alerting:
  alertmanagers:
    - static_configs:
        - targets:
          - alertmanager:9093

rule_files:
  - "alert_rules.yml"

scrape_configs:
  # Monitor Prometheus itu sendiri
  - job_name: 'prometheus'
    static_configs:
      - targets: ['localhost:9090']

  # Monitor aplikasi Node.js
  - job_name: 'node-app'
    static_configs:
      - targets: ['app-server:9100']

  # Monitor dengan service discovery Kubernetes
  - job_name: 'kubernetes-pods'
    kubernetes_sd_configs:
      - role: pod
    relabel_configs:
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
        action: keep
        regex: true
EOF

# Jalankan Prometheus
./prometheus --config.file=prometheus.yml --storage.tsdb.retention.time=30d

# Rate of HTTP requests per second (5m window)
rate(http_requests_total[5m])

# Error rate (request dengan status 5xx)
sum(rate(http_requests_total{status=~"5.."}[5m])) / sum(rate(http_requests_total[5m]))

# P99 latency
histogram_quantile(0.99, rate(http_request_duration_seconds_bucket[5m]))

# CPU usage per instance
100 - (avg by(instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)

# Memory usage percentage
(1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100

# Top 10 busiest endpoints
topk(10, sum by (handler) (rate(http_requests_total[5m])))

# Alert: Error rate lebih dari 5% dalam 5 menit
sum(rate(http_requests_total{status=~"5.."}[5m])) / sum(rate(http_requests_total[5m])) > 0.05

# Instalasi Grafana OSS di Ubuntu/Debian
sudo apt-get install -y apt-transport-https software-properties-common wget
wget -q -O - https://apt.grafana.com/gpg.key | sudo gpg --dearmor > /etc/apt/keyrings/grafana.gpg
echo "deb [signed-by=/etc/apt/keyrings/grafana.gpg] https://apt.grafana.com stable main" | sudo tee /etc/apt/sources.list.d/grafana.list
sudo apt-get update
sudo apt-get install grafana

# Jalankan Grafana
sudo systemctl daemon-reload
sudo systemctl enable grafana-server
sudo systemctl start grafana-server

# Akses dashboard di http://localhost:3000
# Default login: admin / admin

# /etc/grafana/provisioning/datasources/prometheus.yml
apiVersion: 1

datasources:
  - name: Prometheus
    type: prometheus
    access: proxy
    url: http://prometheus:9090
    isDefault: true
    editable: false

  - name: Loki
    type: loki
    access: proxy
    url: http://loki:3100
    editable: false

  - name: Jaeger
    type: jaeger
    access: proxy
    url: http://jaeger:16686
    editable: false

# alert_rules.yml
groups:
  - name: application_alerts
    rules:
      # Alert: Error rate tinggi
      - alert: HighErrorRate
        expr: |
          sum(rate(http_requests_total{status=~"5.."}[5m]))
          / sum(rate(http_requests_total[5m]))
          > 0.05
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "Error rate tinggi terdeteksi"
          description: "Error rate {{ $value | humanizePercentage }} melebihi 5% selama 5 menit"
          runbook_url: "https://wiki.internal/runbook/high-error-rate"

      # Alert: Latency P99 tinggi
      - alert: HighP99Latency
        expr: |
          histogram_quantile(0.99,
            sum(rate(http_request_duration_seconds_bucket[5m])) by (le)
          ) > 2
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "P99 latency tinggi: {{ $value }}s"

      # Alert: Disk usage hampir penuh
      - alert: DiskSpaceRunningLow
        expr: |
          (1 - node_filesystem_avail_bytes / node_filesystem_size_bytes)
          * 100 > 85
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "Disk usage {{ $value }}% pada {{ $labels.instance }}"

      # Alert: Pod restart berulang
      - alert: PodCrashLooping
        expr: |
          increase(kube_pod_container_status_restarts_total[1h]) > 3
        labels:
          severity: critical
        annotations:
          summary: "Pod {{ $labels.pod }} crash looping"

# alertmanager.yml
global:
  resolve_timeout: 5m

route:
  receiver: 'default'
  group_by: ['alertname', 'severity']
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 4h
  routes:
    - match:
        severity: critical
      receiver: 'pagerduty-critical'
      repeat_interval: 15m
    - match:
        severity: warning
      receiver: 'slack-warning'

receivers:
  - name: 'default'
    slack_configs:
      - api_url: 'https://hooks.slack.com/services/xxx'
        channel: '#alerts'
        title: '[{{ .Status | toUpper }}] {{ .GroupLabels.alertname }}'
        text: '{{ range .Alerts }}{{ .Annotations.description }}\n{{ end }}'

  - name: 'pagerduty-critical'
    pagerduty_configs:
      - service_key: 'your-pagerduty-key'

  - name: 'slack-warning'
    slack_configs:
      - api_url: 'https://hooks.slack.com/services/yyy'
        channel: '#alerts-warning'
        title: '[WARNING] {{ .GroupLabels.alertname }}'

# SLO: 99.9% availability (error budget = 0.1%)

# Total request dalam 30 hari
total_requests = sum(increase(http_requests_total[30d]))

# Total error request dalam 30 hari
error_requests = sum(increase(http_requests_total{status=~"5.."}[30d]))

# Actual availability
actual_availability = (1 - error_requests / total_requests) * 100

# Error budget remaining (dalam persentase)
error_budget_total = 0.1  # 0.1% for 99.9% SLO
error_budget_consumed = (error_requests / total_requests) * 100
error_budget_remaining = error_budget_total - error_budget_consumed

# Error budget remaining (dalam menit per bulan)
error_budget_minutes = error_budget_remaining / 100 * 30 * 24 * 60