Monitoring & Observability untuk Developer

# Instalasi Grafana OSS di Ubuntu/Debian
sudo apt-get install -y apt-transport-https software-properties-common wget
wget -q -O - https://apt.grafana.com/gpg.key | sudo gpg --dearmor > /etc/apt/keyrings/grafana.gpg
echo "deb [signed-by=/etc/apt/keyrings/grafana.gpg] https://apt.grafana.com stable main" | sudo tee /etc/apt/sources.list.d/grafana.list
sudo apt-get update
sudo apt-get install grafana

# Jalankan Grafana
sudo systemctl daemon-reload
sudo systemctl enable grafana-server
sudo systemctl start grafana-server

# Akses dashboard di http://localhost:3000
# Default login: admin / admin

# /etc/grafana/provisioning/datasources/prometheus.yml
apiVersion: 1

datasources:
  - name: Prometheus
    type: prometheus
    access: proxy
    url: http://prometheus:9090
    isDefault: true
    editable: false

  - name: Loki
    type: loki
    access: proxy
    url: http://loki:3100
    editable: false

  - name: Jaeger
    type: jaeger
    access: proxy
    url: http://jaeger:16686
    editable: false

# alert_rules.yml
groups:
  - name: application_alerts
    rules:
      # Alert: Error rate tinggi
      - alert: HighErrorRate
        expr: |
          sum(rate(http_requests_total{status=~"5.."}[5m]))
          / sum(rate(http_requests_total[5m]))
          > 0.05
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "Error rate tinggi terdeteksi"
          description: "Error rate {{ $value | humanizePercentage }} melebihi 5% selama 5 menit"
          runbook_url: "https://wiki.internal/runbook/high-error-rate"

      # Alert: Latency P99 tinggi
      - alert: HighP99Latency
        expr: |
          histogram_quantile(0.99,
            sum(rate(http_request_duration_seconds_bucket[5m])) by (le)
          ) > 2
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "P99 latency tinggi: {{ $value }}s"

      # Alert: Disk usage hampir penuh
      - alert: DiskSpaceRunningLow
        expr: |
          (1 - node_filesystem_avail_bytes / node_filesystem_size_bytes)
          * 100 > 85
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "Disk usage {{ $value }}% pada {{ $labels.instance }}"

      # Alert: Pod restart berulang
      - alert: PodCrashLooping
        expr: |
          increase(kube_pod_container_status_restarts_total[1h]) > 3
        labels:
          severity: critical
        annotations:
          summary: "Pod {{ $labels.pod }} crash looping"

# alertmanager.yml
global:
  resolve_timeout: 5m

route:
  receiver: 'default'
  group_by: ['alertname', 'severity']
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 4h
  routes:
    - match:
        severity: critical
      receiver: 'pagerduty-critical'
      repeat_interval: 15m
    - match:
        severity: warning
      receiver: 'slack-warning'

receivers:
  - name: 'default'
    slack_configs:
      - api_url: 'https://hooks.slack.com/services/xxx'
        channel: '#alerts'
        title: '[{{ .Status | toUpper }}] {{ .GroupLabels.alertname }}'
        text: '{{ range .Alerts }}{{ .Annotations.description }}\n{{ end }}'

  - name: 'pagerduty-critical'
    pagerduty_configs:
      - service_key: 'your-pagerduty-key'

  - name: 'slack-warning'
    slack_configs:
      - api_url: 'https://hooks.slack.com/services/yyy'
        channel: '#alerts-warning'
        title: '[WARNING] {{ .GroupLabels.alertname }}'

# SLO: 99.9% availability (error budget = 0.1%)

# Total request dalam 30 hari
total_requests = sum(increase(http_requests_total[30d]))

# Total error request dalam 30 hari
error_requests = sum(increase(http_requests_total{status=~"5.."}[30d]))

# Actual availability
actual_availability = (1 - error_requests / total_requests) * 100

# Error budget remaining (dalam persentase)
error_budget_total = 0.1  # 0.1% for 99.9% SLO
error_budget_consumed = (error_requests / total_requests) * 100
error_budget_remaining = error_budget_total - error_budget_consumed

# Error budget remaining (dalam menit per bulan)
error_budget_minutes = error_budget_remaining / 100 * 30 * 24 * 60