Prometheus & Grafana: Monitoring Infrastructure

# === Alur Kerja Monitoring ===

# 1. Target (server, broker, ESP32) mengekspos metrik
#    di endpoint HTTP: http://target:9100/metrics

# 2. Prometheus Server melakukan SCRAPE setiap N detik
#    Mengambil metrik dari semua target yang dikonfigurasi

# 3. Metrik disimpan di Time Series Database (lokal)

# 4. User membuat QUERY menggunakan PromQL
#    Contoh: rate(http_requests_total[5m])

# 5. GRAFANA mengambil data dari Prometheus dan menampilkan
#    dalam bentuk grafik, tabel, dan dashboard

# 6. ALERT RULES mengevaluasi metrik secara periodik
#    Jika kondisi terpenuhi → kirim ke Alertmanager

# 7. ALERTMANAGER mengirim notifikasi
#    Email, Slack, Telegram, PagerDuty, webhook, dll.

# === Format: metric_name{label="value"} timestamp value ===

# Contoh metrik dari Node Exporter
node_cpu_seconds_total{cpu="0",mode="idle"} 1.23456789e+06
node_cpu_seconds_total{cpu="0",mode="user"} 56789.123
node_memory_MemAvailable_bytes 4.294967296e+09
node_filesystem_avail_bytes{mountpoint="/",device="/dev/sda1"} 5.0e+10

# Contoh metrik dari HTTP server
http_requests_total{method="GET",handler="/api/sensor",status="200"} 12345
http_requests_total{method="POST",handler="/api/sensor",status="201"} 678

# Contoh metrik custom IoT
iot_sensor_temperature_celsius{device="esp32-001",room="ruang_tamu"} 28.5
iot_sensor_humidity_percent{device="esp32-001",room="ruang_tamu"} 65.2
iot_device_uptime_seconds{device="esp32-001"} 86400
iot_messages_received_total{topic="rumah/suhu"} 45678

# Download Prometheus terbaru
PROM_VERSION="2.52.0"
wget https://github.com/prometheus/prometheus/releases/download/v${PROM_VERSION}/prometheus-${PROM_VERSION}.linux-amd64.tar.gz

# Extract
tar xvfz prometheus-${PROM_VERSION}.linux-amd64.tar.gz
cd prometheus-${PROM_VERSION}.linux-amd64/

# Buat direktori
sudo mkdir -p /etc/prometheus /var/lib/prometheus

# Copy binary dan konfigurasi
sudo cp prometheus promtool /usr/local/bin/
sudo cp -r consoles console_libraries /etc/prometheus/
sudo cp prometheus.yml /etc/prometheus/

# Buat user dan set permissions
sudo useradd --no-create-home --shell /bin/false prometheus
sudo chown -R prometheus:prometheus /etc/prometheus /var/lib/prometheus

# Verifikasi
prometheus --version

# /etc/prometheus/prometheus.yml
# Konfigurasi dasar Prometheus

global:
  scrape_interval: 15s      # Scrape setiap 15 detik (default)
  evaluation_interval: 15s  # Evaluasi alert rules setiap 15 detik
  scrape_timeout: 10s       # Timeout untuk setiap scrape

# Alertmanager configuration
alerting:
  alertmanagers:
    - static_configs:
        - targets:
            - 'localhost:9093'

# Rules files
rule_files:
  - "alert_rules.yml"
  - "recording_rules.yml"

# Scrape targets
scrape_configs:
  # Monitor Prometheus sendiri
  - job_name: 'prometheus'
    static_configs:
      - targets: ['localhost:9090']

  # Monitor Node Exporter (server)
  - job_name: 'node-exporter'
    static_configs:
      - targets:
          - '192.168.1.100:9100'   # Server utama
          - '192.168.1.101:9100'   # Server backup
          - '192.168.1.102:9100'   # Gateway IoT

  # Monitor MQTT Broker (Mosquitto exporter)
  - job_name: 'mqtt-broker'
    static_configs:
      - targets: ['192.168.1.100:9234']

  # Monitor custom IoT metrics
  - job_name: 'iot-gateway'
    scrape_interval: 5s     # Scrape lebih sering untuk IoT
    static_configs:
      - targets: ['192.168.1.50:8000']
        labels:
          environment: 'production'
          location: 'rumah'

# docker-compose.yml
version: '3.8'

services:
  prometheus:
    image: prom/prometheus:latest
    container_name: prometheus
    ports:
      - "9090:9090"
    volumes:
      - ./prometheus.yml:/etc/prometheus/prometheus.yml
      - prometheus_data:/prometheus
    command:
      - '--config.file=/etc/prometheus/prometheus.yml'
      - '--storage.tsdb.retention.time=30d'
    restart: unless-stopped

  grafana:
    image: grafana/grafana:latest
    container_name: grafana
    ports:
      - "3000:3000"
    volumes:
      - grafana_data:/var/lib/grafana
    environment:
      - GF_SECURITY_ADMIN_PASSWORD=admin123
    restart: unless-stopped

  node-exporter:
    image: prom/node-exporter:latest
    container_name: node-exporter
    ports:
      - "9100:9100"
    restart: unless-stopped

volumes:
  prometheus_data:
  grafana_data:

# Jalankan:
# docker-compose up -d

# Akses:
# Prometheus: http://localhost:9090
# Grafana: http://localhost:3000 (admin/admin123)

# app.py — Custom IoT Metrics dengan Prometheus
# pip install prometheus_client flask

from prometheus_client import Counter, Gauge, Histogram, generate_latest
from flask import Flask, Response
import random
import time

app = Flask(__name__)

# === Counter: menghitung total pesan MQTT yang diterima ===
mqtt_messages_total = Counter(
    'iot_mqtt_messages_total',
    'Total pesan MQTT yang diterima',
    ['topic', 'device']  # Label
)

# === Gauge: menyimpan nilai sensor terkini ===
sensor_temperature = Gauge(
    'iot_sensor_temperature_celsius',
    'Suhu sensor saat ini dalam Celsius',
    ['device', 'room']
)

sensor_humidity = Gauge(
    'iot_sensor_humidity_percent',
    'Kelembaban sensor saat ini dalam persen',
    ['device', 'room']
)

# === Histogram: distribusi waktu pemrosesan pesan ===
message_processing_time = Histogram(
    'iot_message_processing_seconds',
    'Waktu pemrosesan pesan MQTT',
    ['topic'],
    buckets=[0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0]
)

# Endpoint untuk Prometheus scrape
@app.route('/metrics')
def metrics():
    return Response(generate_latest(), mimetype='text/plain')

# Simulasi menerima data sensor
@app.route('/api/sensor', methods=['POST'])
def receive_sensor():
    start = time.time()

    # Simulasi data
    device = "esp32-001"
    room = "ruang_tamu"
    suhu = round(random.uniform(25, 35), 1)
    kelembaban = round(random.uniform(40, 80), 1)

    # Update metrik
    mqtt_messages_total.labels(topic="rumah/suhu", device=device).inc()
    sensor_temperature.labels(device=device, room=room).set(suhu)
    sensor_humidity.labels(device=device, room=room).set(kelembaban)

    # Catat waktu pemrosesan
    duration = time.time() - start
    message_processing_time.labels(topic="rumah/suhu").observe(duration)

    return {"suhu": suhu, "kelembaban": kelembaban}

if __name__ == '__main__':
    app.run(host='0.0.0.0', port=8000)

# === Selector Dasar ===

# Ambil semua nilai metrik
node_cpu_seconds_total

# Filter berdasarkan label
node_cpu_seconds_total{mode="idle"}

# Filter dengan regex
node_cpu_seconds_total{mode=~"user|system"}

# Kecualikan label tertentu
node_cpu_seconds_total{mode!="idle"}

# Filter berdasarkan nama metrik
{__name__=~"node_.*"}

# === Instant Vector (nilai saat ini) ===
# Mengembalikan satu nilai terakhir untuk setiap time series
node_memory_MemAvailable_bytes

# === Range Vector (nilai dalam rentang waktu) ===
# Mengembalikan kumpulan nilai dalam periode waktu
http_requests_total[5m]          # 5 menit terakhir
node_cpu_seconds_total[1h]       # 1 jam terakhir
iot_sensor_temperature[15m]      # 15 menit terakhir

# === Rate & Increase (untuk Counter) ===

# Rate: rata-rata per detik dalam 5 menit terakhir
rate(http_requests_total[5m])

# Increase: total penambahan dalam 1 jam
increase(http_requests_total[1h])

# === Agregasi ===

# Sum total request semua instance
sum(rate(http_requests_total[5m]))

# Rata-rata suhu semua device
avg(iot_sensor_temperature_celsius)

# Suhu maksimum
max(iot_sensor_temperature_celsius)

# Suhu minimum per room
min by(room) (iot_sensor_temperature_celsius)

# Jumlah device unik
count(iot_sensor_temperature_celsius)

# Top 5 suhu tertinggi
topk(5, iot_sensor_temperature_celsius)

# === Matematika ===

# Konversi byte ke GB
node_memory_MemTotal_bytes / 1024 / 1024 / 1024

# Persentase penggunaan CPU
(1 - avg(rate(node_cpu_seconds_total{mode="idle"}[5m]))) * 100

# Persentase penggunaan memori
(1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100

# Suhu dalam Fahrenheit
iot_sensor_temperature_celsius * 9/5 + 32

# === Fungsi Waktu ===

# Data dari 1 jam yang lalu
iot_sensor_temperature_celsius offset 1h

# Perbandingan sekarang vs 1 hari lalu
iot_sensor_temperature_celsius - iot_sensor_temperature_celsius offset 1d

# === Predict (prediksi linear) ===

# Prediksi disk penuh dalam 4 jam ke dejang
predict_linear(node_filesystem_avail_bytes[6h], 4*3600)

# === Panel 1: CPU Usage (%) ===
# Query:
(1 - avg(rate(node_cpu_seconds_total{mode="idle"}[5m]))) * 100
# Grafik: Time Series, unit: percent (0-100)

# === Panel 2: Memory Usage (%) ===
# Query:
(1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100
# Grafik: Gauge, unit: percent

# === Panel 3: Disk Usage (%) ===
# Query:
(1 - node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) * 100
# Grafik: Gauge, threshold: warn=80%, crit=90%

# === Panel 4: Network Traffic ===
# Query (RX):
rate(node_network_receive_bytes_total{device="eth0"}[5m]) * 8
# Query (TX):
rate(node_network_transmit_bytes_total{device="eth0"}[5m]) * 8
# Grafik: Time Series, unit: bits/sec

# === Panel 5: IoT Sensor Temperature ===
# Query:
iot_sensor_temperature_celsius
# Grafik: Time Series, unit: celsius

# === Panel 6: MQTT Messages Rate ===
# Query:
sum(rate(iot_mqtt_messages_total[5m])) by(topic)
# Grafik: Time Series, unit: msg/sec

# === Panel 7: HTTP Request Rate ===
# Query:
sum(rate(http_requests_total[5m])) by(handler, status)
# Grafik: Stacked Area, unit: req/sec

# === Variabel: $device ===
# Type: Query
# Query: label_values(iot_sensor_temperature_celsius, device)
# Result: [esp32-001, esp32-002, esp32-003]

# === Variabel: $room ===
# Type: Query
# Query: label_values(iot_sensor_temperature_celsius{device="$device"}, room)
# Result: [ruang_tamu, kamar_tidur, dapur]

# === Penggunaan dalam query ===
iot_sensor_temperature_celsius{device="$device", room="$room"}

# User bisa memilih device dan room dari dropdown di dashboard
# sehingga satu dashboard bisa memantau semua device

# /etc/prometheus/alert_rules.yml

groups:
  - name: server_alerts
    rules:
      # Alert: CPU usage > 80% selama 5 menit
      - alert: HighCpuUsage
        expr: (1 - avg(rate(node_cpu_seconds_total{mode="idle"}[5m]))) * 100 > 80
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "CPU usage tinggi pada {{ $labels.instance }}"
          description: "CPU usage {{ $value | printf \"%.1f\" }}% selama 5 menit terakhir."

      # Alert: Memory usage > 90%
      - alert: HighMemoryUsage
        expr: (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 90
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "Memory hampir penuh pada {{ $labels.instance }}"
          description: "Memory usage {{ $value | printf \"%.1f\" }}%."

      # Alert: Disk akan penuh dalam 24 jam
      - alert: DiskWillFillIn24h
        expr: predict_linear(node_filesystem_avail_bytes{mountpoint="/"}[6h], 24*3600) < 0
        for: 30m
        labels:
          severity: critical
        annotations:
          summary: "Disk akan penuh dalam 24 jam pada {{ $labels.instance }}"

  - name: iot_alerts
    rules:
      # Alert: Suhu sensor terlalu tinggi
      - alert: HighTemperature
        expr: iot_sensor_temperature_celsius > 40
        for: 2m
        labels:
          severity: warning
        annotations:
          summary: "Suhu tinggi terdeteksi!"
          description: "Device {{ $labels.device }} di {{ $labels.room }}: {{ $value }}°C"

      # Alert: Device IoT offline (tidak ada metrik selama 1 menit)
      - alert: IoTDeviceOffline
        expr: up{job="iot-gateway"} == 0
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "Device IoT offline!"
          description: "Target {{ $labels.instance }} tidak merespons scrape."

      # Alert: MQTT broker down
      - alert: MQTTBrokerDown
        expr: up{job="mqtt-broker"} == 0
        for: 30s
        labels:
          severity: critical
        annotations:
          summary: "MQTT Broker tidak merespons!"

      # Alert: Rate pesan MQTT turun drastis
      - alert: LowMQTTMessageRate
        expr: sum(rate(iot_mqtt_messages_total[5m])) < 0.1
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "Rate pesan MQTT sangat rendah"
          description: "Kemungkinan device berhenti mengirim data."

# /etc/alertmanager/alertmanager.yml

global:
  smtp_smarthost: 'smtp.gmail.com:587'
  smtp_from: 'alert@beebanelabs.com'
  smtp_auth_username: 'alert@beebanelabs.com'
  smtp_auth_password: 'app-password-here'

route:
  group_by: ['alertname', 'severity']
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 4h
  receiver: 'default-receiver'

  routes:
    - match:
        severity: critical
      receiver: 'critical-receiver'
      repeat_interval: 15m

receivers:
  - name: 'default-receiver'
    email_configs:
      - to: 'admin@beebanelabs.com'
        subject: '[{{ .Status | toUpper }}] {{ .GroupLabels.alertname }}'

  - name: 'critical-receiver'
    email_configs:
      - to: 'admin@beebanelabs.com'
    webhook_configs:
      - url: 'http://localhost:5001/webhook/telegram'

# Download dan instal Node Exporter
NODE_EXP_VERSION="1.7.0"
wget https://github.com/prometheus/node_exporter/releases/download/v${NODE_EXP_VERSION}/node_exporter-${NODE_EXP_VERSION}.linux-amd64.tar.gz
tar xvfz node_exporter-${NODE_EXP_VERSION}.linux-amd64.tar.gz
sudo cp node_exporter-${NODE_EXP_VERSION}.linux-amd64/node_exporter /usr/local/bin/

# Buat systemd service
sudo tee /etc/systemd/system/node_exporter.service <<'EOF'
[Unit]
Description=Node Exporter
After=network.target

[Service]
User=prometheus
ExecStart=/usr/local/bin/node_exporter
Restart=always

[Install]
WantedBy=multi-user.target
EOF

# Jalankan
sudo systemctl daemon-reload
sudo systemctl enable node_exporter
sudo systemctl start node_exporter

# Verifikasi — buka di browser:
# http://server-ip:9100/metrics

# mqtt_exporter.py — Mengambil data MQTT dan mengekspos sebagai Prometheus metrics
# pip install prometheus_client paho-mqtt

from prometheus_client import start_http_server, Gauge, Counter
import paho.mqtt.client as mqtt
import json
import time

# === Prometheus Metrics ===
temperature = Gauge(
    'iot_sensor_temperature_celsius',
    'Suhu sensor',
    ['device', 'room']
)
humidity = Gauge(
    'iot_sensor_humidity_percent',
    'Kelembaban sensor',
    ['device', 'room']
)
messages_total = Counter(
    'iot_mqtt_messages_total',
    'Total pesan MQTT diterima',
    ['topic']
)
device_last_seen = Gauge(
    'iot_device_last_seen_timestamp',
    'Timestamp terakhir device mengirim data',
    ['device']
)

# === MQTT Callback ===
def on_connect(client, userdata, flags, rc, properties=None):
    print(f"[MQTT] Terhubung (rc={rc})")
    client.subscribe("iothub/sensor/#")

def on_message(client, userdata, msg):
    try:
        data = json.loads(msg.payload.decode('utf-8'))
        device = data.get('device', 'unknown')
        topic = msg.topic

        messages_total.labels(topic=topic).inc()

        if 'suhu' in data:
            temperature.labels(device=device, room=data.get('room', 'unknown')).set(data['suhu'])
        if 'kelembaban' in data:
            humidity.labels(device=device, room=data.get('room', 'unknown')).set(data['kelembaban'])

        device_last_seen.labels(device=device).set(time.time())
        print(f"[DATA] {device}: suhu={data.get('suhu')}, kelembaban={data.get('kelembaban')}")

    except Exception as e:
        print(f"[ERROR] {e}")

# === Main ===
if __name__ == '__main__':
    # Start Prometheus HTTP server di port 8000
    start_http_server(8000)
    print("[PROM] Metrics tersedia di http://localhost:8000/metrics")

    # Koneksi MQTT
    client = mqtt.Client(mqtt.CallbackAPIVersion.VERSION2)
    client.on_connect = on_connect
    client.on_message = on_message
    client.connect("localhost", 1883, 60)
    client.loop_forever()