ELK Stack: Logging Platform

# Contoh arsitektur production multi-node
#
# Master Nodes (3): Cluster coordination, tidak menyimpan data
# Data Nodes (3+): Menyimpan index shards, melakukan search
# Coordinating Nodes (2): Load balancer untuk query
# Logstash Nodes (2): Data processing pipeline
# Kibana Node (1): Dashboard UI
#
# Minimum production setup:
# - 3 master-eligible nodes (bisa juga data node)
# - Logstash cluster (min 2 node)
# - 1 Kibana instance
# - Load balancer di depan

# Elasticsearch node roles di elasticsearch.yml:
node.roles: [ master, data, ingest ]
# Atau terpisah:
# node.roles: [ master ]
# node.roles: [ data_hot ]
# node.roles: [ data_warm ]
# node.roles: [ ingest ]

# Pastikan Elasticsearch berjalan
curl -s http://localhost:9200

# === CREATE INDEX ===
curl -X PUT "http://localhost:9200/app-logs" -H 'Content-Type: application/json' -d '{
  "settings": {
    "number_of_shards": 2,
    "number_of_replicas": 1,
    "refresh_interval": "5s"
  },
  "mappings": {
    "properties": {
      "timestamp": { "type": "date" },
      "level": { "type": "keyword" },
      "message": { "type": "text" },
      "service": { "type": "keyword" },
      "host": { "type": "keyword" },
      "response_time_ms": { "type": "integer" },
      "user_id": { "type": "keyword" },
      "request_path": { "type": "keyword" },
      "stack_trace": { "type": "text" }
    }
  }
}'

# === INDEX DOCUMENT ===
curl -X POST "http://localhost:9200/app-logs/_doc" -H 'Content-Type: application/json' -d '{
  "timestamp": "2026-06-26T10:30:00Z",
  "level": "ERROR",
  "message": "Failed to connect to database",
  "service": "user-api",
  "host": "web-01",
  "response_time_ms": 5000,
  "user_id": "U123",
  "request_path": "/api/users",
  "stack_trace": "ConnectionError: ECONNREFUSED 10.0.0.5:5432"
}'

# === SEARCH ===
# Cari semua ERROR di service user-api
curl -X GET "http://localhost:9200/app-logs/_search" -H 'Content-Type: application/json' -d '{
  "query": {
    "bool": {
      "must": [
        { "match": { "level": "ERROR" } },
        { "term": { "service": "user-api" } }
      ],
      "filter": [
        { "range": { "timestamp": { "gte": "now-1h" } } }
      ]
    }
  },
  "sort": [{ "timestamp": "desc" }],
  "size": 20
}'

# === AGGREGATION ===
# Hitung jumlah log per level
curl -X GET "http://localhost:9200/app-logs/_search" -H 'Content-Type: application/json' -d '{
  "size": 0,
  "aggs": {
    "log_levels": {
      "terms": { "field": "level", "size": 10 }
    },
    "avg_response_time": {
      "avg": { "field": "response_time_ms" }
    },
    "errors_over_time": {
      "date_histogram": {
        "field": "timestamp",
        "calendar_interval": "hour"
      },
      "aggs": {
        "error_count": {
          "filter": { "term": { "level": "ERROR" } }
        }
      }
    }
  }
}'

# === UPDATE ===
curl -X POST "http://localhost:9200/app-logs/_update/doc_id" -H 'Content-Type: application/json' -d '{
  "doc": { "level": "WARN" }
}'

# === DELETE ===
curl -X DELETE "http://localhost:9200/app-logs/_doc/doc_id"

# === DELETE INDEX ===
curl -X DELETE "http://localhost:9200/app-logs"

# logstash.conf — Pipeline konfigurasi

# === INPUT: Sumber data ===
input {
  # Terima dari Filebeat
  beats {
    port => 5044
  }
  
  # Baca file langsung
  file {
    path => "/var/log/nginx/access.log"
    start_position => "beginning"
    sincedb_path => "/dev/null"
  }
  
  # HTTP endpoint
  http {
    port => 8080
    codec => json
  }
  
  # Kafka consumer
  kafka {
    bootstrap_servers => "kafka:9092"
    topics => ["app-logs", "system-logs"]
    group_id => "logstash-consumer"
  }
}

# === FILTER: Transformasi data ===
filter {
  # Parse nginx access log
  if [fields][log_type] == "nginx" {
    grok {
      match => { "message" => '%{IPORHOST:client_ip} - %{DATA:user} \[%{HTTPDATE:timestamp}\] "%{WORD:method} %{URIPATHPARAM:request} HTTP/%{NUMBER:http_version}" %{NUMBER:status} %{NUMBER:bytes} "%{DATA:referrer}" "%{DATA:user_agent}" %{NUMBER:response_time}' }
    }
    
    mutate {
      convert => {
        "status" => "integer"
        "bytes" => "integer"
        "response_time" => "float"
      }
    }
    
    date {
      match => [ "timestamp", "dd/MMM/yyyy:HH:mm:ss Z" ]
      target => "@timestamp"
    }
    
    geoip {
      source => "client_ip"
      target => "geoip"
    }
    
    useragent {
      source => "user_agent"
      target => "ua"
    }
  }
  
  # Parse JSON log dari aplikasi
  if [fields][log_type] == "application" {
    json {
      source => "message"
      target => "app"
    }
    
    # Tambahkan field derived
    mutate {
      add_field => { "service_name" => "%{[app][service]}" }
    }
  }
  
  # Enrich semua log
  mutate {
    add_field => { "environment" => "production" }
    remove_field => [ "agent", "ecs", "host" ]
  }
  
  # Drop DEBUG log untuk menghemat storage
  if [level] == "DEBUG" {
    drop { }
  }
}

# === OUTPUT: Tujuan data ===
output {
  # Kirim ke Elasticsearch
  elasticsearch {
    hosts => ["http://elasticsearch:9200"]
    index => "app-logs-%{+YYYY.MM.dd}"
    user => "elastic"
    password => "changeme"
  }
  
  # Debug: print ke stdout (development only)
  stdout {
    codec => rubydebug
  }
  
  # Kirim error ke Slack
  if [level] == "ERROR" {
    http {
      url => "https://hooks.slack.com/services/xxx/yyy/zzz"
      http_method => "post"
      format => "json"
      content_type => "application/json"
      body => '{"text": "🚨 ERROR in %{service_name}: %{message}"}'
    }
  }
}

# Contoh Grok Patterns

# Input: "2026-06-26 10:30:45 ERROR [user-api] Connection timeout"
# Pattern:
%{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:level} \[%{DATA:service}\] %{GREEDYDATA:message}

# Input: "192.168.1.1 - admin [26/Jun/2026:10:30:45 +0700] "GET /api/users HTTP/1.1" 200 1234"
# Pattern:
%{IPORHOST:client_ip} - %{DATA:user} \[%{HTTPDATE:timestamp}\] "%{WORD:method} %{URIPATHPARAM:path} HTTP/%{NUMBER:version}" %{NUMBER:status:int} %{NUMBER:bytes:int}

# Common Grok Patterns:
# %{IPORHOST}   → IP atau hostname
# %{NUMBER}      → Angka (integer/float)
# %{WORD}        → Kata tunggal
# %{DATA}        → Data apapun (non-greedy)
# %{GREEDYDATA}  → Sisa dari baris
# %{TIMESTAMP_ISO8601} → ISO 8601 timestamp
# %{LOGLEVEL}    → DEBUG/INFO/WARN/ERROR/FATAL

# Langkah-langkah menggunakan Kibana:

1. BUAT DATA VIEW (sebelumnya: Index Pattern)
   - Buka: Kibana → Stack Management → Data Views
   - Klik "Create data view"
   - Name: "app-logs"
   - Pattern: "app-logs-*" (sesuaikan dengan index name)
   - Timestamp field: "@timestamp"
   - Klik "Save data view"

2. EKSPLORASI LOG (Discover)
   - Buka: Kibana → Discover
   - Pilih data view "app-logs"
   - Set time range: misal "Last 24 hours"
   - KQL Query examples:
     * level: "ERROR"
     * service: "user-api" and level: "ERROR"
     * message: "timeout" and response_time_ms > 3000
     * NOT level: "DEBUG"

3. BUAT VISUALISASI (Lens)
   - Buka: Kibana → Dashboard → Create dashboard
   - Tambah visualisasi:
     * Bar chart: Count log per service
     * Line chart: Error rate over time
     * Pie chart: Log distribution by level
     * Data table: Top 10 error messages
     * Metric: Total errors (last hour)

4. KQL (Kibana Query Language) — Contoh:
     * service.user-api                     → field match
     * level: "ERROR"                       → exact value
     * message: "timeout"                   → full text search
     * response_time_ms > 1000              → range
     * level: ("ERROR" or "WARN")           → OR
     * service: "api" and NOT level: "INFO" → AND + NOT
     * service: user-*                      → wildcard

# filebeat.yml

# === INPUT: Baca file log ===
filebeat.inputs:
  # Application logs
  - type: log
    enabled: true
    paths:
      - /var/log/myapp/*.log
    fields:
      log_type: application
    fields_under_root: true
    multiline.pattern: '^\d{4}-\d{2}-\d{2}'  # Multi-line stack trace
    multiline.negate: true
    multiline.match: after

  # Nginx access log
  - type: log
    enabled: true
    paths:
      - /var/log/nginx/access.log
    fields:
      log_type: nginx
    fields_under_root: true

  # System logs
  - type: log
    enabled: true
    paths:
      - /var/log/syslog
      - /var/log/auth.log
    fields:
      log_type: system
    fields_under_root: true

# === MODULES: Built-in log parsing ===
filebeat.modules:
  - module: nginx
    access:
      enabled: true
      var.paths: ["/var/log/nginx/access.log"]
    error:
      enabled: true
      var.paths: ["/var/log/nginx/error.log"]
  
  - module: system
    syslog:
      enabled: true
    auth:
      enabled: true

# === PROCESSORS: Transformasi ringan ===
processors:
  - add_host_metadata: ~
  - add_cloud_metadata: ~
  - drop_event:
      when:
        and:
          - equals:
              message: "healthcheck"
  - dissect:
      tokenizer: "%{timestamp} %{level} %{message}"
      field: "message"
      target_prefix: "parsed"

# === OUTPUT: Kirim ke Logstash (atau langsung ke ES) ===
# Opsi 1: Kirim ke Logstash (rekomendasi)
output.logstash:
  hosts: ["logstash-01:5044", "logstash-02:5044"]
  loadbalance: true
  bulk_max_size: 2048

# Opsi 2: Kirim langsung ke Elasticsearch
# output.elasticsearch:
#   hosts: ["http://elasticsearch:9200"]
#   index: "filebeat-%{[log_type]}-%{+yyyy.MM.dd}"
#   username: "elastic"
#   password: "changeme"

# === LOGGING ===
logging.level: info
logging.to_files: true
logging.files:
  path: /var/log/filebeat
  name: filebeat
  keepfiles: 7
  permissions: 0640

# Install Filebeat di Ubuntu/Debian
wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add -
echo "deb https://artifacts.elastic.co/packages/8.x/apt stable main" | sudo tee /etc/apt/sources.list.d/elastic-8.x.list
sudo apt update && sudo apt install filebeat

# Enable modules
sudo filebeat modules enable nginx system

# Setup dashboards Kibana
sudo filebeat setup --dashboards

# Start Filebeat
sudo systemctl enable filebeat
sudo systemctl start filebeat

# Cek status
sudo systemctl status filebeat

# Test konfigurasi
sudo filebeat test config -c /etc/filebeat/filebeat.yml
sudo filebeat test output

# docker-compose.yml — ELK Stack dengan Filebeat
version: '3.8'

services:
  # === Elasticsearch ===
  elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch:8.13.0
    container_name: elasticsearch
    environment:
      - discovery.type=single-node
      - xpack.security.enabled=false  # Development only!
      - xpack.security.http.ssl.enabled=false
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
    ports:
      - "9200:9200"
      - "9300:9300"
    volumes:
      - es-data:/usr/share/elasticsearch/data
    networks:
      - elk
    healthcheck:
      test: ["CMD-SHELL", "curl -f http://localhost:9200/_cluster/health || exit 1"]
      interval: 30s
      timeout: 10s
      retries: 5

  # === Logstash ===
  logstash:
    image: docker.elastic.co/logstash/logstash:8.13.0
    container_name: logstash
    ports:
      - "5044:5044"   # Beats input
      - "8080:8080"   # HTTP input
    volumes:
      - ./logstash/pipeline:/usr/share/logstash/pipeline
      - ./logstash/config/logstash.yml:/usr/share/logstash/config/logstash.yml
    environment:
      - "LS_JAVA_OPTS=-Xms256m -Xmx256m"
    networks:
      - elk
    depends_on:
      elasticsearch:
        condition: service_healthy

  # === Kibana ===
  kibana:
    image: docker.elastic.co/kibana/kibana:8.13.0
    container_name: kibana
    ports:
      - "5601:5601"
    environment:
      - ELASTICSEARCH_HOSTS=http://elasticsearch:9200
    networks:
      - elk
    depends_on:
      elasticsearch:
        condition: service_healthy

  # === Filebeat ===
  filebeat:
    image: docker.elastic.co/beats/filebeat:8.13.0
    container_name: filebeat
    user: root
    volumes:
      - ./filebeat/filebeat.yml:/usr/share/filebeat/filebeat.yml:ro
      - /var/log:/var/log:ro
      - filebeat-data:/usr/share/filebeat/data
    networks:
      - elk
    depends_on:
      - elasticsearch

volumes:
  es-data:
  filebeat-data:

networks:
  elk:
    driver: bridge

# Jalankan ELK Stack
docker compose up -d

# Cek semua services
docker compose ps

# Lihat Elasticsearch cluster health
curl http://localhost:9200/_cluster/health?pretty

# Lihat semua index
curl http://localhost:9200/_cat/indices?v

# Buka Kibana
open http://localhost:5601

# Lihat log Logstash
docker compose logs -f logstash

# Lihat log Filebeat
docker compose logs -f filebeat

# Stop dan hapus semua data
docker compose down -v

# Membuat ILM policy
curl -X PUT "http://localhost:9200/_ilm/policy/app-logs-policy" -H 'Content-Type: application/json' -d '{
  "policy": {
    "phases": {
      "hot": {
        "min_age": "0ms",
        "actions": {
          "rollover": {
            "max_primary_shard_size": "50gb",
            "max_age": "1d"
          },
          "set_priority": { "priority": 100 }
        }
      },
      "warm": {
        "min_age": "3d",
        "actions": {
          "shrink": { "number_of_shards": 1 },
          "forcemerge": { "max_num_segments": 1 },
          "set_priority": { "priority": 50 }
        }
      },
      "cold": {
        "min_age": "30d",
        "actions": {
          "set_priority": { "priority": 0 },
          "freeze": {}
        }
      },
      "delete": {
        "min_age": "90d",
        "actions": {
          "delete": {}
        }
      }
    }
  }
}'

# Buat index template dengan ILM policy
curl -X PUT "http://localhost:9200/_index_template/app-logs-template" -H 'Content-Type: application/json' -d '{
  "index_patterns": ["app-logs-*"],
  "template": {
    "settings": {
      "number_of_shards": 2,
      "number_of_replicas": 1,
      "index.lifecycle.name": "app-logs-policy",
      "index.lifecycle.rollover_alias": "app-logs"
    }
  }
}'

# Buat initial index dengan alias
curl -X PUT "http://localhost:9200/app-logs-000001" -H 'Content-Type: application/json' -d '{
  "aliases": {
    "app-logs": {
      "is_write_index": true
    }
  }
}'

# Cek ILM status
curl "http://localhost:9200/app-logs-*/_ilm/explain?pretty"

# Force rollover (untuk testing)
curl -X POST "http://localhost:9200/app-logs/_rollover"

# Membuat alert: Error rate > 100 dalam 5 menit

# 1. Buat connector (Slack webhook)
curl -X POST "http://localhost:5601/api/actions/connector" \
  -H 'kbn-xsrf: true' \
  -H 'Content-Type: application/json' \
  -d '{
    "name": "Slack Alerts",
    "connector_type_id": ".slack",
    "config": {
      "webhookUrl": "https://hooks.slack.com/services/xxx/yyy/zzz"
    }
  }'

# 2. Buat alert rule (via Kibana UI lebih mudah)
# Kibana → Stack Management → Rules → Create Rule
# 
# Rule type: Elasticsearch query
# Index: app-logs-*
# Query: level: "ERROR"
# Time window: 5 minutes
# Threshold: count > 100
# Action: Send to Slack dengan message template

# Monitoring cluster health
curl "http://localhost:9200/_cluster/health?pretty"
curl "http://localhost:9200/_cat/nodes?v&h=name,heap.percent,ram.percent,cpu,load_1m,node.role"
curl "http://localhost:9200/_cat/indices?v&s=store.size:desc&h=index,health,pri,rep,docs.count,store.size"
curl "http://localhost:9200/_cat/shards?v&h=index,shard,prirep,state,docs,store,node"