OpenTelemetry: Observability untuk Developer

Data Flow dalam OpenTelemetry:

Application Code
  → OTel API (create spans, record metrics)
    → OTel SDK (process, sample, batch)
      → OTel Exporter (send data)
        → OTel Collector (optional: receive, process, export)
          → Backend (Jaeger, Prometheus, Grafana, dll)

Modes:
  1. Direct Export: App → Exporter → Backend
  2. Via Collector: App → Exporter → Collector → Backend (recommended)
  3. Via Agent: App → gRPC/HTTP → Collector Agent → Backend

// tracing.js — Setup OpenTelemetry tracing (load SEBELUM app code)
const { NodeSDK } = require('@opentelemetry/sdk-node');
const { getNodeAutoInstrumentations } = require('@opentelemetry/auto-instrumentations-node');
const { OTLPTraceExporter } = require('@opentelemetry/exporter-trace-otlp-http');
const { OTLPMetricExporter } = require('@opentelemetry/exporter-metrics-otlp-http');
const { PeriodicExportingMetricReader } = require('@opentelemetry/sdk-metrics');
const { Resource } = require('@opentelemetry/resources');
const { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } = require('@opentelemetry/semantic-conventions');

// Definisikan resource (service metadata)
const resource = new Resource({
  [ATTR_SERVICE_NAME]: 'user-service',
  [ATTR_SERVICE_VERSION]: '1.0.0',
  'deployment.environment': process.env.NODE_ENV || 'development',
});

// Setup trace exporter
const traceExporter = new OTLPTraceExporter({
  url: process.env.OTEL_EXPORTER_OTLP_ENDPOINT || 'http://localhost:4318/v1/traces',
});

// Setup metric exporter
const metricExporter = new OTLPMetricExporter({
  url: process.env.OTEL_EXPORTER_OTLP_ENDPOINT || 'http://localhost:4318/v1/metrics',
});

// Initialize SDK
const sdk = new NodeSDK({
  resource,
  traceExporter,
  metricReader: new PeriodicExportingMetricReader({
    exporter: metricExporter,
    exportIntervalMillis: 15000,
  }),
  instrumentations: [
    getNodeAutoInstrumentations({
      // Konfigurasi instrumentasi
      '@opentelemetry/instrumentation-http': {
        enabled: true,
      },
      '@opentelemetry/instrumentation-express': {
        enabled: true,
      },
      '@opentelemetry/instrumentation-pg': {
        enabled: true,
      },
      '@opentelemetry/instrumentation-redis': {
        enabled: true,
      },
    }),
  ],
});

// Start SDK
sdk.start();
console.log('OpenTelemetry SDK initialized');

// Graceful shutdown
process.on('SIGTERM', () => {
  sdk.shutdown()
    .then(() => console.log('OpenTelemetry SDK shut down'))
    .catch((err) => console.error('Error shutting down SDK:', err))
    .finally(() => process.exit(0));
});

// app.js — Express app dengan OpenTelemetry
// PENTING: require('./tracing') harus di-import PALING PERTAMA
require('./tracing');

const express = require('express');
const { trace, SpanStatusCode } = require('@opentelemetry/api');
const app = express();

// Dapatkan tracer untuk service ini
const tracer = trace.getTracer('user-service', '1.0.0');

app.get('/users/:id', async (req, res) => {
  // Auto-instrumentation sudah membuat span untuk request ini
  // Tapi kita bisa menambahkan custom spans

  const span = tracer.startSpan('fetch-user-data', {
    attributes: {
      'user.id': req.params.id,
      'http.method': req.method,
    },
  });

  try {
    // Simulate database call
    const user = await tracer.startActiveSpan('database.query', async (dbSpan) => {
      dbSpan.setAttribute('db.system', 'postgresql');
      dbSpan.setAttribute('db.statement', 'SELECT * FROM users WHERE id = ?');

      const result = await db.getUser(req.params.id);

      dbSpan.setAttribute('db.rows_affected', result.rowCount);
      dbSpan.end();
      return result;
    });

    // Tambahkan event ke span
    span.addEvent('user_found', {
      'user.name': user.name,
      'user.email': user.email,
    });

    span.setStatus({ code: SpanStatusCode.OK });
    res.json(user);
  } catch (error) {
    // Record error di span
    span.recordException(error);
    span.setStatus({
      code: SpanStatusCode.ERROR,
      message: error.message,
    });
    res.status(500).json({ error: error.message });
  } finally {
    span.end();
  }
});

app.listen(3000, () => {
  console.log('Server running on port 3000');
});

# tracing.py — Setup OpenTelemetry untuk Python
from opentelemetry import trace
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
from opentelemetry.sdk.resources import Resource
from opentelemetry.semantics.resource import ResourceAttributes
from opentelemetry.instrumentation.flask import FlaskInstrumentor
from opentelemetry.instrumentation.requests import RequestsInstrumentor
from opentelemetry.instrumentation.psycopg2 import Psycopg2Instrumentor

# Definisikan resource
resource = Resource.create({
    ResourceAttributes.SERVICE_NAME: "python-api",
    ResourceAttributes.SERVICE_VERSION: "1.0.0",
    "deployment.environment": "production",
})

# Setup provider dan exporter
provider = TracerProvider(resource=resource)
processor = BatchSpanProcessor(OTLPSpanExporter(
    endpoint="http://localhost:4318/v1/traces"
))
provider.add_span_processor(processor)
trace.set_tracer_provider(provider)

# Auto-instrumentasi
FlaskInstrumentor().instrument()
RequestsInstrumentor().instrument()
Psycopg2Instrumentor().instrument()

# app.py — Flask app dengan custom spans
from flask import Flask, jsonify
from opentelemetry import trace

app = Flask(__name__)
tracer = trace.get_tracer("python-api")

@app.route('/api/users/')
def get_user(user_id):
    with tracer.start_as_current_span("get-user-details") as span:
        span.set_attribute("user.id", user_id)

        # Database query (auto-instrumented)
        with tracer.start_as_current_span("db-query") as db_span:
            db_span.set_attribute("db.system", "postgresql")
            user = db.execute("SELECT * FROM users WHERE id = %s", [user_id])

        span.add_event("user_fetched", {"user.id": user_id})
        return jsonify(user)

if __name__ == '__main__':
    app.run(host='0.0.0.0', port=5000)

// metrics.js — Custom metrics dengan OpenTelemetry
const { metrics } = require('@opentelemetry/api');

// Dapatkan meter
const meter = metrics.getMeter('user-service', '1.0.0');

// 1. Counter — total requests
const requestCounter = meter.createCounter('http.server.requests', {
  description: 'Total HTTP requests',
  unit: '1',
});

// 2. Histogram — request duration
const requestDuration = meter.createHistogram('http.server.duration', {
  description: 'HTTP request duration',
  unit: 'ms',
});

// 3. Gauge — active connections
const activeConnections = meter.createUpDownCounter('http.server.active_connections', {
  description: 'Number of active connections',
  unit: '1',
});

// 4. Custom business metric
const orderCounter = meter.createCounter('business.orders.created', {
  description: 'Total orders created',
  unit: '1',
});

const orderValue = meter.createHistogram('business.orders.value', {
  description: 'Order value distribution',
  unit: 'USD',
});

// Menggunakan metrics di dalam kode
app.use((req, res, next) => {
  const startTime = Date.now();

  // Increment active connections
  activeConnections.add(1, { method: req.method });

  res.on('finish', () => {
    const duration = Date.now() - startTime;

    // Record request
    requestCounter.add(1, {
      method: req.method,
      route: req.route?.path || 'unknown',
      status_code: res.statusCode,
    });

    // Record duration
    requestDuration.record(duration, {
      method: req.method,
      route: req.route?.path || 'unknown',
    });

    // Decrement active connections
    activeConnections.add(-1, { method: req.method });
  });

  next();
});

// Record business metrics
app.post('/api/orders', async (req, res) => {
  const order = await createOrder(req.body);

  orderCounter.add(1, {
    category: order.category,
    region: order.region,
  });

  orderValue.record(order.total, {
    category: order.category,
  });

  res.json(order);
});

// logging.js — Logs dengan OpenTelemetry correlation
const { logs, SeverityNumber } = require('@opentelemetry/api-logs');
const { trace } = require('@opentelemetry/api');

const logger = logs.getLogger('user-service');

function logWithContext(level, message, attributes = {}) {
  const span = trace.getActiveSpan();
  const spanContext = span?.spanContext();

  logger.emit({
    severityText: level,
    severityNumber: level === 'ERROR' ? SeverityNumber.ERROR
      : level === 'WARN' ? SeverityNumber.WARN
      : SeverityNumber.INFO,
    body: message,
    attributes: {
      ...attributes,
      // OTel SDK otomatis menambahkan trace_id dan span_id
      // jika log di-emit dari dalam span context
    },
  });
}

// Contoh penggunaan
app.get('/api/users/:id', async (req, res) => {
  const span = trace.getActiveSpan();

  logWithContext('INFO', 'Processing user request', {
    'user.id': req.params.id,
    'http.method': req.method,
  });

  try {
    const user = await getUser(req.params.id);
    logWithContext('INFO', 'User fetched successfully', {
      'user.id': user.id,
      'user.name': user.name,
    });
    res.json(user);
  } catch (error) {
    logWithContext('ERROR', 'Failed to fetch user', {
      'error.message': error.message,
      'error.stack': error.stack,
    });
    res.status(500).json({ error: error.message });
  }
});

{
  "timestamp": "2026-06-26T10:00:00.123Z",
  "severityText": "INFO",
  "severityNumber": 9,
  "body": "User fetched successfully",
  "resource": {
    "service.name": "user-service",
    "service.version": "1.0.0",
    "deployment.environment": "production"
  },
  "attributes": {
    "user.id": "12345",
    "user.name": "Budi Santoso",
    "trace_id": "abc123def45678901234567890123456",
    "span_id": "def4567890123456"
  }
}

// Dengan trace_id dan span_id, Anda bisa:
// 1. Click dari log ke trace di Jaeger/Grafana
// 2. Filter logs berdasarkan trace_id
// 3. Melihat semua log dalam satu request

# Java Agent — Zero code changes!
# Download otel java agent
curl -LO https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/latest/download/opentelemetry-javaagent.jar

# Jalankan aplikasi dengan agent
java -javaagent:opentelemetry-javaagent.jar \
  -Dotel.service.name=my-java-app \
  -Dotel.exporter.otlp.endpoint=http://localhost:4317 \
  -jar my-app.jar

# Environment variables configuration
export OTEL_SERVICE_NAME=my-java-app
export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317
export OTEL_TRACES_SAMPLER=parentbased_traceidratio
export OTEL_TRACES_SAMPLER_ARG=0.1
export OTEL_RESOURCE_ATTRIBUTES="service.version=1.0.0,deployment.env=prod"

java -javaagent:opentelemetry-javaagent.jar -jar my-app.jar

# Python — Auto-instrumentasi via opentelemetry-instrument CLI
pip install opentelemetry-distro opentelemetry-exporter-otlp
opentelemetry-bootstrap -a install

# Jalankan dengan auto-instrumentasi
opentelemetry-instrument \
  --service_name my-python-app \
  --exporter_otlp_endpoint http://localhost:4318 \
  python my_app.py

# Atau dengan gunicorn
opentelemetry-instrument \
  --service_name my-python-app \
  gunicorn -w 4 -b 0.0.0.0:8000 myapp:app

// Manual instrumentation untuk business logic
const { trace, context, SpanKind, SpanStatusCode } = require('@opentelemetry/api');

const tracer = trace.getTracer('order-service', '1.0.0');

async function processOrder(orderData) {
  // Buat span parent untuk operasi bisnis
  return tracer.startActiveSpan(
    'process-order',
    {
      kind: SpanKind.INTERNAL,
      attributes: {
        'order.id': orderData.id,
        'order.customer_id': orderData.customerId,
        'order.item_count': orderData.items.length,
        'order.total': orderData.total,
      },
    },
    async (parentSpan) => {
      try {
        // 1. Validasi order
        const validatedOrder = await tracer.startActiveSpan(
          'validate-order',
          async (span) => {
            span.setAttribute('order.items', JSON.stringify(orderData.items));
            const result = await validateOrder(orderData);
            span.addEvent('validation_passed');
            span.end();
            return result;
          }
        );

        // 2. Proses pembayaran
        const payment = await tracer.startActiveSpan(
          'process-payment',
          {
            attributes: {
              'payment.method': orderData.paymentMethod,
              'payment.amount': orderData.total,
              'payment.currency': 'IDR',
            },
          },
          async (span) => {
            try {
              const result = await paymentService.charge(orderData);
              span.setAttribute('payment.transaction_id', result.transactionId);
              span.setStatus({ code: SpanStatusCode.OK });
              span.end();
              return result;
            } catch (err) {
              span.recordException(err);
              span.setStatus({ code: SpanStatusCode.ERROR, message: err.message });
              span.end();
              throw err;
            }
          }
        );

        // 3. Kirim notifikasi
        await tracer.startActiveSpan('send-notification', async (span) => {
          span.setAttribute('notification.type', 'order_confirmation');
          span.setAttribute('notification.channel', 'email');
          await sendEmail(orderData.customerId, 'Order confirmed');
          span.addEvent('notification_sent');
          span.end();
        });

        parentSpan.setStatus({ code: SpanStatusCode.OK });
        return { success: true, orderId: orderData.id };
      } catch (error) {
        parentSpan.recordException(error);
        parentSpan.setStatus({ code: SpanStatusCode.ERROR, message: error.message });
        throw error;
      } finally {
        parentSpan.end();
      }
    }
  );
}

# docker-compose.yml — Full Grafana Observability Stack
version: '3.8'
services:
  # OpenTelemetry Collector
  otel-collector:
    image: otel/opentelemetry-collector-contrib:latest
    command: ["--config=/etc/otel-collector-config.yaml"]
    volumes:
      - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml
    ports:
      - "4317:4317"   # gRPC
      - "4318:4318"   # HTTP

  # Jaeger — Traces
  jaeger:
    image: jaegertracing/all-in-one:latest
    environment:
      COLLECTOR_OTLP_ENABLED: "true"
    ports:
      - "16686:16686"  # Jaeger UI
      - "14250:14250"  # gRPC

  # Prometheus — Metrics
  prometheus:
    image: prom/prometheus:latest
    volumes:
      - ./prometheus.yml:/etc/prometheus/prometheus.yml
    ports:
      - "9090:9090"    # Prometheus UI

  # Grafana — Dashboard
  grafana:
    image: grafana/grafana:latest
    environment:
      GF_SECURITY_ADMIN_PASSWORD: admin
    ports:
      - "3000:3000"    # Grafana UI
    volumes:
      - grafana-storage:/var/lib/grafana

  # Loki — Logs
  loki:
    image: grafana/loki:latest
    ports:
      - "3100:3100"

  # Tempo — Traces (alternatif Jaeger)
  tempo:
    image: grafana/tempo:latest
    ports:
      - "3200:3200"

volumes:
  grafana-storage:

# otel-collector-config.yaml
receivers:
  otlp:
    protocols:
      grpc:
        endpoint: "0.0.0.0:4317"
      http:
        endpoint: "0.0.0.0:4318"

processors:
  batch:
    timeout: 5s
    send_batch_size: 1024

  memory_limiter:
    check_interval: 1s
    limit_mib: 512
    spike_limit_mib: 128

  # Tail-based sampling — keep errors and slow requests
  tail_sampling:
    decision_wait: 10s
    policies:
      - name: error-policy
        type: status_code
        status_code: {status_codes: [ERROR]}
      - name: slow-request-policy
        type: latency
        latency: {threshold_ms: 1000}
      - name: probabilistic-policy
        type: probabilistic
        probabilistic: {sampling_percentage: 10}

exporters:
  # Traces → Jaeger
  otlp/jaeger:
    endpoint: "jaeger:4317"
    tls:
      insecure: true

  # Metrics → Prometheus
  prometheus:
    endpoint: "0.0.0.0:8889"

  # Logs → Loki
  loki:
    endpoint: "http://loki:3100/loki/api/v1/push"

  # Debug — print ke stdout
  debug:
    verbosity: basic

service:
  pipelines:
    traces:
      receivers: [otlp]
      processors: [memory_limiter, batch, tail_sampling]
      exporters: [otlp/jaeger]
    metrics:
      receivers: [otlp]
      processors: [memory_limiter, batch]
      exporters: [prometheus]
    logs:
      receivers: [otlp]
      processors: [memory_limiter, batch]
      exporters: [loki]