Correlate Traces and Logs with OpenTelemetry

# Add Grafana Helm charts
helm repo add grafana https://grafana.github.io/helm-charts
helm repo update

# Install Tempo in monolithic mode (development setup)
helm install tempo grafana/tempo \
  --namespace monitoring \
  --set tempo.storage.trace.backend=local \
  --wait

# Install Loki for logs
helm install loki grafana/loki \
  --namespace monitoring \
  --set loki.commonConfig.replication_factor=1 \
  --set loki.storage.type=filesystem \
  --wait

kubectl get pods -n monitoring | grep -E "tempo|loki"

# Install OTel Operator first
helm install opentelemetry-operator \
  open-telemetry/opentelemetry-operator \
  --namespace monitoring

# Create Collector configuration
cat << 'EOF' | kubectl apply -f -
apiVersion: opentelemetry.io/v1alpha1
kind: OpenTelemetryCollector
metadata:
  name: otel-gateway
  namespace: monitoring
spec:
  mode: Deployment
  config: |
    receivers:
      otlp:
        protocols:
          grpc:
            endpoint: 0.0.0.0:4317
          http:
            endpoint: 0.0.0.0:4318

    processors:
      batch:
        timeout: 1s
      memory_limiter:
        check_interval: 1s
        limit_mib: 400

    exporters:
      otlp/tempo:
        endpoint: tempo.monitoring:4317
        tls:
          insecure: true
      loki:
        endpoint: http://loki.monitoring:3100/loki/api/v1/push
        default_labels_enabled:
          exporter: false
          job: true

    service:
      pipelines:
        traces:
          receivers: [otlp]
          processors: [memory_limiter, batch]
          exporters: [otlp/tempo]
        logs:
          receivers: [otlp]
          processors: [memory_limiter, batch]
          exporters: [loki]
EOF

# Deploy OTel demo app (astronomy shop)
helm install otel-demo open-telemetry/opentelemetry-demo \
  --namespace otel-demo \
  --create-namespace \
  --set components.frontendProxy.service.type=ClusterIP \
  --set opentelemetry-collector.enabled=false \
  --set "default.env[0].name=OTEL_EXPORTER_OTLP_ENDPOINT" \
  --set "default.env[0].value=http://otel-gateway-collector.monitoring:4317" \
  --wait --timeout 5m

kubectl get pods -n otel-demo

# Add Tempo data source to Grafana via ConfigMap
cat << 'EOF' | kubectl apply -f -
apiVersion: v1
kind: ConfigMap
metadata:
  name: grafana-datasources-tracing
  namespace: monitoring
  labels:
    grafana_datasource: "1"
data:
  tracing-datasources.yaml: |
    apiVersion: 1
    datasources:
    - name: Tempo
      type: tempo
      url: http://tempo.monitoring:3100
      jsonData:
        tracesToLogsV2:
          datasourceUid: loki
          filterByTraceID: true
          filterBySpanID: false
        serviceMap:
          datasourceUid: prometheus
    - name: Loki
      type: loki
      url: http://loki.monitoring:3100
      jsonData:
        derivedFields:
        - name: TraceID
          matcherRegex: '"traceId":"(\w+)"'
          url: '$${__value.raw}'
          datasourceUid: tempo
EOF

# Port-forward to the demo app frontend
kubectl port-forward svc/otel-demo-frontendproxy \
  -n otel-demo 8080:8080 &

# Generate some traffic including errors
for i in $(seq 1 50); do
  curl -s http://localhost:8080/api/products >/dev/null &
done
wait

# In Grafana → Explore → Tempo data source
# Search for: Service = "frontend" with status = "error"
# Or run TraceQL query:
# { .http.status_code = 500 }

# Find a failing trace and note the TraceID
# Example: 4a3b2c1d0e9f8a7b6c5d4e3f2a1b0c9d

# In Grafana Tempo trace view:
# 1. Click on a span in the trace
# 2. Click "Logs for this span" (appears when Loki is configured)
# 3. Grafana auto-queries Loki using the traceId

# Alternatively, search Loki directly with the TraceID:
# In Grafana → Explore → Loki data source
# Query: {job="otel-demo/frontend"} | json | traceId="4a3b2c1d..."

# Via API:
TRACE_ID="your-trace-id-here"
kubectl exec -n monitoring \
  $(kubectl get pod -l app=loki -n monitoring -o name | head -1) \
  -- wget -qO- "http://localhost:3100/loki/api/v1/query_range?query=\
{job%3D\"otel-demo%2Ffrontend\"}+|+json+|+traceId%3D%22${TRACE_ID}%22&limit=50"

# The logs should show all log lines emitted during that specific
# request, correlated by the shared traceId field