Observability Guide

1docker compose -f deploy/monitoring/docker-compose-monitoring.yml up -d

1# Raw metrics from StreamFlow
2curl http://localhost:8080/metrics
3
4# Verify in Prometheus
5curl 'http://localhost:9090/api/v1/query?query=streamflow_events_published_total'

1# Copy dashboard JSON into Grafana
2cp my-dashboard.json deploy/monitoring/
3# Add to provider, then restart Grafana
4docker compose -f deploy/monitoring/docker-compose-monitoring.yml restart grafana

1# alertmanager.yml
2route:
3  receiver: 'slack-critical'
4  routes:
5    - match:
6        severity: critical
7      receiver: 'pagerduty'
8
9receivers:
10  - name: 'slack-critical'
11    slack_configs:
12      - api_url: 'https://hooks.slack.com/services/xxx/yyy/zzz'
13        channel: '#streamflow-alerts'
14        title: '{{ .GroupLabels.alertname }}'
15        text: '{{ .Annotations.description }}'
16
17  - name: 'pagerduty'
18    pagerduty_configs:
19      - service_key: 'xxx-yyy-zzz'

1# streamflow.properties
2otel.enabled=true
3otel.endpoint=http://jaeger:4317       # or Tempo, Datadog Agent
4otel.service.name=streamflow-node-1
5otel.metrics.interval.seconds=30
6otel.traces.delay.seconds=5

1# Already configured in deploy/kubernetes/streamflow-statefulset.yaml
2livenessProbe:
3  httpGet:
4    path: /health
5    port: 8080
6  initialDelaySeconds: 30
7  periodSeconds: 10
8
9readinessProbe:
10  httpGet:
11    path: /ready
12    port: 8080
13  initialDelaySeconds: 15
14  periodSeconds: 5

1# docker-compose.yml
2healthcheck:
3  test: ["CMD", "curl", "-f", "http://localhost:8080/health"]
4  interval: 10s
5  timeout: 5s
6  retries: 12
7  start_period: 30s

1# List agents with live metrics
2curl http://localhost:8080/api/v1/agents | jq '.[].metrics'

1{
2  "name": "fraud-detector",
3  "status": "RUNNING",
4  "instances": 5,
5  "metrics": {
6    "eventsPerSec": 1250.0,
7    "totalDecisions": 847293,
8    "errorRate": 0.02,
9    "avgLatencyMs": 45.3,
10    "p99LatencyMs": 120.0,
11    "queueDepth": 42,
12    "tokensUsedToday": 152847,
13    "costToday": 2.13
14  }
15}

1# Throughput per agent
2rate(agent_events_processed{agent="fraud-detector"}[5m])
3
4# LLM latency p99
5agent_llm_latency_p99{agent="fraud-detector"}
6
7# Circuit breaker open?
8agent_llm_circuit_breaker_open{agent="fraud-detector"} == 1
9
10# Tokens consumed per day
11increase(agent_llm_tokens_input[24h]) + increase(agent_llm_tokens_output[24h])

1<!-- logback.xml (streamflow-app/src/main/resources/) -->
2<configuration>
3    <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
4        <encoder>
5            <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
6        </encoder>
7    </appender>
8
9    <root level="WARN"/>
10    <logger name="com.streamflow" level="INFO"/>
11    <!-- Debug for a specific module -->
12    <logger name="com.streamflow.connect.kafka" level="DEBUG"/>
13</configuration>

1<appender name="JSON" class="ch.qos.logback.core.ConsoleAppender">
2    <encoder class="net.logstash.logback.encoder.LogstashEncoder"/>
3</appender>

1# 1. Check disk latency
2curl -s http://localhost:8080/metrics | grep fsync_latency
3# If p99 > 50ms → disk issue (saturated pd-ssd, IOPS throttled)
4
5# 2. Check GC
6curl -s http://localhost:8080/metrics | grep gc_pause
7# If GC > 5% CPU → increase -Xmx or reduce batch.size
8
9# 3. Check backpressure
10curl -s http://localhost:8080/metrics | grep backpressure
11# If throttle > 0 → consumers can't keep up

1# 1. Check lag
2curl -s http://localhost:8080/metrics | grep consumer_lag
3# lag > 100K → scale consumers
4
5# 2. Check replication
6curl -s http://localhost:8080/metrics | grep replication_lag
7# If raft lag > 0 → a follower is slow
8
9# 3. Check agents
10curl http://localhost:8080/api/v1/agents | jq '.[].metrics.queueDepth'
11# If queue > 5000 → scale agent instances

1# 1. Check circuit breaker
2curl -s http://localhost:8080/metrics | grep circuit_breaker
3# If open=1 → LLM is down or rate-limited
4
5# 2. Check errors
6curl -s http://localhost:8080/metrics | grep agent_events_errors
7# Error rate > 5% → check agent logs
8
9# 3. View recent traces
10curl http://localhost:8080/api/v1/agents/{id} | jq '.recentTraces[-3:]'

1# 1. Check leadership
2curl -s http://localhost:8080/metrics | grep leader
3# If leader_lost → check network connectivity between nodes
4
5# 2. Check gRPC ports
6nc -z streamflow-2 9169 && echo "OK" || echo "BLOCKED"
7
8# 3. Verify TLS certificates
9openssl s_client -connect streamflow-2:9169 < /dev/null 2>&1 | grep "Verify return code"