{
  "uid": "duocthu-observability",
  "title": "Dược Thư — Request path observability",
  "tags": ["duocthu", "rag", "opentelemetry"],
  "timezone": "browser",
  "schemaVersion": 39,
  "version": 2,
  "refresh": "10s",
  "time": { "from": "now-1h", "to": "now" },
  "panels": [
    {
      "id": 1,
      "type": "stat",
      "title": "Request rate",
      "gridPos": { "h": 5, "w": 6, "x": 0, "y": 0 },
      "datasource": { "type": "prometheus", "uid": "prometheus" },
      "targets": [{ "refId": "A", "expr": "sum(rate(duocthu_requests_total{route=\"/v1/rag/query\"}[5m]))", "legendFormat": "requests/s" }],
      "fieldConfig": { "defaults": { "unit": "reqps", "decimals": 2 }, "overrides": [] },
      "options": { "graphMode": "area", "colorMode": "value", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false } }
    },
    {
      "id": 2,
      "type": "stat",
      "title": "Request p95",
      "description": "The latency histogram carries Tempo trace exemplars. Click an exemplar in the latency panel below to inspect the complete request path.",
      "gridPos": { "h": 5, "w": 6, "x": 6, "y": 0 },
      "datasource": { "type": "prometheus", "uid": "prometheus" },
      "targets": [{ "refId": "A", "expr": "histogram_quantile(0.95, sum by (le) (rate(duocthu_request_duration_seconds_bucket{route=\"/v1/rag/query\"}[5m])))", "legendFormat": "p95" }],
      "fieldConfig": { "defaults": { "unit": "s", "decimals": 2 }, "overrides": [] },
      "options": { "graphMode": "area", "colorMode": "value", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false } }
    },
    {
      "id": 3,
      "type": "stat",
      "title": "Answerable ratio",
      "gridPos": { "h": 5, "w": 6, "x": 12, "y": 0 },
      "datasource": { "type": "prometheus", "uid": "prometheus" },
      "targets": [{ "refId": "A", "expr": "sum(rate(duocthu_decision_total{decision=\"answerable\"}[5m])) / clamp_min(sum(rate(duocthu_decision_total[5m])), 0.000001)", "legendFormat": "answerable" }],
      "fieldConfig": { "defaults": { "unit": "percentunit", "min": 0, "max": 1, "decimals": 1 }, "overrides": [] },
      "options": { "graphMode": "area", "colorMode": "value", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false } }
    },
    {
      "id": 4,
      "type": "stat",
      "title": "Trace-write failures",
      "gridPos": { "h": 5, "w": 6, "x": 18, "y": 0 },
      "datasource": { "type": "prometheus", "uid": "prometheus" },
      "targets": [{ "refId": "A", "expr": "sum(increase(duocthu_trace_write_failed_total[1h]))", "legendFormat": "fail-open writes" }],
      "fieldConfig": { "defaults": { "unit": "short", "decimals": 0 }, "overrides": [] },
      "options": { "graphMode": "area", "colorMode": "value", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false } }
    },
    {
      "id": 5,
      "type": "timeseries",
      "title": "End-to-end latency — click exemplar for Tempo trace",
      "gridPos": { "h": 9, "w": 12, "x": 0, "y": 5 },
      "datasource": { "type": "prometheus", "uid": "prometheus" },
      "targets": [
        { "refId": "A", "expr": "histogram_quantile(0.50, sum by (le) (rate(duocthu_request_duration_seconds_bucket{route=\"/v1/rag/query\"}[5m])))", "legendFormat": "p50" },
        { "refId": "B", "expr": "histogram_quantile(0.95, sum by (le) (rate(duocthu_request_duration_seconds_bucket{route=\"/v1/rag/query\"}[5m])))", "legendFormat": "p95" },
        { "refId": "C", "expr": "histogram_quantile(0.99, sum by (le) (rate(duocthu_request_duration_seconds_bucket{route=\"/v1/rag/query\"}[5m])))", "legendFormat": "p99" }
      ],
      "fieldConfig": { "defaults": { "unit": "s", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 10, "showPoints": "never" } }, "overrides": [] },
      "options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull"] }, "tooltip": { "mode": "multi", "sort": "desc" } }
    },
    {
      "id": 6,
      "type": "timeseries",
      "title": "Stage p95 latency",
      "gridPos": { "h": 9, "w": 12, "x": 12, "y": 5 },
      "datasource": { "type": "prometheus", "uid": "prometheus" },
      "targets": [{ "refId": "A", "expr": "histogram_quantile(0.95, sum by (stage, le) (rate(duocthu_stage_duration_seconds_bucket[5m])))", "legendFormat": "{{stage}}" }],
      "fieldConfig": { "defaults": { "unit": "s", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 8, "showPoints": "never" } }, "overrides": [] },
      "options": { "legend": { "displayMode": "table", "placement": "right", "calcs": ["lastNotNull"] }, "tooltip": { "mode": "multi", "sort": "desc" } }
    },
    {
      "id": 7,
      "type": "timeseries",
      "title": "Decision and reason rate",
      "gridPos": { "h": 9, "w": 12, "x": 0, "y": 14 },
      "datasource": { "type": "prometheus", "uid": "prometheus" },
      "targets": [{ "refId": "A", "expr": "sum by (decision, reason) (rate(duocthu_decision_total[5m]))", "legendFormat": "{{decision}} · {{reason}}" }],
      "fieldConfig": { "defaults": { "unit": "reqps", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 12, "showPoints": "never", "stacking": { "mode": "normal", "group": "A" } } }, "overrides": [] },
      "options": { "legend": { "displayMode": "table", "placement": "right", "calcs": ["sum"] }, "tooltip": { "mode": "multi", "sort": "desc" } }
    },
    {
      "id": 8,
      "type": "timeseries",
      "title": "Provider failures",
      "gridPos": { "h": 9, "w": 12, "x": 12, "y": 14 },
      "datasource": { "type": "prometheus", "uid": "prometheus" },
      "targets": [{ "refId": "A", "expr": "sum by (provider, operation, reason) (rate(duocthu_provider_failure_total[5m]))", "legendFormat": "{{provider}} · {{operation}} · {{reason}}" }],
      "fieldConfig": { "defaults": { "unit": "reqps", "custom": { "drawStyle": "bars", "lineWidth": 1, "fillOpacity": 35, "showPoints": "never" } }, "overrides": [] },
      "options": { "legend": { "displayMode": "table", "placement": "right", "calcs": ["sum"] }, "tooltip": { "mode": "multi", "sort": "desc" } }
    }
  ]
}