VuQuangBao/duocthu
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Verify exact production traces and record rollout

Browse Source
This commit is contained in:
BaoVu2k4
2026-08-11 11:29:59 +07:00
parent 6b8f7584ed
commit 59e6ad2d0d
46 changed files with 2795 additions and 290 deletions
Download Patch File Download Diff File Expand all files Collapse all files
infra/docker/docker-compose.observability.yml
+62
View File
@@ -0,0 +1,62 @@
# Optional overlay for the existing EC2 production Compose topology:
# docker compose -f docker-compose.prod.yml -f docker-compose.observability.yml up -d
# The base production file remains unchanged and does not require this stack.
services:
ai-service:
environment:
OTEL_ENABLED: "true"
OTEL_SERVICE_NAME: ai-service
OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318/v1/traces
ENVIRONMENT: compose
depends_on:
- otel-collector
prometheus:
image: prom/prometheus:v3.3.0
command:
- --config.file=/etc/prometheus/prometheus.yml
- --storage.tsdb.path=/prometheus
- --enable-feature=exemplar-storage
volumes:
- ./prometheus/prometheus-compose.yml:/etc/prometheus/prometheus.yml:ro
- prometheus-data:/prometheus
restart: unless-stopped
tempo:
image: grafana/tempo:2.7.2
command: ["-config.file=/etc/tempo/tempo.yml"]
volumes:
- ./tempo/tempo.yml:/etc/tempo/tempo.yml:ro
- tempo-data:/var/tempo
restart: unless-stopped
otel-collector:
image: otel/opentelemetry-collector-contrib:0.123.0
command: ["--config=/etc/otelcol/config.yml"]
volumes:
- ./otel/collector.yml:/etc/otelcol/config.yml:ro
depends_on:
- tempo
restart: unless-stopped
grafana:
image: grafana/grafana:11.5.2
ports:
- "3002:3000"
environment:
GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin}
GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-change-me}
GF_AUTH_ANONYMOUS_ENABLED: "false"
volumes:
- ./grafana/provisioning:/etc/grafana/provisioning:ro
- ./grafana/dashboards:/var/lib/grafana/dashboards:ro
- grafana-data:/var/lib/grafana
depends_on:
- prometheus
- tempo
restart: unless-stopped
volumes:
prometheus-data:
tempo-data:
grafana-data:
infra/docker/docker-compose.yml
+29 -2
View File
@@ -35,7 +35,11 @@ services:
# when ai-service moves into this compose file, change the target to
# `ai-service:8000` and drop the extra_hosts entry.
prometheus:
image: prom/prometheus:latest
image: prom/prometheus:v3.3.0
command:
- --config.file=/etc/prometheus/prometheus.yml
- --storage.tsdb.path=/prometheus
- --enable-feature=exemplar-storage
ports:
- "9090:9090"
volumes:
@@ -45,7 +49,7 @@ services:
- "host.docker.internal:host-gateway"
grafana:
image: grafana/grafana:latest
image: grafana/grafana:11.5.2
ports:
- "3002:3000"
environment:
@@ -61,6 +65,28 @@ services:
- grafana-data:/var/lib/grafana
depends_on:
- prometheus
- tempo
tempo:
image: grafana/tempo:2.7.2
command: ["-config.file=/etc/tempo/tempo.yml"]
ports:
- "3200:3200"
volumes:
- ./tempo/tempo.yml:/etc/tempo/tempo.yml:ro
- tempo-data:/var/tempo
otel-collector:
image: otel/opentelemetry-collector-contrib:0.123.0
command: ["--config=/etc/otelcol/config.yml"]
ports:
- "4317:4317"
- "4318:4318"
- "13133:13133"
volumes:
- ./otel/collector.yml:/etc/otelcol/config.yml:ro
depends_on:
- tempo
# ai-service:
# build: ../../apps/ai-service
@@ -100,3 +126,4 @@ volumes:
redis-data:
prometheus-data:
grafana-data:
tempo-data:
infra/docker/grafana/dashboards/duocthu-grounding.json
+63 -181
View File
@@ -1,215 +1,97 @@
{
"uid": "duocthu-grounding",
"title": "Dược thư — Grounding & Retrieval",
"tags": ["duocthu", "rag"],
"uid": "duocthu-observability",
"title": "Dược Thư — Request path observability",
"tags": ["duocthu", "rag", "opentelemetry"],
"timezone": "browser",
"schemaVersion": 39,
"version": 1,
"version": 2,
"refresh": "10s",
"time": { "from": "now-1h", "to": "now" },
"panels": [
{
"id": 1,
"type": "stat",
"title": "Số lần LLM bịa số và bị chặn",
"description": "Generations discarded because they stated a figure that does not appear character-for-character in the cited source. This is the measured form of the claim that the answer layer cannot invent a dose. Non-zero is not a failure — it is the guardrail doing its job.",
"gridPos": { "h": 6, "w": 6, "x": 0, "y": 0 },
"targets": [
{
"refId": "A",
"expr": "sum(duocthu_generation_rejected_total{reason=\"ungrounded_number\"})",
"legendFormat": "blocked"
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"decimals": 0,
"color": { "mode": "thresholds" },
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null },
{ "color": "orange", "value": 1 }
]
}
},
"overrides": []
},
"options": {
"graphMode": "area",
"textMode": "value",
"colorMode": "value",
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }
}
"title": "Request rate",
"gridPos": { "h": 5, "w": 6, "x": 0, "y": 0 },
"datasource": { "type": "prometheus", "uid": "prometheus" },
"targets": [{ "refId": "A", "expr": "sum(rate(duocthu_requests_total{route=\"/v1/rag/query\"}[5m]))", "legendFormat": "requests/s" }],
"fieldConfig": { "defaults": { "unit": "reqps", "decimals": 2 }, "overrides": [] },
"options": { "graphMode": "area", "colorMode": "value", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false } }
},
{
"id": 2,
"type": "stat",
"title": "Tỷ lệ câu trả lời có kiểm chứng",
"description": "Share of served answers that were LLM-generated and passed grounding verification. The remainder are served as verbatim source text — safe, just less readable.",
"gridPos": { "h": 6, "w": 6, "x": 6, "y": 0 },
"targets": [
{
"refId": "A",
"expr": "sum(duocthu_generation_served_total) / clamp_min(sum(duocthu_generation_served_total) + sum(duocthu_answer_extractive_total), 1)",
"legendFormat": "verified"
}
],
"fieldConfig": {
"defaults": {
"unit": "percentunit",
"decimals": 1,
"min": 0,
"max": 1,
"color": { "mode": "thresholds" },
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "red", "value": null },
{ "color": "orange", "value": 0.5 },
{ "color": "green", "value": 0.8 }
]
}
},
"overrides": []
},
"options": {
"graphMode": "area",
"textMode": "value",
"colorMode": "value",
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }
}
"title": "Request p95",
"description": "The latency histogram carries Tempo trace exemplars. Click an exemplar in the latency panel below to inspect the complete request path.",
"gridPos": { "h": 5, "w": 6, "x": 6, "y": 0 },
"datasource": { "type": "prometheus", "uid": "prometheus" },
"targets": [{ "refId": "A", "expr": "histogram_quantile(0.95, sum by (le) (rate(duocthu_request_duration_seconds_bucket{route=\"/v1/rag/query\"}[5m])))", "legendFormat": "p95" }],
"fieldConfig": { "defaults": { "unit": "s", "decimals": 2 }, "overrides": [] },
"options": { "graphMode": "area", "colorMode": "value", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false } }
},
{
"id": 3,
"type": "stat",
"title": "Tỷ lệ từ chối trả lời",
"description": "Share of requests the system declined. A medical reference tool is expected to abstain — symptom questions, invented drug names and out-of-scope asks all land here by design.",
"gridPos": { "h": 6, "w": 6, "x": 12, "y": 0 },
"targets": [
{
"refId": "A",
"expr": "sum(duocthu_abstention_total) / clamp_min(sum(duocthu_abstention_total) + sum(duocthu_generation_served_total) + sum(duocthu_answer_extractive_total), 1)",
"legendFormat": "abstained"
}
],
"fieldConfig": {
"defaults": {
"unit": "percentunit",
"decimals": 1,
"min": 0,
"max": 1,
"color": { "mode": "fixed", "fixedColor": "blue" }
},
"overrides": []
},
"options": {
"graphMode": "area",
"textMode": "value",
"colorMode": "value",
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }
}
"title": "Answerable ratio",
"gridPos": { "h": 5, "w": 6, "x": 12, "y": 0 },
"datasource": { "type": "prometheus", "uid": "prometheus" },
"targets": [{ "refId": "A", "expr": "sum(rate(duocthu_decision_total{decision=\"answerable\"}[5m])) / clamp_min(sum(rate(duocthu_decision_total[5m])), 0.000001)", "legendFormat": "answerable" }],
"fieldConfig": { "defaults": { "unit": "percentunit", "min": 0, "max": 1, "decimals": 1 }, "overrides": [] },
"options": { "graphMode": "area", "colorMode": "value", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false } }
},
{
"id": 4,
"type": "stat",
"title": "Câu đi đúng đường section route",
"description": "Retrievals resolved by section filter — the route measured at 16/16 on human-written questions. The remainder fall back to similarity, measured at hit@1 0.544.",
"gridPos": { "h": 6, "w": 6, "x": 18, "y": 0 },
"targets": [
{
"refId": "A",
"expr": "sum(duocthu_retrieval_route_total{route=\"section\"}) / clamp_min(sum(duocthu_retrieval_route_total), 1)",
"legendFormat": "section"
}
],
"fieldConfig": {
"defaults": {
"unit": "percentunit",
"decimals": 1,
"min": 0,
"max": 1,
"color": { "mode": "thresholds" },
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "red", "value": null },
{ "color": "orange", "value": 0.6 },
{ "color": "green", "value": 0.85 }
]
}
},
"overrides": []
},
"options": {
"graphMode": "area",
"textMode": "value",
"colorMode": "value",
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }
}
"title": "Trace-write failures",
"gridPos": { "h": 5, "w": 6, "x": 18, "y": 0 },
"datasource": { "type": "prometheus", "uid": "prometheus" },
"targets": [{ "refId": "A", "expr": "sum(increase(duocthu_trace_write_failed_total[1h]))", "legendFormat": "fail-open writes" }],
"fieldConfig": { "defaults": { "unit": "short", "decimals": 0 }, "overrides": [] },
"options": { "graphMode": "area", "colorMode": "value", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false } }
},
{
"id": 5,
"type": "timeseries",
"title": "Vì sao một bản sinh bị loại",
"description": "Every reason a generation was discarded before reaching a clinician. `ungrounded_number` is a fabrication caught; `provider_unavailable` is an outage; `evidence_insufficient` is the model correctly declining.",
"gridPos": { "h": 9, "w": 12, "x": 0, "y": 6 },
"title": "End-to-end latency — click exemplar for Tempo trace",
"gridPos": { "h": 9, "w": 12, "x": 0, "y": 5 },
"datasource": { "type": "prometheus", "uid": "prometheus" },
"targets": [
{
"refId": "A",
"expr": "sum by (reason) (rate(duocthu_generation_rejected_total[5m]))",
"legendFormat": "{{reason}}"
}
{ "refId": "A", "expr": "histogram_quantile(0.50, sum by (le) (rate(duocthu_request_duration_seconds_bucket{route=\"/v1/rag/query\"}[5m])))", "legendFormat": "p50" },
{ "refId": "B", "expr": "histogram_quantile(0.95, sum by (le) (rate(duocthu_request_duration_seconds_bucket{route=\"/v1/rag/query\"}[5m])))", "legendFormat": "p95" },
{ "refId": "C", "expr": "histogram_quantile(0.99, sum by (le) (rate(duocthu_request_duration_seconds_bucket{route=\"/v1/rag/query\"}[5m])))", "legendFormat": "p99" }
],
"fieldConfig": {
"defaults": {
"unit": "reqps",
"custom": {
"drawStyle": "line",
"lineWidth": 2,
"fillOpacity": 12,
"showPoints": "never",
"stacking": { "mode": "normal", "group": "A" }
}
},
"overrides": []
},
"options": {
"legend": { "displayMode": "table", "placement": "right", "calcs": ["sum"] },
"tooltip": { "mode": "multi", "sort": "desc" }
}
"fieldConfig": { "defaults": { "unit": "s", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 10, "showPoints": "never" } }, "overrides": [] },
"options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull"] }, "tooltip": { "mode": "multi", "sort": "desc" } }
},
{
"id": 6,
"type": "timeseries",
"title": "Vì sao hệ thống từ chối trả lời",
"description": "Abstentions by the reason retrieval gave. `drug_not_resolved` dominating means most refusals are questions that never named a drug in the formulary — symptom questions and invented names.",
"gridPos": { "h": 9, "w": 12, "x": 12, "y": 6 },
"targets": [
{
"refId": "A",
"expr": "sum by (reason) (rate(duocthu_abstention_total[5m]))",
"legendFormat": "{{reason}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "reqps",
"custom": {
"drawStyle": "line",
"lineWidth": 2,
"fillOpacity": 12,
"showPoints": "never",
"stacking": { "mode": "normal", "group": "A" }
}
},
"overrides": []
},
"options": {
"legend": { "displayMode": "table", "placement": "right", "calcs": ["sum"] },
"tooltip": { "mode": "multi", "sort": "desc" }
}
"title": "Stage p95 latency",
"gridPos": { "h": 9, "w": 12, "x": 12, "y": 5 },
"datasource": { "type": "prometheus", "uid": "prometheus" },
"targets": [{ "refId": "A", "expr": "histogram_quantile(0.95, sum by (stage, le) (rate(duocthu_stage_duration_seconds_bucket[5m])))", "legendFormat": "{{stage}}" }],
"fieldConfig": { "defaults": { "unit": "s", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 8, "showPoints": "never" } }, "overrides": [] },
"options": { "legend": { "displayMode": "table", "placement": "right", "calcs": ["lastNotNull"] }, "tooltip": { "mode": "multi", "sort": "desc" } }
},
{
"id": 7,
"type": "timeseries",
"title": "Decision and reason rate",
"gridPos": { "h": 9, "w": 12, "x": 0, "y": 14 },
"datasource": { "type": "prometheus", "uid": "prometheus" },
"targets": [{ "refId": "A", "expr": "sum by (decision, reason) (rate(duocthu_decision_total[5m]))", "legendFormat": "{{decision}} · {{reason}}" }],
"fieldConfig": { "defaults": { "unit": "reqps", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 12, "showPoints": "never", "stacking": { "mode": "normal", "group": "A" } } }, "overrides": [] },
"options": { "legend": { "displayMode": "table", "placement": "right", "calcs": ["sum"] }, "tooltip": { "mode": "multi", "sort": "desc" } }
},
{
"id": 8,
"type": "timeseries",
"title": "Provider failures",
"gridPos": { "h": 9, "w": 12, "x": 12, "y": 14 },
"datasource": { "type": "prometheus", "uid": "prometheus" },
"targets": [{ "refId": "A", "expr": "sum by (provider, operation, reason) (rate(duocthu_provider_failure_total[5m]))", "legendFormat": "{{provider}} · {{operation}} · {{reason}}" }],
"fieldConfig": { "defaults": { "unit": "reqps", "custom": { "drawStyle": "bars", "lineWidth": 1, "fillOpacity": 35, "showPoints": "never" } }, "overrides": [] },
"options": { "legend": { "displayMode": "table", "placement": "right", "calcs": ["sum"] }, "tooltip": { "mode": "multi", "sort": "desc" } }
}
]
}
infra/docker/grafana/provisioning/datasources/prometheus.yml
+25
View File
@@ -2,8 +2,33 @@ apiVersion: 1
datasources:
- name: Prometheus
uid: prometheus
type: prometheus
access: proxy
url: http://prometheus:9090
isDefault: true
editable: false
jsonData:
httpMethod: POST
exemplarTraceIdDestinations:
- datasourceUid: tempo
name: trace_id
- name: Tempo
uid: tempo
type: tempo
access: proxy
url: http://tempo:3200
editable: false
jsonData:
nodeGraph:
enabled: true
serviceMap:
datasourceUid: prometheus
tracesToMetrics:
datasourceUid: prometheus
spanStartTimeShift: -2m
spanEndTimeShift: 2m
tags:
- key: service.name
value: service
infra/docker/otel/collector.yml
+34
View File
@@ -0,0 +1,34 @@
receivers:
otlp:
protocols:
grpc:
endpoint: 0.0.0.0:4317
http:
endpoint: 0.0.0.0:4318
processors:
memory_limiter:
check_interval: 1s
limit_mib: 256
spike_limit_mib: 64
batch:
timeout: 2s
send_batch_size: 512
exporters:
otlp/tempo:
endpoint: tempo:4317
tls:
insecure: true
extensions:
health_check:
endpoint: 0.0.0.0:13133
service:
extensions: [health_check]
pipelines:
traces:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [otlp/tempo]
infra/docker/prometheus/prometheus-compose.yml
+16
View File
@@ -0,0 +1,16 @@
global:
scrape_interval: 15s
evaluation_interval: 15s
storage:
exemplars:
max_exemplars: 100000
scrape_configs:
- job_name: ai-service
metrics_path: /metrics
static_configs:
- targets: ["ai-service:8000"]
labels:
service: ai-service
env: compose
infra/docker/prometheus/prometheus.yml
+4
View File
@@ -2,6 +2,10 @@ global:
scrape_interval: 15s
evaluation_interval: 15s
storage:
exemplars:
max_exemplars: 100000
scrape_configs:
- job_name: ai-service
metrics_path: /metrics
infra/docker/tempo/tempo.yml
+26
View File
@@ -0,0 +1,26 @@
server:
http_listen_port: 3200
distributor:
receivers:
otlp:
protocols:
grpc:
endpoint: 0.0.0.0:4317
http:
endpoint: 0.0.0.0:4318
ingester:
max_block_duration: 5m
compactor:
compaction:
block_retention: 24h
storage:
trace:
backend: local
wal:
path: /var/tempo/wal
local:
path: /var/tempo/blocks