Fix live multi-turn: pass the resolved drug, stop did-you-mean garbage
This commit is contained in:
@@ -10,25 +10,25 @@
|
||||
"Resource": "*"
|
||||
},
|
||||
{
|
||||
"Sid": "ReadTheTwoEmbeddingModels",
|
||||
"Sid": "ReadModelMetadata",
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"bedrock:GetFoundationModel"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:bedrock:us-east-1::foundation-model/amazon.titan-embed-text-v2:0",
|
||||
"arn:aws:bedrock:us-east-1::foundation-model/cohere.embed-v4:0"
|
||||
]
|
||||
"Resource": "*"
|
||||
},
|
||||
{
|
||||
"Sid": "InvokeOnlyTheTwoEmbeddingModels",
|
||||
"Sid": "InvokeEmbeddingGenerationRerank",
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"bedrock:InvokeModel"
|
||||
"bedrock:InvokeModel",
|
||||
"bedrock:InvokeModelWithResponseStream"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:bedrock:us-east-1::foundation-model/amazon.titan-embed-text-v2:0",
|
||||
"arn:aws:bedrock:us-east-1::foundation-model/cohere.embed-v4:0"
|
||||
"arn:aws:bedrock:us-east-1::foundation-model/cohere.embed-v4:0",
|
||||
"arn:aws:bedrock:us-east-1::foundation-model/deepseek.v3.2",
|
||||
"arn:aws:bedrock:us-east-1::foundation-model/cohere.rerank-v3-5:0"
|
||||
]
|
||||
}
|
||||
]
|
||||
|
||||
@@ -30,6 +30,38 @@ services:
|
||||
volumes:
|
||||
- redis-data:/data
|
||||
|
||||
# Observability. `ai-service` runs on the host during development, so
|
||||
# Prometheus reaches it via host.docker.internal rather than a service name;
|
||||
# when ai-service moves into this compose file, change the target to
|
||||
# `ai-service:8000` and drop the extra_hosts entry.
|
||||
prometheus:
|
||||
image: prom/prometheus:latest
|
||||
ports:
|
||||
- "9090:9090"
|
||||
volumes:
|
||||
- ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
|
||||
- prometheus-data:/prometheus
|
||||
extra_hosts:
|
||||
- "host.docker.internal:host-gateway"
|
||||
|
||||
grafana:
|
||||
image: grafana/grafana:latest
|
||||
ports:
|
||||
- "3002:3000"
|
||||
environment:
|
||||
# Local development only. The dashboard is the demo surface, and a login
|
||||
# wall in front of it during a review is friction with no security value
|
||||
# on a laptop-local stack. Do not carry this into the k3s deployment.
|
||||
GF_AUTH_ANONYMOUS_ENABLED: "true"
|
||||
GF_AUTH_ANONYMOUS_ORG_ROLE: Admin
|
||||
GF_AUTH_DISABLE_LOGIN_FORM: "true"
|
||||
volumes:
|
||||
- ./grafana/provisioning:/etc/grafana/provisioning:ro
|
||||
- ./grafana/dashboards:/var/lib/grafana/dashboards:ro
|
||||
- grafana-data:/var/lib/grafana
|
||||
depends_on:
|
||||
- prometheus
|
||||
|
||||
# ai-service:
|
||||
# build: ../../apps/ai-service
|
||||
# env_file: ../../apps/ai-service/.env
|
||||
@@ -66,3 +98,5 @@ volumes:
|
||||
postgres-data:
|
||||
qdrant-data:
|
||||
redis-data:
|
||||
prometheus-data:
|
||||
grafana-data:
|
||||
|
||||
@@ -0,0 +1,215 @@
|
||||
{
|
||||
"uid": "duocthu-grounding",
|
||||
"title": "Dược thư — Grounding & Retrieval",
|
||||
"tags": ["duocthu", "rag"],
|
||||
"timezone": "browser",
|
||||
"schemaVersion": 39,
|
||||
"version": 1,
|
||||
"refresh": "10s",
|
||||
"time": { "from": "now-1h", "to": "now" },
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"type": "stat",
|
||||
"title": "Số lần LLM bịa số và bị chặn",
|
||||
"description": "Generations discarded because they stated a figure that does not appear character-for-character in the cited source. This is the measured form of the claim that the answer layer cannot invent a dose. Non-zero is not a failure — it is the guardrail doing its job.",
|
||||
"gridPos": { "h": 6, "w": 6, "x": 0, "y": 0 },
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum(duocthu_generation_rejected_total{reason=\"ungrounded_number\"})",
|
||||
"legendFormat": "blocked"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short",
|
||||
"decimals": 0,
|
||||
"color": { "mode": "thresholds" },
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "orange", "value": 1 }
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"graphMode": "area",
|
||||
"textMode": "value",
|
||||
"colorMode": "value",
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "stat",
|
||||
"title": "Tỷ lệ câu trả lời có kiểm chứng",
|
||||
"description": "Share of served answers that were LLM-generated and passed grounding verification. The remainder are served as verbatim source text — safe, just less readable.",
|
||||
"gridPos": { "h": 6, "w": 6, "x": 6, "y": 0 },
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum(duocthu_generation_served_total) / clamp_min(sum(duocthu_generation_served_total) + sum(duocthu_answer_extractive_total), 1)",
|
||||
"legendFormat": "verified"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percentunit",
|
||||
"decimals": 1,
|
||||
"min": 0,
|
||||
"max": 1,
|
||||
"color": { "mode": "thresholds" },
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "red", "value": null },
|
||||
{ "color": "orange", "value": 0.5 },
|
||||
{ "color": "green", "value": 0.8 }
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"graphMode": "area",
|
||||
"textMode": "value",
|
||||
"colorMode": "value",
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"type": "stat",
|
||||
"title": "Tỷ lệ từ chối trả lời",
|
||||
"description": "Share of requests the system declined. A medical reference tool is expected to abstain — symptom questions, invented drug names and out-of-scope asks all land here by design.",
|
||||
"gridPos": { "h": 6, "w": 6, "x": 12, "y": 0 },
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum(duocthu_abstention_total) / clamp_min(sum(duocthu_abstention_total) + sum(duocthu_generation_served_total) + sum(duocthu_answer_extractive_total), 1)",
|
||||
"legendFormat": "abstained"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percentunit",
|
||||
"decimals": 1,
|
||||
"min": 0,
|
||||
"max": 1,
|
||||
"color": { "mode": "fixed", "fixedColor": "blue" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"graphMode": "area",
|
||||
"textMode": "value",
|
||||
"colorMode": "value",
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"type": "stat",
|
||||
"title": "Câu đi đúng đường section route",
|
||||
"description": "Retrievals resolved by section filter — the route measured at 16/16 on human-written questions. The remainder fall back to similarity, measured at hit@1 0.544.",
|
||||
"gridPos": { "h": 6, "w": 6, "x": 18, "y": 0 },
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum(duocthu_retrieval_route_total{route=\"section\"}) / clamp_min(sum(duocthu_retrieval_route_total), 1)",
|
||||
"legendFormat": "section"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percentunit",
|
||||
"decimals": 1,
|
||||
"min": 0,
|
||||
"max": 1,
|
||||
"color": { "mode": "thresholds" },
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "red", "value": null },
|
||||
{ "color": "orange", "value": 0.6 },
|
||||
{ "color": "green", "value": 0.85 }
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"graphMode": "area",
|
||||
"textMode": "value",
|
||||
"colorMode": "value",
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"type": "timeseries",
|
||||
"title": "Vì sao một bản sinh bị loại",
|
||||
"description": "Every reason a generation was discarded before reaching a clinician. `ungrounded_number` is a fabrication caught; `provider_unavailable` is an outage; `evidence_insufficient` is the model correctly declining.",
|
||||
"gridPos": { "h": 9, "w": 12, "x": 0, "y": 6 },
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by (reason) (rate(duocthu_generation_rejected_total[5m]))",
|
||||
"legendFormat": "{{reason}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "reqps",
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineWidth": 2,
|
||||
"fillOpacity": 12,
|
||||
"showPoints": "never",
|
||||
"stacking": { "mode": "normal", "group": "A" }
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["sum"] },
|
||||
"tooltip": { "mode": "multi", "sort": "desc" }
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"type": "timeseries",
|
||||
"title": "Vì sao hệ thống từ chối trả lời",
|
||||
"description": "Abstentions by the reason retrieval gave. `drug_not_resolved` dominating means most refusals are questions that never named a drug in the formulary — symptom questions and invented names.",
|
||||
"gridPos": { "h": 9, "w": 12, "x": 12, "y": 6 },
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by (reason) (rate(duocthu_abstention_total[5m]))",
|
||||
"legendFormat": "{{reason}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "reqps",
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineWidth": 2,
|
||||
"fillOpacity": 12,
|
||||
"showPoints": "never",
|
||||
"stacking": { "mode": "normal", "group": "A" }
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["sum"] },
|
||||
"tooltip": { "mode": "multi", "sort": "desc" }
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
apiVersion: 1
|
||||
|
||||
providers:
|
||||
- name: duocthu
|
||||
orgId: 1
|
||||
folder: Dược thư
|
||||
type: file
|
||||
disableDeletion: false
|
||||
updateIntervalSeconds: 30
|
||||
allowUiUpdates: true
|
||||
options:
|
||||
path: /var/lib/grafana/dashboards
|
||||
foldersFromFilesStructure: false
|
||||
@@ -0,0 +1,9 @@
|
||||
apiVersion: 1
|
||||
|
||||
datasources:
|
||||
- name: Prometheus
|
||||
type: prometheus
|
||||
access: proxy
|
||||
url: http://prometheus:9090
|
||||
isDefault: true
|
||||
editable: false
|
||||
@@ -0,0 +1,16 @@
|
||||
global:
|
||||
scrape_interval: 15s
|
||||
evaluation_interval: 15s
|
||||
|
||||
scrape_configs:
|
||||
- job_name: ai-service
|
||||
metrics_path: /metrics
|
||||
static_configs:
|
||||
# ai-service runs on the host during development. `/metrics` returns 404
|
||||
# when METRICS_ENABLED is false or prometheus-client is not installed, so
|
||||
# a target that is UP-but-404 means the app is running without metrics —
|
||||
# not that the app is down.
|
||||
- targets: ["host.docker.internal:8079"]
|
||||
labels:
|
||||
service: ai-service
|
||||
env: local
|
||||
Reference in New Issue
Block a user