Verify exact production traces and record rollout
This commit is contained in:
@@ -0,0 +1,62 @@
|
||||
# Optional overlay for the existing EC2 production Compose topology:
|
||||
# docker compose -f docker-compose.prod.yml -f docker-compose.observability.yml up -d
|
||||
# The base production file remains unchanged and does not require this stack.
|
||||
services:
|
||||
ai-service:
|
||||
environment:
|
||||
OTEL_ENABLED: "true"
|
||||
OTEL_SERVICE_NAME: ai-service
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318/v1/traces
|
||||
ENVIRONMENT: compose
|
||||
depends_on:
|
||||
- otel-collector
|
||||
|
||||
prometheus:
|
||||
image: prom/prometheus:v3.3.0
|
||||
command:
|
||||
- --config.file=/etc/prometheus/prometheus.yml
|
||||
- --storage.tsdb.path=/prometheus
|
||||
- --enable-feature=exemplar-storage
|
||||
volumes:
|
||||
- ./prometheus/prometheus-compose.yml:/etc/prometheus/prometheus.yml:ro
|
||||
- prometheus-data:/prometheus
|
||||
restart: unless-stopped
|
||||
|
||||
tempo:
|
||||
image: grafana/tempo:2.7.2
|
||||
command: ["-config.file=/etc/tempo/tempo.yml"]
|
||||
volumes:
|
||||
- ./tempo/tempo.yml:/etc/tempo/tempo.yml:ro
|
||||
- tempo-data:/var/tempo
|
||||
restart: unless-stopped
|
||||
|
||||
otel-collector:
|
||||
image: otel/opentelemetry-collector-contrib:0.123.0
|
||||
command: ["--config=/etc/otelcol/config.yml"]
|
||||
volumes:
|
||||
- ./otel/collector.yml:/etc/otelcol/config.yml:ro
|
||||
depends_on:
|
||||
- tempo
|
||||
restart: unless-stopped
|
||||
|
||||
grafana:
|
||||
image: grafana/grafana:11.5.2
|
||||
ports:
|
||||
- "3002:3000"
|
||||
environment:
|
||||
GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin}
|
||||
GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-change-me}
|
||||
GF_AUTH_ANONYMOUS_ENABLED: "false"
|
||||
volumes:
|
||||
- ./grafana/provisioning:/etc/grafana/provisioning:ro
|
||||
- ./grafana/dashboards:/var/lib/grafana/dashboards:ro
|
||||
- grafana-data:/var/lib/grafana
|
||||
depends_on:
|
||||
- prometheus
|
||||
- tempo
|
||||
restart: unless-stopped
|
||||
|
||||
volumes:
|
||||
prometheus-data:
|
||||
tempo-data:
|
||||
grafana-data:
|
||||
@@ -35,7 +35,11 @@ services:
|
||||
# when ai-service moves into this compose file, change the target to
|
||||
# `ai-service:8000` and drop the extra_hosts entry.
|
||||
prometheus:
|
||||
image: prom/prometheus:latest
|
||||
image: prom/prometheus:v3.3.0
|
||||
command:
|
||||
- --config.file=/etc/prometheus/prometheus.yml
|
||||
- --storage.tsdb.path=/prometheus
|
||||
- --enable-feature=exemplar-storage
|
||||
ports:
|
||||
- "9090:9090"
|
||||
volumes:
|
||||
@@ -45,7 +49,7 @@ services:
|
||||
- "host.docker.internal:host-gateway"
|
||||
|
||||
grafana:
|
||||
image: grafana/grafana:latest
|
||||
image: grafana/grafana:11.5.2
|
||||
ports:
|
||||
- "3002:3000"
|
||||
environment:
|
||||
@@ -61,6 +65,28 @@ services:
|
||||
- grafana-data:/var/lib/grafana
|
||||
depends_on:
|
||||
- prometheus
|
||||
- tempo
|
||||
|
||||
tempo:
|
||||
image: grafana/tempo:2.7.2
|
||||
command: ["-config.file=/etc/tempo/tempo.yml"]
|
||||
ports:
|
||||
- "3200:3200"
|
||||
volumes:
|
||||
- ./tempo/tempo.yml:/etc/tempo/tempo.yml:ro
|
||||
- tempo-data:/var/tempo
|
||||
|
||||
otel-collector:
|
||||
image: otel/opentelemetry-collector-contrib:0.123.0
|
||||
command: ["--config=/etc/otelcol/config.yml"]
|
||||
ports:
|
||||
- "4317:4317"
|
||||
- "4318:4318"
|
||||
- "13133:13133"
|
||||
volumes:
|
||||
- ./otel/collector.yml:/etc/otelcol/config.yml:ro
|
||||
depends_on:
|
||||
- tempo
|
||||
|
||||
# ai-service:
|
||||
# build: ../../apps/ai-service
|
||||
@@ -100,3 +126,4 @@ volumes:
|
||||
redis-data:
|
||||
prometheus-data:
|
||||
grafana-data:
|
||||
tempo-data:
|
||||
|
||||
@@ -1,215 +1,97 @@
|
||||
{
|
||||
"uid": "duocthu-grounding",
|
||||
"title": "Dược thư — Grounding & Retrieval",
|
||||
"tags": ["duocthu", "rag"],
|
||||
"uid": "duocthu-observability",
|
||||
"title": "Dược Thư — Request path observability",
|
||||
"tags": ["duocthu", "rag", "opentelemetry"],
|
||||
"timezone": "browser",
|
||||
"schemaVersion": 39,
|
||||
"version": 1,
|
||||
"version": 2,
|
||||
"refresh": "10s",
|
||||
"time": { "from": "now-1h", "to": "now" },
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"type": "stat",
|
||||
"title": "Số lần LLM bịa số và bị chặn",
|
||||
"description": "Generations discarded because they stated a figure that does not appear character-for-character in the cited source. This is the measured form of the claim that the answer layer cannot invent a dose. Non-zero is not a failure — it is the guardrail doing its job.",
|
||||
"gridPos": { "h": 6, "w": 6, "x": 0, "y": 0 },
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum(duocthu_generation_rejected_total{reason=\"ungrounded_number\"})",
|
||||
"legendFormat": "blocked"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short",
|
||||
"decimals": 0,
|
||||
"color": { "mode": "thresholds" },
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "orange", "value": 1 }
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"graphMode": "area",
|
||||
"textMode": "value",
|
||||
"colorMode": "value",
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }
|
||||
}
|
||||
"title": "Request rate",
|
||||
"gridPos": { "h": 5, "w": 6, "x": 0, "y": 0 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "refId": "A", "expr": "sum(rate(duocthu_requests_total{route=\"/v1/rag/query\"}[5m]))", "legendFormat": "requests/s" }],
|
||||
"fieldConfig": { "defaults": { "unit": "reqps", "decimals": 2 }, "overrides": [] },
|
||||
"options": { "graphMode": "area", "colorMode": "value", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false } }
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "stat",
|
||||
"title": "Tỷ lệ câu trả lời có kiểm chứng",
|
||||
"description": "Share of served answers that were LLM-generated and passed grounding verification. The remainder are served as verbatim source text — safe, just less readable.",
|
||||
"gridPos": { "h": 6, "w": 6, "x": 6, "y": 0 },
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum(duocthu_generation_served_total) / clamp_min(sum(duocthu_generation_served_total) + sum(duocthu_answer_extractive_total), 1)",
|
||||
"legendFormat": "verified"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percentunit",
|
||||
"decimals": 1,
|
||||
"min": 0,
|
||||
"max": 1,
|
||||
"color": { "mode": "thresholds" },
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "red", "value": null },
|
||||
{ "color": "orange", "value": 0.5 },
|
||||
{ "color": "green", "value": 0.8 }
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"graphMode": "area",
|
||||
"textMode": "value",
|
||||
"colorMode": "value",
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }
|
||||
}
|
||||
"title": "Request p95",
|
||||
"description": "The latency histogram carries Tempo trace exemplars. Click an exemplar in the latency panel below to inspect the complete request path.",
|
||||
"gridPos": { "h": 5, "w": 6, "x": 6, "y": 0 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "refId": "A", "expr": "histogram_quantile(0.95, sum by (le) (rate(duocthu_request_duration_seconds_bucket{route=\"/v1/rag/query\"}[5m])))", "legendFormat": "p95" }],
|
||||
"fieldConfig": { "defaults": { "unit": "s", "decimals": 2 }, "overrides": [] },
|
||||
"options": { "graphMode": "area", "colorMode": "value", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false } }
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"type": "stat",
|
||||
"title": "Tỷ lệ từ chối trả lời",
|
||||
"description": "Share of requests the system declined. A medical reference tool is expected to abstain — symptom questions, invented drug names and out-of-scope asks all land here by design.",
|
||||
"gridPos": { "h": 6, "w": 6, "x": 12, "y": 0 },
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum(duocthu_abstention_total) / clamp_min(sum(duocthu_abstention_total) + sum(duocthu_generation_served_total) + sum(duocthu_answer_extractive_total), 1)",
|
||||
"legendFormat": "abstained"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percentunit",
|
||||
"decimals": 1,
|
||||
"min": 0,
|
||||
"max": 1,
|
||||
"color": { "mode": "fixed", "fixedColor": "blue" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"graphMode": "area",
|
||||
"textMode": "value",
|
||||
"colorMode": "value",
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }
|
||||
}
|
||||
"title": "Answerable ratio",
|
||||
"gridPos": { "h": 5, "w": 6, "x": 12, "y": 0 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "refId": "A", "expr": "sum(rate(duocthu_decision_total{decision=\"answerable\"}[5m])) / clamp_min(sum(rate(duocthu_decision_total[5m])), 0.000001)", "legendFormat": "answerable" }],
|
||||
"fieldConfig": { "defaults": { "unit": "percentunit", "min": 0, "max": 1, "decimals": 1 }, "overrides": [] },
|
||||
"options": { "graphMode": "area", "colorMode": "value", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false } }
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"type": "stat",
|
||||
"title": "Câu đi đúng đường section route",
|
||||
"description": "Retrievals resolved by section filter — the route measured at 16/16 on human-written questions. The remainder fall back to similarity, measured at hit@1 0.544.",
|
||||
"gridPos": { "h": 6, "w": 6, "x": 18, "y": 0 },
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum(duocthu_retrieval_route_total{route=\"section\"}) / clamp_min(sum(duocthu_retrieval_route_total), 1)",
|
||||
"legendFormat": "section"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percentunit",
|
||||
"decimals": 1,
|
||||
"min": 0,
|
||||
"max": 1,
|
||||
"color": { "mode": "thresholds" },
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "red", "value": null },
|
||||
{ "color": "orange", "value": 0.6 },
|
||||
{ "color": "green", "value": 0.85 }
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"graphMode": "area",
|
||||
"textMode": "value",
|
||||
"colorMode": "value",
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }
|
||||
}
|
||||
"title": "Trace-write failures",
|
||||
"gridPos": { "h": 5, "w": 6, "x": 18, "y": 0 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "refId": "A", "expr": "sum(increase(duocthu_trace_write_failed_total[1h]))", "legendFormat": "fail-open writes" }],
|
||||
"fieldConfig": { "defaults": { "unit": "short", "decimals": 0 }, "overrides": [] },
|
||||
"options": { "graphMode": "area", "colorMode": "value", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false } }
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"type": "timeseries",
|
||||
"title": "Vì sao một bản sinh bị loại",
|
||||
"description": "Every reason a generation was discarded before reaching a clinician. `ungrounded_number` is a fabrication caught; `provider_unavailable` is an outage; `evidence_insufficient` is the model correctly declining.",
|
||||
"gridPos": { "h": 9, "w": 12, "x": 0, "y": 6 },
|
||||
"title": "End-to-end latency — click exemplar for Tempo trace",
|
||||
"gridPos": { "h": 9, "w": 12, "x": 0, "y": 5 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by (reason) (rate(duocthu_generation_rejected_total[5m]))",
|
||||
"legendFormat": "{{reason}}"
|
||||
}
|
||||
{ "refId": "A", "expr": "histogram_quantile(0.50, sum by (le) (rate(duocthu_request_duration_seconds_bucket{route=\"/v1/rag/query\"}[5m])))", "legendFormat": "p50" },
|
||||
{ "refId": "B", "expr": "histogram_quantile(0.95, sum by (le) (rate(duocthu_request_duration_seconds_bucket{route=\"/v1/rag/query\"}[5m])))", "legendFormat": "p95" },
|
||||
{ "refId": "C", "expr": "histogram_quantile(0.99, sum by (le) (rate(duocthu_request_duration_seconds_bucket{route=\"/v1/rag/query\"}[5m])))", "legendFormat": "p99" }
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "reqps",
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineWidth": 2,
|
||||
"fillOpacity": 12,
|
||||
"showPoints": "never",
|
||||
"stacking": { "mode": "normal", "group": "A" }
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["sum"] },
|
||||
"tooltip": { "mode": "multi", "sort": "desc" }
|
||||
}
|
||||
"fieldConfig": { "defaults": { "unit": "s", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 10, "showPoints": "never" } }, "overrides": [] },
|
||||
"options": { "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull"] }, "tooltip": { "mode": "multi", "sort": "desc" } }
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"type": "timeseries",
|
||||
"title": "Vì sao hệ thống từ chối trả lời",
|
||||
"description": "Abstentions by the reason retrieval gave. `drug_not_resolved` dominating means most refusals are questions that never named a drug in the formulary — symptom questions and invented names.",
|
||||
"gridPos": { "h": 9, "w": 12, "x": 12, "y": 6 },
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by (reason) (rate(duocthu_abstention_total[5m]))",
|
||||
"legendFormat": "{{reason}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "reqps",
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineWidth": 2,
|
||||
"fillOpacity": 12,
|
||||
"showPoints": "never",
|
||||
"stacking": { "mode": "normal", "group": "A" }
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["sum"] },
|
||||
"tooltip": { "mode": "multi", "sort": "desc" }
|
||||
}
|
||||
"title": "Stage p95 latency",
|
||||
"gridPos": { "h": 9, "w": 12, "x": 12, "y": 5 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "refId": "A", "expr": "histogram_quantile(0.95, sum by (stage, le) (rate(duocthu_stage_duration_seconds_bucket[5m])))", "legendFormat": "{{stage}}" }],
|
||||
"fieldConfig": { "defaults": { "unit": "s", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 8, "showPoints": "never" } }, "overrides": [] },
|
||||
"options": { "legend": { "displayMode": "table", "placement": "right", "calcs": ["lastNotNull"] }, "tooltip": { "mode": "multi", "sort": "desc" } }
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"type": "timeseries",
|
||||
"title": "Decision and reason rate",
|
||||
"gridPos": { "h": 9, "w": 12, "x": 0, "y": 14 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "refId": "A", "expr": "sum by (decision, reason) (rate(duocthu_decision_total[5m]))", "legendFormat": "{{decision}} · {{reason}}" }],
|
||||
"fieldConfig": { "defaults": { "unit": "reqps", "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 12, "showPoints": "never", "stacking": { "mode": "normal", "group": "A" } } }, "overrides": [] },
|
||||
"options": { "legend": { "displayMode": "table", "placement": "right", "calcs": ["sum"] }, "tooltip": { "mode": "multi", "sort": "desc" } }
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
"type": "timeseries",
|
||||
"title": "Provider failures",
|
||||
"gridPos": { "h": 9, "w": 12, "x": 12, "y": 14 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "refId": "A", "expr": "sum by (provider, operation, reason) (rate(duocthu_provider_failure_total[5m]))", "legendFormat": "{{provider}} · {{operation}} · {{reason}}" }],
|
||||
"fieldConfig": { "defaults": { "unit": "reqps", "custom": { "drawStyle": "bars", "lineWidth": 1, "fillOpacity": 35, "showPoints": "never" } }, "overrides": [] },
|
||||
"options": { "legend": { "displayMode": "table", "placement": "right", "calcs": ["sum"] }, "tooltip": { "mode": "multi", "sort": "desc" } }
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -2,8 +2,33 @@ apiVersion: 1
|
||||
|
||||
datasources:
|
||||
- name: Prometheus
|
||||
uid: prometheus
|
||||
type: prometheus
|
||||
access: proxy
|
||||
url: http://prometheus:9090
|
||||
isDefault: true
|
||||
editable: false
|
||||
jsonData:
|
||||
httpMethod: POST
|
||||
exemplarTraceIdDestinations:
|
||||
- datasourceUid: tempo
|
||||
name: trace_id
|
||||
|
||||
- name: Tempo
|
||||
uid: tempo
|
||||
type: tempo
|
||||
access: proxy
|
||||
url: http://tempo:3200
|
||||
editable: false
|
||||
jsonData:
|
||||
nodeGraph:
|
||||
enabled: true
|
||||
serviceMap:
|
||||
datasourceUid: prometheus
|
||||
tracesToMetrics:
|
||||
datasourceUid: prometheus
|
||||
spanStartTimeShift: -2m
|
||||
spanEndTimeShift: 2m
|
||||
tags:
|
||||
- key: service.name
|
||||
value: service
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
grpc:
|
||||
endpoint: 0.0.0.0:4317
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
|
||||
processors:
|
||||
memory_limiter:
|
||||
check_interval: 1s
|
||||
limit_mib: 256
|
||||
spike_limit_mib: 64
|
||||
batch:
|
||||
timeout: 2s
|
||||
send_batch_size: 512
|
||||
|
||||
exporters:
|
||||
otlp/tempo:
|
||||
endpoint: tempo:4317
|
||||
tls:
|
||||
insecure: true
|
||||
|
||||
extensions:
|
||||
health_check:
|
||||
endpoint: 0.0.0.0:13133
|
||||
|
||||
service:
|
||||
extensions: [health_check]
|
||||
pipelines:
|
||||
traces:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [otlp/tempo]
|
||||
@@ -0,0 +1,16 @@
|
||||
global:
|
||||
scrape_interval: 15s
|
||||
evaluation_interval: 15s
|
||||
|
||||
storage:
|
||||
exemplars:
|
||||
max_exemplars: 100000
|
||||
|
||||
scrape_configs:
|
||||
- job_name: ai-service
|
||||
metrics_path: /metrics
|
||||
static_configs:
|
||||
- targets: ["ai-service:8000"]
|
||||
labels:
|
||||
service: ai-service
|
||||
env: compose
|
||||
@@ -2,6 +2,10 @@ global:
|
||||
scrape_interval: 15s
|
||||
evaluation_interval: 15s
|
||||
|
||||
storage:
|
||||
exemplars:
|
||||
max_exemplars: 100000
|
||||
|
||||
scrape_configs:
|
||||
- job_name: ai-service
|
||||
metrics_path: /metrics
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
server:
|
||||
http_listen_port: 3200
|
||||
|
||||
distributor:
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
grpc:
|
||||
endpoint: 0.0.0.0:4317
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
|
||||
ingester:
|
||||
max_block_duration: 5m
|
||||
|
||||
compactor:
|
||||
compaction:
|
||||
block_retention: 24h
|
||||
|
||||
storage:
|
||||
trace:
|
||||
backend: local
|
||||
wal:
|
||||
path: /var/tempo/wal
|
||||
local:
|
||||
path: /var/tempo/blocks
|
||||
@@ -2,5 +2,5 @@ apiVersion: v2
|
||||
name: medical-chatbot
|
||||
description: Umbrella Helm chart for the Duoc Thu RAG medical chatbot platform
|
||||
type: application
|
||||
version: 0.0.0
|
||||
appVersion: "0.0.0"
|
||||
version: 0.1.0
|
||||
appVersion: "0.1.0"
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
{{- define "medical-chatbot.name" -}}
|
||||
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "medical-chatbot.fullname" -}}
|
||||
{{- if .Values.fullnameOverride -}}
|
||||
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
|
||||
{{- else -}}
|
||||
{{- printf "%s-%s" .Release.Name (include "medical-chatbot.name" .) | trunc 63 | trimSuffix "-" -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "medical-chatbot.labels" -}}
|
||||
app.kubernetes.io/name: {{ include "medical-chatbot.name" . }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | quote }}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "medical-chatbot.serviceAccountName" -}}
|
||||
{{- if .Values.serviceAccount.create -}}
|
||||
{{- default (include "medical-chatbot.fullname" .) .Values.serviceAccount.name -}}
|
||||
{{- else -}}
|
||||
{{- default "default" .Values.serviceAccount.name -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "medical-chatbot.secretName" -}}
|
||||
{{- if .Values.secret.create -}}
|
||||
{{- printf "%s-runtime" (include "medical-chatbot.fullname" .) -}}
|
||||
{{- else -}}
|
||||
{{- required "secret.existingSecret is required when secret.create=false" .Values.secret.existingSecret -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
@@ -0,0 +1,107 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: {{ include "medical-chatbot.fullname" . }}-ai-service
|
||||
labels:
|
||||
{{- include "medical-chatbot.labels" . | nindent 4 }}
|
||||
data:
|
||||
ENVIRONMENT: {{ .Values.global.environment | quote }}
|
||||
QDRANT_URL: {{ default (printf "http://%s-qdrant:6333" (include "medical-chatbot.fullname" .)) .Values.qdrant.url | quote }}
|
||||
QDRANT_COLLECTION: {{ .Values.aiService.config.qdrantCollection | quote }}
|
||||
EMBEDDING_PROVIDER: {{ .Values.aiService.config.embeddingProvider | quote }}
|
||||
ANSWER_PROVIDER: {{ .Values.aiService.config.answerProvider | quote }}
|
||||
ANSWER_MODEL_ID: {{ .Values.aiService.config.answerModelId | quote }}
|
||||
METRICS_ENABLED: {{ .Values.aiService.config.metricsEnabled | quote }}
|
||||
OTEL_ENABLED: {{ and .Values.observability.enabled .Values.aiService.config.otelEnabled | quote }}
|
||||
OTEL_SERVICE_NAME: ai-service
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT: {{ printf "http://%s-otel-collector:4318/v1/traces" (include "medical-chatbot.fullname" .) | quote }}
|
||||
OTEL_SAMPLE_RATIO: {{ .Values.aiService.config.otelSampleRatio | quote }}
|
||||
MAX_WALL_CLOCK_MS: {{ .Values.aiService.config.maxWallClockMs | quote }}
|
||||
MAX_LLM_CALLS_PER_TURN: {{ .Values.aiService.config.maxLlmCallsPerTurn | quote }}
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: {{ include "medical-chatbot.fullname" . }}-ai-service
|
||||
labels:
|
||||
{{- include "medical-chatbot.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: ai-service
|
||||
spec:
|
||||
replicas: {{ .Values.aiService.replicaCount }}
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/component: ai-service
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/component: ai-service
|
||||
annotations:
|
||||
prometheus.io/scrape: "true"
|
||||
prometheus.io/path: /metrics
|
||||
prometheus.io/port: "8000"
|
||||
spec:
|
||||
serviceAccountName: {{ include "medical-chatbot.serviceAccountName" . }}
|
||||
imagePullSecrets:
|
||||
{{- toYaml .Values.global.imagePullSecrets | nindent 8 }}
|
||||
{{- if .Values.aiService.migration.enabled }}
|
||||
initContainers:
|
||||
- name: migrate
|
||||
image: "{{ .Values.aiService.image.repository }}:{{ .Values.aiService.image.tag }}"
|
||||
imagePullPolicy: {{ .Values.aiService.image.pullPolicy }}
|
||||
command: ["python", "migrate.py"]
|
||||
envFrom:
|
||||
- configMapRef: { name: {{ include "medical-chatbot.fullname" . }}-ai-service }
|
||||
env:
|
||||
- name: POSTGRES_DSN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ include "medical-chatbot.secretName" . }}
|
||||
key: postgres-dsn
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: ai-service
|
||||
image: "{{ .Values.aiService.image.repository }}:{{ .Values.aiService.image.tag }}"
|
||||
imagePullPolicy: {{ .Values.aiService.image.pullPolicy }}
|
||||
ports:
|
||||
- { name: http, containerPort: 8000 }
|
||||
envFrom:
|
||||
- configMapRef: { name: {{ include "medical-chatbot.fullname" . }}-ai-service }
|
||||
env:
|
||||
- name: POSTGRES_DSN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ include "medical-chatbot.secretName" . }}
|
||||
key: postgres-dsn
|
||||
readinessProbe:
|
||||
httpGet: { path: /ready, port: http }
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 10
|
||||
livenessProbe:
|
||||
httpGet: { path: /health, port: http }
|
||||
initialDelaySeconds: 15
|
||||
periodSeconds: 20
|
||||
startupProbe:
|
||||
httpGet: { path: /health, port: http }
|
||||
failureThreshold: 30
|
||||
periodSeconds: 5
|
||||
resources:
|
||||
{{- toYaml .Values.aiService.resources | nindent 12 }}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{ include "medical-chatbot.fullname" . }}-ai-service
|
||||
labels:
|
||||
{{- include "medical-chatbot.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: ai-service
|
||||
spec:
|
||||
type: {{ .Values.aiService.service.type }}
|
||||
selector:
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/component: ai-service
|
||||
ports:
|
||||
- name: http
|
||||
port: {{ .Values.aiService.service.port }}
|
||||
targetPort: http
|
||||
@@ -0,0 +1,131 @@
|
||||
{{- if .Values.postgres.enabled }}
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{ include "medical-chatbot.fullname" . }}-postgres
|
||||
labels:
|
||||
{{- include "medical-chatbot.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: postgres
|
||||
spec:
|
||||
clusterIP: None
|
||||
selector:
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/component: postgres
|
||||
ports:
|
||||
- { name: postgres, port: 5432, targetPort: postgres }
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: StatefulSet
|
||||
metadata:
|
||||
name: {{ include "medical-chatbot.fullname" . }}-postgres
|
||||
labels:
|
||||
{{- include "medical-chatbot.labels" . | nindent 4 }}
|
||||
spec:
|
||||
serviceName: {{ include "medical-chatbot.fullname" . }}-postgres
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/component: postgres
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/component: postgres
|
||||
spec:
|
||||
containers:
|
||||
- name: postgres
|
||||
image: {{ .Values.postgres.image }}
|
||||
ports:
|
||||
- { name: postgres, containerPort: 5432 }
|
||||
env:
|
||||
- { name: POSTGRES_USER, value: duoc_thu }
|
||||
- { name: POSTGRES_DB, value: duoc_thu }
|
||||
- name: POSTGRES_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ include "medical-chatbot.secretName" . }}
|
||||
key: postgres-password
|
||||
readinessProbe:
|
||||
exec: { command: ["pg_isready", "-U", "duoc_thu", "-d", "duoc_thu"] }
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 10
|
||||
livenessProbe:
|
||||
exec: { command: ["pg_isready", "-U", "duoc_thu", "-d", "duoc_thu"] }
|
||||
initialDelaySeconds: 20
|
||||
periodSeconds: 20
|
||||
resources:
|
||||
{{- toYaml .Values.postgres.resources | nindent 12 }}
|
||||
volumeMounts:
|
||||
- { name: data, mountPath: /var/lib/postgresql/data }
|
||||
volumeClaimTemplates:
|
||||
- metadata: { name: data }
|
||||
spec:
|
||||
accessModes: [ReadWriteOnce]
|
||||
resources:
|
||||
requests: { storage: {{ .Values.postgres.storage }} }
|
||||
{{- end }}
|
||||
{{- if and .Values.postgres.enabled .Values.qdrant.enabled }}
|
||||
---
|
||||
{{- end }}
|
||||
{{- if .Values.qdrant.enabled }}
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{ include "medical-chatbot.fullname" . }}-qdrant
|
||||
labels:
|
||||
{{- include "medical-chatbot.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: qdrant
|
||||
spec:
|
||||
clusterIP: None
|
||||
selector:
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/component: qdrant
|
||||
ports:
|
||||
- { name: http, port: 6333, targetPort: http }
|
||||
- { name: grpc, port: 6334, targetPort: grpc }
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: StatefulSet
|
||||
metadata:
|
||||
name: {{ include "medical-chatbot.fullname" . }}-qdrant
|
||||
labels:
|
||||
{{- include "medical-chatbot.labels" . | nindent 4 }}
|
||||
spec:
|
||||
serviceName: {{ include "medical-chatbot.fullname" . }}-qdrant
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/component: qdrant
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/component: qdrant
|
||||
spec:
|
||||
containers:
|
||||
- name: qdrant
|
||||
image: {{ .Values.qdrant.image }}
|
||||
ports:
|
||||
- { name: http, containerPort: 6333 }
|
||||
- { name: grpc, containerPort: 6334 }
|
||||
readinessProbe:
|
||||
httpGet: { path: /readyz, port: http }
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 10
|
||||
livenessProbe:
|
||||
httpGet: { path: /healthz, port: http }
|
||||
initialDelaySeconds: 20
|
||||
periodSeconds: 20
|
||||
resources:
|
||||
{{- toYaml .Values.qdrant.resources | nindent 12 }}
|
||||
volumeMounts:
|
||||
- { name: data, mountPath: /qdrant/storage }
|
||||
volumeClaimTemplates:
|
||||
- metadata: { name: data }
|
||||
spec:
|
||||
accessModes: [ReadWriteOnce]
|
||||
resources:
|
||||
requests: { storage: {{ .Values.qdrant.storage }} }
|
||||
{{- end }}
|
||||
@@ -0,0 +1,32 @@
|
||||
{{- if .Values.ingress.enabled }}
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: {{ include "medical-chatbot.fullname" . }}
|
||||
labels:
|
||||
{{- include "medical-chatbot.labels" . | nindent 4 }}
|
||||
annotations:
|
||||
{{- toYaml .Values.ingress.annotations | nindent 4 }}
|
||||
spec:
|
||||
ingressClassName: {{ .Values.ingress.className }}
|
||||
{{- with .Values.ingress.tls }}
|
||||
tls:
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
rules:
|
||||
- host: {{ .Values.ingress.host | quote }}
|
||||
http:
|
||||
paths:
|
||||
- path: /v1/rag
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: {{ include "medical-chatbot.fullname" . }}-ai-service
|
||||
port: { name: http }
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: {{ include "medical-chatbot.fullname" . }}-web
|
||||
port: { name: http }
|
||||
{{- end }}
|
||||
@@ -0,0 +1,170 @@
|
||||
{{- if .Values.observability.enabled }}
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: {{ include "medical-chatbot.fullname" . }}-prometheus-config
|
||||
labels:
|
||||
{{- include "medical-chatbot.labels" . | nindent 4 }}
|
||||
data:
|
||||
prometheus.yml: |
|
||||
global:
|
||||
scrape_interval: 15s
|
||||
evaluation_interval: 15s
|
||||
storage:
|
||||
exemplars:
|
||||
max_exemplars: 100000
|
||||
scrape_configs:
|
||||
- job_name: ai-service
|
||||
metrics_path: /metrics
|
||||
static_configs:
|
||||
- targets: [{{ printf "%s-ai-service:%v" (include "medical-chatbot.fullname" .) .Values.aiService.service.port | quote }}]
|
||||
labels:
|
||||
service: ai-service
|
||||
env: {{ .Values.global.environment | quote }}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: {{ include "medical-chatbot.fullname" . }}-tempo-config
|
||||
labels:
|
||||
{{- include "medical-chatbot.labels" . | nindent 4 }}
|
||||
data:
|
||||
tempo.yml: |
|
||||
server:
|
||||
http_listen_port: 3200
|
||||
distributor:
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
grpc: { endpoint: 0.0.0.0:4317 }
|
||||
http: { endpoint: 0.0.0.0:4318 }
|
||||
ingester:
|
||||
max_block_duration: 5m
|
||||
compactor:
|
||||
compaction:
|
||||
block_retention: {{ .Values.observability.tempo.retention }}
|
||||
storage:
|
||||
trace:
|
||||
backend: local
|
||||
wal: { path: /var/tempo/wal }
|
||||
local: { path: /var/tempo/blocks }
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: {{ include "medical-chatbot.fullname" . }}-otel-config
|
||||
labels:
|
||||
{{- include "medical-chatbot.labels" . | nindent 4 }}
|
||||
data:
|
||||
collector.yml: |
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
grpc: { endpoint: 0.0.0.0:4317 }
|
||||
http: { endpoint: 0.0.0.0:4318 }
|
||||
processors:
|
||||
memory_limiter: { check_interval: 1s, limit_mib: 256, spike_limit_mib: 64 }
|
||||
batch: { timeout: 2s, send_batch_size: 512 }
|
||||
exporters:
|
||||
otlp/tempo:
|
||||
endpoint: {{ include "medical-chatbot.fullname" . }}-tempo:4317
|
||||
tls: { insecure: true }
|
||||
extensions:
|
||||
health_check: { endpoint: 0.0.0.0:13133 }
|
||||
service:
|
||||
extensions: [health_check]
|
||||
pipelines:
|
||||
traces:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [otlp/tempo]
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: {{ include "medical-chatbot.fullname" . }}-grafana-provisioning
|
||||
labels:
|
||||
{{- include "medical-chatbot.labels" . | nindent 4 }}
|
||||
data:
|
||||
datasources.yml: |
|
||||
apiVersion: 1
|
||||
datasources:
|
||||
- name: Prometheus
|
||||
uid: prometheus
|
||||
type: prometheus
|
||||
access: proxy
|
||||
url: http://{{ include "medical-chatbot.fullname" . }}-prometheus:9090
|
||||
isDefault: true
|
||||
editable: false
|
||||
jsonData:
|
||||
httpMethod: POST
|
||||
exemplarTraceIdDestinations:
|
||||
- { datasourceUid: tempo, name: trace_id }
|
||||
- name: Tempo
|
||||
uid: tempo
|
||||
type: tempo
|
||||
access: proxy
|
||||
url: http://{{ include "medical-chatbot.fullname" . }}-tempo:3200
|
||||
editable: false
|
||||
jsonData:
|
||||
nodeGraph: { enabled: true }
|
||||
serviceMap: { datasourceUid: prometheus }
|
||||
tracesToMetrics:
|
||||
datasourceUid: prometheus
|
||||
spanStartTimeShift: -2m
|
||||
spanEndTimeShift: 2m
|
||||
tags:
|
||||
- { key: service.name, value: service }
|
||||
dashboards.yml: |
|
||||
apiVersion: 1
|
||||
providers:
|
||||
- name: duocthu
|
||||
folder: Dược Thư
|
||||
type: file
|
||||
disableDeletion: false
|
||||
updateIntervalSeconds: 30
|
||||
options: { path: /var/lib/grafana/dashboards }
|
||||
dashboard.json: |
|
||||
{
|
||||
"uid": "duocthu-observability",
|
||||
"title": "Dược Thư — Request path observability",
|
||||
"tags": ["duocthu", "rag", "opentelemetry"],
|
||||
"schemaVersion": 39,
|
||||
"refresh": "10s",
|
||||
"time": {"from": "now-1h", "to": "now"},
|
||||
"panels": [
|
||||
{
|
||||
"id": 1, "type": "timeseries", "title": "Request p50/p95 — exemplars open Tempo",
|
||||
"gridPos": {"h": 9, "w": 12, "x": 0, "y": 0},
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"targets": [
|
||||
{"refId": "A", "expr": "histogram_quantile(0.50, sum by (le) (rate(duocthu_request_duration_seconds_bucket{route=\"/v1/rag/query\"}[5m])))", "legendFormat": "p50"},
|
||||
{"refId": "B", "expr": "histogram_quantile(0.95, sum by (le) (rate(duocthu_request_duration_seconds_bucket{route=\"/v1/rag/query\"}[5m])))", "legendFormat": "p95"}
|
||||
],
|
||||
"fieldConfig": {"defaults": {"unit": "s"}, "overrides": []}
|
||||
},
|
||||
{
|
||||
"id": 2, "type": "timeseries", "title": "Stage p95 latency",
|
||||
"gridPos": {"h": 9, "w": 12, "x": 12, "y": 0},
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"targets": [{"refId": "A", "expr": "histogram_quantile(0.95, sum by (stage, le) (rate(duocthu_stage_duration_seconds_bucket[5m])))", "legendFormat": "{{`{{stage}}`}}"}],
|
||||
"fieldConfig": {"defaults": {"unit": "s"}, "overrides": []}
|
||||
},
|
||||
{
|
||||
"id": 3, "type": "timeseries", "title": "Decision / reason",
|
||||
"gridPos": {"h": 9, "w": 12, "x": 0, "y": 9},
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"targets": [{"refId": "A", "expr": "sum by (decision, reason) (rate(duocthu_decision_total[5m]))", "legendFormat": "{{`{{decision}}`}} · {{`{{reason}}`}}"}]
|
||||
},
|
||||
{
|
||||
"id": 4, "type": "timeseries", "title": "Provider and trace-write failures",
|
||||
"gridPos": {"h": 9, "w": 12, "x": 12, "y": 9},
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"targets": [
|
||||
{"refId": "A", "expr": "sum by (provider, operation, reason) (rate(duocthu_provider_failure_total[5m]))", "legendFormat": "{{`{{provider}}`}} · {{`{{operation}}`}} · {{`{{reason}}`}}"},
|
||||
{"refId": "B", "expr": "rate(duocthu_trace_write_failed_total[5m])", "legendFormat": "trace write"}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,292 @@
|
||||
{{- if .Values.observability.enabled }}
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: {{ include "medical-chatbot.fullname" . }}-prometheus
|
||||
labels:
|
||||
{{- include "medical-chatbot.labels" . | nindent 4 }}
|
||||
spec:
|
||||
accessModes: [ReadWriteOnce]
|
||||
resources:
|
||||
requests: { storage: {{ .Values.observability.prometheus.storage }} }
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: {{ include "medical-chatbot.fullname" . }}-prometheus
|
||||
labels:
|
||||
{{- include "medical-chatbot.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: prometheus
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/component: prometheus
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/component: prometheus
|
||||
spec:
|
||||
securityContext: { fsGroup: 65534 }
|
||||
containers:
|
||||
- name: prometheus
|
||||
image: {{ .Values.observability.prometheus.image }}
|
||||
args:
|
||||
- --config.file=/etc/prometheus/prometheus.yml
|
||||
- --storage.tsdb.path=/prometheus
|
||||
- --storage.tsdb.retention.time={{ .Values.observability.prometheus.retention }}
|
||||
- --enable-feature=exemplar-storage
|
||||
ports:
|
||||
- { name: http, containerPort: 9090 }
|
||||
readinessProbe:
|
||||
httpGet: { path: /-/ready, port: http }
|
||||
livenessProbe:
|
||||
httpGet: { path: /-/healthy, port: http }
|
||||
initialDelaySeconds: 15
|
||||
resources:
|
||||
{{- toYaml .Values.observability.prometheus.resources | nindent 12 }}
|
||||
volumeMounts:
|
||||
- { name: config, mountPath: /etc/prometheus }
|
||||
- { name: data, mountPath: /prometheus }
|
||||
volumes:
|
||||
- name: config
|
||||
configMap: { name: {{ include "medical-chatbot.fullname" . }}-prometheus-config }
|
||||
- name: data
|
||||
persistentVolumeClaim: { claimName: {{ include "medical-chatbot.fullname" . }}-prometheus }
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{ include "medical-chatbot.fullname" . }}-prometheus
|
||||
labels:
|
||||
{{- include "medical-chatbot.labels" . | nindent 4 }}
|
||||
spec:
|
||||
selector:
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/component: prometheus
|
||||
ports:
|
||||
- { name: http, port: 9090, targetPort: http }
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: {{ include "medical-chatbot.fullname" . }}-tempo
|
||||
labels:
|
||||
{{- include "medical-chatbot.labels" . | nindent 4 }}
|
||||
spec:
|
||||
accessModes: [ReadWriteOnce]
|
||||
resources:
|
||||
requests: { storage: {{ .Values.observability.tempo.storage }} }
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: {{ include "medical-chatbot.fullname" . }}-tempo
|
||||
labels:
|
||||
{{- include "medical-chatbot.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: tempo
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/component: tempo
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/component: tempo
|
||||
spec:
|
||||
securityContext: { fsGroup: 10001 }
|
||||
containers:
|
||||
- name: tempo
|
||||
image: {{ .Values.observability.tempo.image }}
|
||||
args: ["-config.file=/etc/tempo/tempo.yml"]
|
||||
ports:
|
||||
- { name: http, containerPort: 3200 }
|
||||
- { name: otlp-grpc, containerPort: 4317 }
|
||||
readinessProbe:
|
||||
httpGet: { path: /ready, port: http }
|
||||
initialDelaySeconds: 5
|
||||
livenessProbe:
|
||||
httpGet: { path: /ready, port: http }
|
||||
initialDelaySeconds: 20
|
||||
resources:
|
||||
{{- toYaml .Values.observability.tempo.resources | nindent 12 }}
|
||||
volumeMounts:
|
||||
- { name: config, mountPath: /etc/tempo }
|
||||
- { name: data, mountPath: /var/tempo }
|
||||
volumes:
|
||||
- name: config
|
||||
configMap: { name: {{ include "medical-chatbot.fullname" . }}-tempo-config }
|
||||
- name: data
|
||||
persistentVolumeClaim: { claimName: {{ include "medical-chatbot.fullname" . }}-tempo }
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{ include "medical-chatbot.fullname" . }}-tempo
|
||||
labels:
|
||||
{{- include "medical-chatbot.labels" . | nindent 4 }}
|
||||
spec:
|
||||
selector:
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/component: tempo
|
||||
ports:
|
||||
- { name: http, port: 3200, targetPort: http }
|
||||
- { name: otlp-grpc, port: 4317, targetPort: otlp-grpc }
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: {{ include "medical-chatbot.fullname" . }}-otel-collector
|
||||
labels:
|
||||
{{- include "medical-chatbot.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: otel-collector
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/component: otel-collector
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/component: otel-collector
|
||||
spec:
|
||||
containers:
|
||||
- name: otel-collector
|
||||
image: {{ .Values.observability.collector.image }}
|
||||
args: ["--config=/etc/otelcol/collector.yml"]
|
||||
ports:
|
||||
- { name: otlp-grpc, containerPort: 4317 }
|
||||
- { name: otlp-http, containerPort: 4318 }
|
||||
- { name: health, containerPort: 13133 }
|
||||
readinessProbe:
|
||||
httpGet: { path: /, port: health }
|
||||
livenessProbe:
|
||||
httpGet: { path: /, port: health }
|
||||
initialDelaySeconds: 10
|
||||
resources:
|
||||
{{- toYaml .Values.observability.collector.resources | nindent 12 }}
|
||||
volumeMounts:
|
||||
- { name: config, mountPath: /etc/otelcol }
|
||||
volumes:
|
||||
- name: config
|
||||
configMap: { name: {{ include "medical-chatbot.fullname" . }}-otel-config }
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{ include "medical-chatbot.fullname" . }}-otel-collector
|
||||
labels:
|
||||
{{- include "medical-chatbot.labels" . | nindent 4 }}
|
||||
spec:
|
||||
selector:
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/component: otel-collector
|
||||
ports:
|
||||
- { name: otlp-grpc, port: 4317, targetPort: otlp-grpc }
|
||||
- { name: otlp-http, port: 4318, targetPort: otlp-http }
|
||||
- { name: health, port: 13133, targetPort: health }
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: {{ include "medical-chatbot.fullname" . }}-grafana
|
||||
labels:
|
||||
{{- include "medical-chatbot.labels" . | nindent 4 }}
|
||||
spec:
|
||||
accessModes: [ReadWriteOnce]
|
||||
resources:
|
||||
requests: { storage: {{ .Values.observability.grafana.storage }} }
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: {{ include "medical-chatbot.fullname" . }}-grafana
|
||||
labels:
|
||||
{{- include "medical-chatbot.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: grafana
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/component: grafana
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/component: grafana
|
||||
spec:
|
||||
securityContext: { fsGroup: 472 }
|
||||
containers:
|
||||
- name: grafana
|
||||
image: {{ .Values.observability.grafana.image }}
|
||||
ports:
|
||||
- { name: http, containerPort: 3000 }
|
||||
env:
|
||||
- { name: GF_SECURITY_ADMIN_USER, value: admin }
|
||||
- name: GF_SECURITY_ADMIN_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ include "medical-chatbot.secretName" . }}
|
||||
key: grafana-admin-password
|
||||
- name: GF_AUTH_ANONYMOUS_ENABLED
|
||||
value: {{ .Values.observability.grafana.anonymousAdmin | quote }}
|
||||
- name: GF_AUTH_ANONYMOUS_ORG_ROLE
|
||||
value: Admin
|
||||
- name: GF_AUTH_DISABLE_LOGIN_FORM
|
||||
value: {{ .Values.observability.grafana.anonymousAdmin | quote }}
|
||||
readinessProbe:
|
||||
httpGet: { path: /api/health, port: http }
|
||||
initialDelaySeconds: 10
|
||||
livenessProbe:
|
||||
httpGet: { path: /api/health, port: http }
|
||||
initialDelaySeconds: 30
|
||||
resources:
|
||||
{{- toYaml .Values.observability.grafana.resources | nindent 12 }}
|
||||
volumeMounts:
|
||||
- { name: datasource, mountPath: /etc/grafana/provisioning/datasources }
|
||||
- { name: dashboard-provider, mountPath: /etc/grafana/provisioning/dashboards }
|
||||
- { name: dashboards, mountPath: /var/lib/grafana/dashboards }
|
||||
- { name: data, mountPath: /var/lib/grafana }
|
||||
volumes:
|
||||
- name: datasource
|
||||
configMap:
|
||||
name: {{ include "medical-chatbot.fullname" . }}-grafana-provisioning
|
||||
items: [{ key: datasources.yml, path: datasources.yml }]
|
||||
- name: dashboard-provider
|
||||
configMap:
|
||||
name: {{ include "medical-chatbot.fullname" . }}-grafana-provisioning
|
||||
items: [{ key: dashboards.yml, path: dashboards.yml }]
|
||||
- name: dashboards
|
||||
configMap:
|
||||
name: {{ include "medical-chatbot.fullname" . }}-grafana-provisioning
|
||||
items: [{ key: dashboard.json, path: dashboard.json }]
|
||||
- name: data
|
||||
persistentVolumeClaim: { claimName: {{ include "medical-chatbot.fullname" . }}-grafana }
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{ include "medical-chatbot.fullname" . }}-grafana
|
||||
labels:
|
||||
{{- include "medical-chatbot.labels" . | nindent 4 }}
|
||||
spec:
|
||||
type: {{ .Values.observability.grafana.service.type }}
|
||||
selector:
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/component: grafana
|
||||
ports:
|
||||
- name: http
|
||||
port: {{ .Values.observability.grafana.service.port }}
|
||||
targetPort: http
|
||||
{{- if and (eq .Values.observability.grafana.service.type "NodePort") .Values.observability.grafana.service.nodePort }}
|
||||
nodePort: {{ .Values.observability.grafana.service.nodePort }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,13 @@
|
||||
{{- if .Values.secret.create }}
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: {{ include "medical-chatbot.secretName" . }}
|
||||
labels:
|
||||
{{- include "medical-chatbot.labels" . | nindent 4 }}
|
||||
type: Opaque
|
||||
stringData:
|
||||
postgres-password: {{ .Values.secret.postgresPassword | quote }}
|
||||
postgres-dsn: {{ printf "postgresql://duoc_thu:%s@%s-postgres:5432/duoc_thu" .Values.secret.postgresPassword (include "medical-chatbot.fullname" .) | quote }}
|
||||
grafana-admin-password: {{ .Values.secret.grafanaAdminPassword | quote }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,10 @@
|
||||
{{- if .Values.serviceAccount.create }}
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: {{ include "medical-chatbot.serviceAccountName" . }}
|
||||
labels:
|
||||
{{- include "medical-chatbot.labels" . | nindent 4 }}
|
||||
annotations:
|
||||
{{- toYaml .Values.serviceAccount.annotations | nindent 4 }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,18 @@
|
||||
{{- if .Values.serviceMonitor.enabled }}
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: ServiceMonitor
|
||||
metadata:
|
||||
name: {{ include "medical-chatbot.fullname" . }}-ai-service
|
||||
labels:
|
||||
{{- include "medical-chatbot.labels" . | nindent 4 }}
|
||||
{{- toYaml .Values.serviceMonitor.additionalLabels | nindent 4 }}
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/component: ai-service
|
||||
endpoints:
|
||||
- port: http
|
||||
path: /metrics
|
||||
interval: {{ .Values.serviceMonitor.interval }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,60 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: {{ include "medical-chatbot.fullname" . }}-web
|
||||
labels:
|
||||
{{- include "medical-chatbot.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: web
|
||||
spec:
|
||||
replicas: {{ .Values.web.replicaCount }}
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/component: web
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/component: web
|
||||
spec:
|
||||
imagePullSecrets:
|
||||
{{- toYaml .Values.global.imagePullSecrets | nindent 8 }}
|
||||
containers:
|
||||
- name: web
|
||||
image: "{{ .Values.web.image.repository }}:{{ .Values.web.image.tag }}"
|
||||
imagePullPolicy: {{ .Values.web.image.pullPolicy }}
|
||||
env:
|
||||
- name: AI_SERVICE_URL
|
||||
value: {{ printf "http://%s-ai-service:%v" (include "medical-chatbot.fullname" .) .Values.aiService.service.port | quote }}
|
||||
ports:
|
||||
- { name: http, containerPort: 3000 }
|
||||
readinessProbe:
|
||||
httpGet: { path: /, port: http }
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 10
|
||||
livenessProbe:
|
||||
httpGet: { path: /, port: http }
|
||||
initialDelaySeconds: 15
|
||||
periodSeconds: 20
|
||||
resources:
|
||||
{{- toYaml .Values.web.resources | nindent 12 }}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{ include "medical-chatbot.fullname" . }}-web
|
||||
labels:
|
||||
{{- include "medical-chatbot.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: web
|
||||
spec:
|
||||
type: {{ .Values.web.service.type }}
|
||||
selector:
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/component: web
|
||||
ports:
|
||||
- name: http
|
||||
port: {{ .Values.web.service.port }}
|
||||
targetPort: http
|
||||
{{- if and (eq .Values.web.service.type "NodePort") .Values.web.service.nodePort }}
|
||||
nodePort: {{ .Values.web.service.nodePort }}
|
||||
{{- end }}
|
||||
@@ -1 +1,17 @@
|
||||
# dev environment overrides (TBD, Phase 6)
|
||||
global:
|
||||
environment: docker-desktop
|
||||
|
||||
web:
|
||||
service:
|
||||
type: NodePort
|
||||
nodePort: 30080
|
||||
|
||||
observability:
|
||||
grafana:
|
||||
anonymousAdmin: true
|
||||
service:
|
||||
type: NodePort
|
||||
nodePort: 30082
|
||||
|
||||
serviceMonitor:
|
||||
enabled: false
|
||||
|
||||
@@ -1 +1,31 @@
|
||||
# prod environment overrides (TBD, Phase 6)
|
||||
global:
|
||||
environment: production
|
||||
|
||||
aiService:
|
||||
replicaCount: 2
|
||||
image:
|
||||
tag: latest
|
||||
config:
|
||||
embeddingProvider: cohere-v4
|
||||
answerProvider: bedrock-converse
|
||||
otelSampleRatio: 0.25
|
||||
|
||||
web:
|
||||
replicaCount: 2
|
||||
image:
|
||||
tag: latest
|
||||
|
||||
ingress:
|
||||
enabled: true
|
||||
host: realvuxbaro.me
|
||||
|
||||
secret:
|
||||
create: false
|
||||
existingSecret: medical-chatbot-prod
|
||||
|
||||
observability:
|
||||
prometheus:
|
||||
retention: 15d
|
||||
|
||||
serviceMonitor:
|
||||
enabled: false
|
||||
|
||||
@@ -1 +1,16 @@
|
||||
# staging environment overrides (TBD, Phase 6)
|
||||
global:
|
||||
environment: staging
|
||||
|
||||
aiService:
|
||||
config:
|
||||
embeddingProvider: cohere-v4
|
||||
answerProvider: bedrock-converse
|
||||
otelSampleRatio: 1.0
|
||||
|
||||
ingress:
|
||||
enabled: true
|
||||
host: staging.duocthu.example.com
|
||||
|
||||
secret:
|
||||
create: false
|
||||
existingSecret: medical-chatbot-staging
|
||||
|
||||
@@ -1,2 +1,119 @@
|
||||
# Base values — filled in during Phase 6. Overridden per-environment by
|
||||
# values-dev.yaml / values-staging.yaml / values-prod.yaml.
|
||||
nameOverride: ""
|
||||
fullnameOverride: ""
|
||||
|
||||
global:
|
||||
environment: local
|
||||
imagePullSecrets: []
|
||||
|
||||
serviceAccount:
|
||||
create: true
|
||||
name: ""
|
||||
annotations: {}
|
||||
|
||||
secret:
|
||||
create: true
|
||||
existingSecret: ""
|
||||
postgresPassword: duoc_thu
|
||||
grafanaAdminPassword: change-me
|
||||
|
||||
aiService:
|
||||
replicaCount: 1
|
||||
image:
|
||||
repository: duocthu-ai-service
|
||||
tag: local
|
||||
pullPolicy: IfNotPresent
|
||||
service:
|
||||
type: ClusterIP
|
||||
port: 8000
|
||||
config:
|
||||
embeddingProvider: disabled
|
||||
answerProvider: disabled
|
||||
answerModelId: deepseek.v3.2
|
||||
qdrantCollection: duocthu_v1
|
||||
metricsEnabled: true
|
||||
otelEnabled: true
|
||||
otelSampleRatio: 1.0
|
||||
maxWallClockMs: 40000
|
||||
maxLlmCallsPerTurn: 8
|
||||
migration:
|
||||
enabled: true
|
||||
resources:
|
||||
requests: { cpu: 100m, memory: 256Mi }
|
||||
limits: { cpu: "1", memory: 1Gi }
|
||||
|
||||
web:
|
||||
replicaCount: 1
|
||||
image:
|
||||
repository: duocthu-web
|
||||
tag: local
|
||||
pullPolicy: IfNotPresent
|
||||
service:
|
||||
type: ClusterIP
|
||||
port: 3000
|
||||
nodePort: null
|
||||
resources:
|
||||
requests: { cpu: 50m, memory: 128Mi }
|
||||
limits: { cpu: 500m, memory: 512Mi }
|
||||
|
||||
ingress:
|
||||
enabled: false
|
||||
className: nginx
|
||||
annotations: {}
|
||||
host: duocthu.local
|
||||
tls: []
|
||||
|
||||
postgres:
|
||||
enabled: true
|
||||
image: postgres:16-alpine
|
||||
storage: 5Gi
|
||||
resources:
|
||||
requests: { cpu: 50m, memory: 128Mi }
|
||||
limits: { cpu: 500m, memory: 512Mi }
|
||||
|
||||
qdrant:
|
||||
enabled: true
|
||||
# Set when using an external/managed Qdrant; empty uses the bundled Service.
|
||||
url: ""
|
||||
image: qdrant/qdrant:v1.13.4
|
||||
storage: 10Gi
|
||||
resources:
|
||||
requests: { cpu: 100m, memory: 256Mi }
|
||||
limits: { cpu: "1", memory: 1Gi }
|
||||
|
||||
observability:
|
||||
enabled: true
|
||||
prometheus:
|
||||
image: prom/prometheus:v3.3.0
|
||||
retention: 7d
|
||||
storage: 5Gi
|
||||
resources:
|
||||
requests: { cpu: 100m, memory: 256Mi }
|
||||
limits: { cpu: 500m, memory: 1Gi }
|
||||
tempo:
|
||||
image: grafana/tempo:2.7.2
|
||||
retention: 24h
|
||||
storage: 5Gi
|
||||
resources:
|
||||
requests: { cpu: 100m, memory: 256Mi }
|
||||
limits: { cpu: 500m, memory: 1Gi }
|
||||
collector:
|
||||
image: otel/opentelemetry-collector-contrib:0.123.0
|
||||
resources:
|
||||
requests: { cpu: 50m, memory: 128Mi }
|
||||
limits: { cpu: 500m, memory: 512Mi }
|
||||
grafana:
|
||||
image: grafana/grafana:11.5.2
|
||||
service:
|
||||
type: ClusterIP
|
||||
port: 3000
|
||||
nodePort: null
|
||||
storage: 2Gi
|
||||
anonymousAdmin: false
|
||||
resources:
|
||||
requests: { cpu: 50m, memory: 128Mi }
|
||||
limits: { cpu: 500m, memory: 512Mi }
|
||||
|
||||
serviceMonitor:
|
||||
enabled: false
|
||||
interval: 15s
|
||||
additionalLabels: {}
|
||||
|
||||
Reference in New Issue
Block a user