171 lines
5.9 KiB
YAML
171 lines
5.9 KiB
YAML
{{- if .Values.observability.enabled }}
|
|
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: {{ include "medical-chatbot.fullname" . }}-prometheus-config
|
|
labels:
|
|
{{- include "medical-chatbot.labels" . | nindent 4 }}
|
|
data:
|
|
prometheus.yml: |
|
|
global:
|
|
scrape_interval: 15s
|
|
evaluation_interval: 15s
|
|
storage:
|
|
exemplars:
|
|
max_exemplars: 100000
|
|
scrape_configs:
|
|
- job_name: ai-service
|
|
metrics_path: /metrics
|
|
static_configs:
|
|
- targets: [{{ printf "%s-ai-service:%v" (include "medical-chatbot.fullname" .) .Values.aiService.service.port | quote }}]
|
|
labels:
|
|
service: ai-service
|
|
env: {{ .Values.global.environment | quote }}
|
|
---
|
|
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: {{ include "medical-chatbot.fullname" . }}-tempo-config
|
|
labels:
|
|
{{- include "medical-chatbot.labels" . | nindent 4 }}
|
|
data:
|
|
tempo.yml: |
|
|
server:
|
|
http_listen_port: 3200
|
|
distributor:
|
|
receivers:
|
|
otlp:
|
|
protocols:
|
|
grpc: { endpoint: 0.0.0.0:4317 }
|
|
http: { endpoint: 0.0.0.0:4318 }
|
|
ingester:
|
|
max_block_duration: 5m
|
|
compactor:
|
|
compaction:
|
|
block_retention: {{ .Values.observability.tempo.retention }}
|
|
storage:
|
|
trace:
|
|
backend: local
|
|
wal: { path: /var/tempo/wal }
|
|
local: { path: /var/tempo/blocks }
|
|
---
|
|
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: {{ include "medical-chatbot.fullname" . }}-otel-config
|
|
labels:
|
|
{{- include "medical-chatbot.labels" . | nindent 4 }}
|
|
data:
|
|
collector.yml: |
|
|
receivers:
|
|
otlp:
|
|
protocols:
|
|
grpc: { endpoint: 0.0.0.0:4317 }
|
|
http: { endpoint: 0.0.0.0:4318 }
|
|
processors:
|
|
memory_limiter: { check_interval: 1s, limit_mib: 256, spike_limit_mib: 64 }
|
|
batch: { timeout: 2s, send_batch_size: 512 }
|
|
exporters:
|
|
otlp/tempo:
|
|
endpoint: {{ include "medical-chatbot.fullname" . }}-tempo:4317
|
|
tls: { insecure: true }
|
|
extensions:
|
|
health_check: { endpoint: 0.0.0.0:13133 }
|
|
service:
|
|
extensions: [health_check]
|
|
pipelines:
|
|
traces:
|
|
receivers: [otlp]
|
|
processors: [memory_limiter, batch]
|
|
exporters: [otlp/tempo]
|
|
---
|
|
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: {{ include "medical-chatbot.fullname" . }}-grafana-provisioning
|
|
labels:
|
|
{{- include "medical-chatbot.labels" . | nindent 4 }}
|
|
data:
|
|
datasources.yml: |
|
|
apiVersion: 1
|
|
datasources:
|
|
- name: Prometheus
|
|
uid: prometheus
|
|
type: prometheus
|
|
access: proxy
|
|
url: http://{{ include "medical-chatbot.fullname" . }}-prometheus:9090
|
|
isDefault: true
|
|
editable: false
|
|
jsonData:
|
|
httpMethod: POST
|
|
exemplarTraceIdDestinations:
|
|
- { datasourceUid: tempo, name: trace_id }
|
|
- name: Tempo
|
|
uid: tempo
|
|
type: tempo
|
|
access: proxy
|
|
url: http://{{ include "medical-chatbot.fullname" . }}-tempo:3200
|
|
editable: false
|
|
jsonData:
|
|
nodeGraph: { enabled: true }
|
|
serviceMap: { datasourceUid: prometheus }
|
|
tracesToMetrics:
|
|
datasourceUid: prometheus
|
|
spanStartTimeShift: -2m
|
|
spanEndTimeShift: 2m
|
|
tags:
|
|
- { key: service.name, value: service }
|
|
dashboards.yml: |
|
|
apiVersion: 1
|
|
providers:
|
|
- name: duocthu
|
|
folder: Dược Thư
|
|
type: file
|
|
disableDeletion: false
|
|
updateIntervalSeconds: 30
|
|
options: { path: /var/lib/grafana/dashboards }
|
|
dashboard.json: |
|
|
{
|
|
"uid": "duocthu-observability",
|
|
"title": "Dược Thư — Request path observability",
|
|
"tags": ["duocthu", "rag", "opentelemetry"],
|
|
"schemaVersion": 39,
|
|
"refresh": "10s",
|
|
"time": {"from": "now-1h", "to": "now"},
|
|
"panels": [
|
|
{
|
|
"id": 1, "type": "timeseries", "title": "Request p50/p95 — exemplars open Tempo",
|
|
"gridPos": {"h": 9, "w": 12, "x": 0, "y": 0},
|
|
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
|
"targets": [
|
|
{"refId": "A", "expr": "histogram_quantile(0.50, sum by (le) (rate(duocthu_request_duration_seconds_bucket{route=\"/v1/rag/query\"}[5m])))", "legendFormat": "p50"},
|
|
{"refId": "B", "expr": "histogram_quantile(0.95, sum by (le) (rate(duocthu_request_duration_seconds_bucket{route=\"/v1/rag/query\"}[5m])))", "legendFormat": "p95"}
|
|
],
|
|
"fieldConfig": {"defaults": {"unit": "s"}, "overrides": []}
|
|
},
|
|
{
|
|
"id": 2, "type": "timeseries", "title": "Stage p95 latency",
|
|
"gridPos": {"h": 9, "w": 12, "x": 12, "y": 0},
|
|
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
|
"targets": [{"refId": "A", "expr": "histogram_quantile(0.95, sum by (stage, le) (rate(duocthu_stage_duration_seconds_bucket[5m])))", "legendFormat": "{{`{{stage}}`}}"}],
|
|
"fieldConfig": {"defaults": {"unit": "s"}, "overrides": []}
|
|
},
|
|
{
|
|
"id": 3, "type": "timeseries", "title": "Decision / reason",
|
|
"gridPos": {"h": 9, "w": 12, "x": 0, "y": 9},
|
|
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
|
"targets": [{"refId": "A", "expr": "sum by (decision, reason) (rate(duocthu_decision_total[5m]))", "legendFormat": "{{`{{decision}}`}} · {{`{{reason}}`}}"}]
|
|
},
|
|
{
|
|
"id": 4, "type": "timeseries", "title": "Provider and trace-write failures",
|
|
"gridPos": {"h": 9, "w": 12, "x": 12, "y": 9},
|
|
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
|
"targets": [
|
|
{"refId": "A", "expr": "sum by (provider, operation, reason) (rate(duocthu_provider_failure_total[5m]))", "legendFormat": "{{`{{provider}}`}} · {{`{{operation}}`}} · {{`{{reason}}`}}"},
|
|
{"refId": "B", "expr": "rate(duocthu_trace_write_failed_total[5m])", "legendFormat": "trace write"}
|
|
]
|
|
}
|
|
]
|
|
}
|
|
{{- end }}
|