Verify exact production traces and record rollout
This commit is contained in:
@@ -0,0 +1,170 @@
|
||||
{{- if .Values.observability.enabled }}
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: {{ include "medical-chatbot.fullname" . }}-prometheus-config
|
||||
labels:
|
||||
{{- include "medical-chatbot.labels" . | nindent 4 }}
|
||||
data:
|
||||
prometheus.yml: |
|
||||
global:
|
||||
scrape_interval: 15s
|
||||
evaluation_interval: 15s
|
||||
storage:
|
||||
exemplars:
|
||||
max_exemplars: 100000
|
||||
scrape_configs:
|
||||
- job_name: ai-service
|
||||
metrics_path: /metrics
|
||||
static_configs:
|
||||
- targets: [{{ printf "%s-ai-service:%v" (include "medical-chatbot.fullname" .) .Values.aiService.service.port | quote }}]
|
||||
labels:
|
||||
service: ai-service
|
||||
env: {{ .Values.global.environment | quote }}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: {{ include "medical-chatbot.fullname" . }}-tempo-config
|
||||
labels:
|
||||
{{- include "medical-chatbot.labels" . | nindent 4 }}
|
||||
data:
|
||||
tempo.yml: |
|
||||
server:
|
||||
http_listen_port: 3200
|
||||
distributor:
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
grpc: { endpoint: 0.0.0.0:4317 }
|
||||
http: { endpoint: 0.0.0.0:4318 }
|
||||
ingester:
|
||||
max_block_duration: 5m
|
||||
compactor:
|
||||
compaction:
|
||||
block_retention: {{ .Values.observability.tempo.retention }}
|
||||
storage:
|
||||
trace:
|
||||
backend: local
|
||||
wal: { path: /var/tempo/wal }
|
||||
local: { path: /var/tempo/blocks }
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: {{ include "medical-chatbot.fullname" . }}-otel-config
|
||||
labels:
|
||||
{{- include "medical-chatbot.labels" . | nindent 4 }}
|
||||
data:
|
||||
collector.yml: |
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
grpc: { endpoint: 0.0.0.0:4317 }
|
||||
http: { endpoint: 0.0.0.0:4318 }
|
||||
processors:
|
||||
memory_limiter: { check_interval: 1s, limit_mib: 256, spike_limit_mib: 64 }
|
||||
batch: { timeout: 2s, send_batch_size: 512 }
|
||||
exporters:
|
||||
otlp/tempo:
|
||||
endpoint: {{ include "medical-chatbot.fullname" . }}-tempo:4317
|
||||
tls: { insecure: true }
|
||||
extensions:
|
||||
health_check: { endpoint: 0.0.0.0:13133 }
|
||||
service:
|
||||
extensions: [health_check]
|
||||
pipelines:
|
||||
traces:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [otlp/tempo]
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: {{ include "medical-chatbot.fullname" . }}-grafana-provisioning
|
||||
labels:
|
||||
{{- include "medical-chatbot.labels" . | nindent 4 }}
|
||||
data:
|
||||
datasources.yml: |
|
||||
apiVersion: 1
|
||||
datasources:
|
||||
- name: Prometheus
|
||||
uid: prometheus
|
||||
type: prometheus
|
||||
access: proxy
|
||||
url: http://{{ include "medical-chatbot.fullname" . }}-prometheus:9090
|
||||
isDefault: true
|
||||
editable: false
|
||||
jsonData:
|
||||
httpMethod: POST
|
||||
exemplarTraceIdDestinations:
|
||||
- { datasourceUid: tempo, name: trace_id }
|
||||
- name: Tempo
|
||||
uid: tempo
|
||||
type: tempo
|
||||
access: proxy
|
||||
url: http://{{ include "medical-chatbot.fullname" . }}-tempo:3200
|
||||
editable: false
|
||||
jsonData:
|
||||
nodeGraph: { enabled: true }
|
||||
serviceMap: { datasourceUid: prometheus }
|
||||
tracesToMetrics:
|
||||
datasourceUid: prometheus
|
||||
spanStartTimeShift: -2m
|
||||
spanEndTimeShift: 2m
|
||||
tags:
|
||||
- { key: service.name, value: service }
|
||||
dashboards.yml: |
|
||||
apiVersion: 1
|
||||
providers:
|
||||
- name: duocthu
|
||||
folder: Dược Thư
|
||||
type: file
|
||||
disableDeletion: false
|
||||
updateIntervalSeconds: 30
|
||||
options: { path: /var/lib/grafana/dashboards }
|
||||
dashboard.json: |
|
||||
{
|
||||
"uid": "duocthu-observability",
|
||||
"title": "Dược Thư — Request path observability",
|
||||
"tags": ["duocthu", "rag", "opentelemetry"],
|
||||
"schemaVersion": 39,
|
||||
"refresh": "10s",
|
||||
"time": {"from": "now-1h", "to": "now"},
|
||||
"panels": [
|
||||
{
|
||||
"id": 1, "type": "timeseries", "title": "Request p50/p95 — exemplars open Tempo",
|
||||
"gridPos": {"h": 9, "w": 12, "x": 0, "y": 0},
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"targets": [
|
||||
{"refId": "A", "expr": "histogram_quantile(0.50, sum by (le) (rate(duocthu_request_duration_seconds_bucket{route=\"/v1/rag/query\"}[5m])))", "legendFormat": "p50"},
|
||||
{"refId": "B", "expr": "histogram_quantile(0.95, sum by (le) (rate(duocthu_request_duration_seconds_bucket{route=\"/v1/rag/query\"}[5m])))", "legendFormat": "p95"}
|
||||
],
|
||||
"fieldConfig": {"defaults": {"unit": "s"}, "overrides": []}
|
||||
},
|
||||
{
|
||||
"id": 2, "type": "timeseries", "title": "Stage p95 latency",
|
||||
"gridPos": {"h": 9, "w": 12, "x": 12, "y": 0},
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"targets": [{"refId": "A", "expr": "histogram_quantile(0.95, sum by (stage, le) (rate(duocthu_stage_duration_seconds_bucket[5m])))", "legendFormat": "{{`{{stage}}`}}"}],
|
||||
"fieldConfig": {"defaults": {"unit": "s"}, "overrides": []}
|
||||
},
|
||||
{
|
||||
"id": 3, "type": "timeseries", "title": "Decision / reason",
|
||||
"gridPos": {"h": 9, "w": 12, "x": 0, "y": 9},
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"targets": [{"refId": "A", "expr": "sum by (decision, reason) (rate(duocthu_decision_total[5m]))", "legendFormat": "{{`{{decision}}`}} · {{`{{reason}}`}}"}]
|
||||
},
|
||||
{
|
||||
"id": 4, "type": "timeseries", "title": "Provider and trace-write failures",
|
||||
"gridPos": {"h": 9, "w": 12, "x": 12, "y": 9},
|
||||
"datasource": {"type": "prometheus", "uid": "prometheus"},
|
||||
"targets": [
|
||||
{"refId": "A", "expr": "sum by (provider, operation, reason) (rate(duocthu_provider_failure_total[5m]))", "legendFormat": "{{`{{provider}}`}} · {{`{{operation}}`}} · {{`{{reason}}`}}"},
|
||||
{"refId": "B", "expr": "rate(duocthu_trace_write_failed_total[5m])", "legendFormat": "trace write"}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
{{- end }}
|
||||
Reference in New Issue
Block a user