Verify exact production traces and record rollout

infra/helm/medical-chatbot/templates/_helpers.tpl

@@ -0,0 +1,34 @@
+{{- define "medical-chatbot.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{- define "medical-chatbot.fullname" -}}
+{{- if .Values.fullnameOverride -}}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s" .Release.Name (include "medical-chatbot.name" .) | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+
+{{- define "medical-chatbot.labels" -}}
+app.kubernetes.io/name: {{ include "medical-chatbot.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | quote }}
+{{- end -}}
+
+{{- define "medical-chatbot.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create -}}
+{{- default (include "medical-chatbot.fullname" .) .Values.serviceAccount.name -}}
+{{- else -}}
+{{- default "default" .Values.serviceAccount.name -}}
+{{- end -}}
+{{- end -}}
+
+{{- define "medical-chatbot.secretName" -}}
+{{- if .Values.secret.create -}}
+{{- printf "%s-runtime" (include "medical-chatbot.fullname" .) -}}
+{{- else -}}
+{{- required "secret.existingSecret is required when secret.create=false" .Values.secret.existingSecret -}}
+{{- end -}}
+{{- end -}}

infra/helm/medical-chatbot/templates/ai-service.yaml

@@ -0,0 +1,107 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "medical-chatbot.fullname" . }}-ai-service
+  labels:
+    {{- include "medical-chatbot.labels" . | nindent 4 }}
+data:
+  ENVIRONMENT: {{ .Values.global.environment | quote }}
+  QDRANT_URL: {{ default (printf "http://%s-qdrant:6333" (include "medical-chatbot.fullname" .)) .Values.qdrant.url | quote }}
+  QDRANT_COLLECTION: {{ .Values.aiService.config.qdrantCollection | quote }}
+  EMBEDDING_PROVIDER: {{ .Values.aiService.config.embeddingProvider | quote }}
+  ANSWER_PROVIDER: {{ .Values.aiService.config.answerProvider | quote }}
+  ANSWER_MODEL_ID: {{ .Values.aiService.config.answerModelId | quote }}
+  METRICS_ENABLED: {{ .Values.aiService.config.metricsEnabled | quote }}
+  OTEL_ENABLED: {{ and .Values.observability.enabled .Values.aiService.config.otelEnabled | quote }}
+  OTEL_SERVICE_NAME: ai-service
+  OTEL_EXPORTER_OTLP_ENDPOINT: {{ printf "http://%s-otel-collector:4318/v1/traces" (include "medical-chatbot.fullname" .) | quote }}
+  OTEL_SAMPLE_RATIO: {{ .Values.aiService.config.otelSampleRatio | quote }}
+  MAX_WALL_CLOCK_MS: {{ .Values.aiService.config.maxWallClockMs | quote }}
+  MAX_LLM_CALLS_PER_TURN: {{ .Values.aiService.config.maxLlmCallsPerTurn | quote }}
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "medical-chatbot.fullname" . }}-ai-service
+  labels:
+    {{- include "medical-chatbot.labels" . | nindent 4 }}
+    app.kubernetes.io/component: ai-service
+spec:
+  replicas: {{ .Values.aiService.replicaCount }}
+  selector:
+    matchLabels:
+      app.kubernetes.io/instance: {{ .Release.Name }}
+      app.kubernetes.io/component: ai-service
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/instance: {{ .Release.Name }}
+        app.kubernetes.io/component: ai-service
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/path: /metrics
+        prometheus.io/port: "8000"
+    spec:
+      serviceAccountName: {{ include "medical-chatbot.serviceAccountName" . }}
+      imagePullSecrets:
+        {{- toYaml .Values.global.imagePullSecrets | nindent 8 }}
+      {{- if .Values.aiService.migration.enabled }}
+      initContainers:
+        - name: migrate
+          image: "{{ .Values.aiService.image.repository }}:{{ .Values.aiService.image.tag }}"
+          imagePullPolicy: {{ .Values.aiService.image.pullPolicy }}
+          command: ["python", "migrate.py"]
+          envFrom:
+            - configMapRef: { name: {{ include "medical-chatbot.fullname" . }}-ai-service }
+          env:
+            - name: POSTGRES_DSN
+              valueFrom:
+                secretKeyRef:
+                  name: {{ include "medical-chatbot.secretName" . }}
+                  key: postgres-dsn
+      {{- end }}
+      containers:
+        - name: ai-service
+          image: "{{ .Values.aiService.image.repository }}:{{ .Values.aiService.image.tag }}"
+          imagePullPolicy: {{ .Values.aiService.image.pullPolicy }}
+          ports:
+            - { name: http, containerPort: 8000 }
+          envFrom:
+            - configMapRef: { name: {{ include "medical-chatbot.fullname" . }}-ai-service }
+          env:
+            - name: POSTGRES_DSN
+              valueFrom:
+                secretKeyRef:
+                  name: {{ include "medical-chatbot.secretName" . }}
+                  key: postgres-dsn
+          readinessProbe:
+            httpGet: { path: /ready, port: http }
+            initialDelaySeconds: 5
+            periodSeconds: 10
+          livenessProbe:
+            httpGet: { path: /health, port: http }
+            initialDelaySeconds: 15
+            periodSeconds: 20
+          startupProbe:
+            httpGet: { path: /health, port: http }
+            failureThreshold: 30
+            periodSeconds: 5
+          resources:
+            {{- toYaml .Values.aiService.resources | nindent 12 }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "medical-chatbot.fullname" . }}-ai-service
+  labels:
+    {{- include "medical-chatbot.labels" . | nindent 4 }}
+    app.kubernetes.io/component: ai-service
+spec:
+  type: {{ .Values.aiService.service.type }}
+  selector:
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/component: ai-service
+  ports:
+    - name: http
+      port: {{ .Values.aiService.service.port }}
+      targetPort: http

infra/helm/medical-chatbot/templates/data-services.yaml

@@ -0,0 +1,131 @@
+{{- if .Values.postgres.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "medical-chatbot.fullname" . }}-postgres
+  labels:
+    {{- include "medical-chatbot.labels" . | nindent 4 }}
+    app.kubernetes.io/component: postgres
+spec:
+  clusterIP: None
+  selector:
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/component: postgres
+  ports:
+    - { name: postgres, port: 5432, targetPort: postgres }
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: {{ include "medical-chatbot.fullname" . }}-postgres
+  labels:
+    {{- include "medical-chatbot.labels" . | nindent 4 }}
+spec:
+  serviceName: {{ include "medical-chatbot.fullname" . }}-postgres
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/instance: {{ .Release.Name }}
+      app.kubernetes.io/component: postgres
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/instance: {{ .Release.Name }}
+        app.kubernetes.io/component: postgres
+    spec:
+      containers:
+        - name: postgres
+          image: {{ .Values.postgres.image }}
+          ports:
+            - { name: postgres, containerPort: 5432 }
+          env:
+            - { name: POSTGRES_USER, value: duoc_thu }
+            - { name: POSTGRES_DB, value: duoc_thu }
+            - name: POSTGRES_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: {{ include "medical-chatbot.secretName" . }}
+                  key: postgres-password
+          readinessProbe:
+            exec: { command: ["pg_isready", "-U", "duoc_thu", "-d", "duoc_thu"] }
+            initialDelaySeconds: 5
+            periodSeconds: 10
+          livenessProbe:
+            exec: { command: ["pg_isready", "-U", "duoc_thu", "-d", "duoc_thu"] }
+            initialDelaySeconds: 20
+            periodSeconds: 20
+          resources:
+            {{- toYaml .Values.postgres.resources | nindent 12 }}
+          volumeMounts:
+            - { name: data, mountPath: /var/lib/postgresql/data }
+  volumeClaimTemplates:
+    - metadata: { name: data }
+      spec:
+        accessModes: [ReadWriteOnce]
+        resources:
+          requests: { storage: {{ .Values.postgres.storage }} }
+{{- end }}
+{{- if and .Values.postgres.enabled .Values.qdrant.enabled }}
+---
+{{- end }}
+{{- if .Values.qdrant.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "medical-chatbot.fullname" . }}-qdrant
+  labels:
+    {{- include "medical-chatbot.labels" . | nindent 4 }}
+    app.kubernetes.io/component: qdrant
+spec:
+  clusterIP: None
+  selector:
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/component: qdrant
+  ports:
+    - { name: http, port: 6333, targetPort: http }
+    - { name: grpc, port: 6334, targetPort: grpc }
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: {{ include "medical-chatbot.fullname" . }}-qdrant
+  labels:
+    {{- include "medical-chatbot.labels" . | nindent 4 }}
+spec:
+  serviceName: {{ include "medical-chatbot.fullname" . }}-qdrant
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/instance: {{ .Release.Name }}
+      app.kubernetes.io/component: qdrant
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/instance: {{ .Release.Name }}
+        app.kubernetes.io/component: qdrant
+    spec:
+      containers:
+        - name: qdrant
+          image: {{ .Values.qdrant.image }}
+          ports:
+            - { name: http, containerPort: 6333 }
+            - { name: grpc, containerPort: 6334 }
+          readinessProbe:
+            httpGet: { path: /readyz, port: http }
+            initialDelaySeconds: 5
+            periodSeconds: 10
+          livenessProbe:
+            httpGet: { path: /healthz, port: http }
+            initialDelaySeconds: 20
+            periodSeconds: 20
+          resources:
+            {{- toYaml .Values.qdrant.resources | nindent 12 }}
+          volumeMounts:
+            - { name: data, mountPath: /qdrant/storage }
+  volumeClaimTemplates:
+    - metadata: { name: data }
+      spec:
+        accessModes: [ReadWriteOnce]
+        resources:
+          requests: { storage: {{ .Values.qdrant.storage }} }
+{{- end }}

infra/helm/medical-chatbot/templates/ingress.yaml

@@ -0,0 +1,32 @@
+{{- if .Values.ingress.enabled }}
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: {{ include "medical-chatbot.fullname" . }}
+  labels:
+    {{- include "medical-chatbot.labels" . | nindent 4 }}
+  annotations:
+    {{- toYaml .Values.ingress.annotations | nindent 4 }}
+spec:
+  ingressClassName: {{ .Values.ingress.className }}
+  {{- with .Values.ingress.tls }}
+  tls:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+  rules:
+    - host: {{ .Values.ingress.host | quote }}
+      http:
+        paths:
+          - path: /v1/rag
+            pathType: Prefix
+            backend:
+              service:
+                name: {{ include "medical-chatbot.fullname" . }}-ai-service
+                port: { name: http }
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: {{ include "medical-chatbot.fullname" . }}-web
+                port: { name: http }
+{{- end }}

infra/helm/medical-chatbot/templates/observability-config.yaml

@@ -0,0 +1,170 @@
+{{- if .Values.observability.enabled }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "medical-chatbot.fullname" . }}-prometheus-config
+  labels:
+    {{- include "medical-chatbot.labels" . | nindent 4 }}
+data:
+  prometheus.yml: |
+    global:
+      scrape_interval: 15s
+      evaluation_interval: 15s
+    storage:
+      exemplars:
+        max_exemplars: 100000
+    scrape_configs:
+      - job_name: ai-service
+        metrics_path: /metrics
+        static_configs:
+          - targets: [{{ printf "%s-ai-service:%v" (include "medical-chatbot.fullname" .) .Values.aiService.service.port | quote }}]
+            labels:
+              service: ai-service
+              env: {{ .Values.global.environment | quote }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "medical-chatbot.fullname" . }}-tempo-config
+  labels:
+    {{- include "medical-chatbot.labels" . | nindent 4 }}
+data:
+  tempo.yml: |
+    server:
+      http_listen_port: 3200
+    distributor:
+      receivers:
+        otlp:
+          protocols:
+            grpc: { endpoint: 0.0.0.0:4317 }
+            http: { endpoint: 0.0.0.0:4318 }
+    ingester:
+      max_block_duration: 5m
+    compactor:
+      compaction:
+        block_retention: {{ .Values.observability.tempo.retention }}
+    storage:
+      trace:
+        backend: local
+        wal: { path: /var/tempo/wal }
+        local: { path: /var/tempo/blocks }
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "medical-chatbot.fullname" . }}-otel-config
+  labels:
+    {{- include "medical-chatbot.labels" . | nindent 4 }}
+data:
+  collector.yml: |
+    receivers:
+      otlp:
+        protocols:
+          grpc: { endpoint: 0.0.0.0:4317 }
+          http: { endpoint: 0.0.0.0:4318 }
+    processors:
+      memory_limiter: { check_interval: 1s, limit_mib: 256, spike_limit_mib: 64 }
+      batch: { timeout: 2s, send_batch_size: 512 }
+    exporters:
+      otlp/tempo:
+        endpoint: {{ include "medical-chatbot.fullname" . }}-tempo:4317
+        tls: { insecure: true }
+    extensions:
+      health_check: { endpoint: 0.0.0.0:13133 }
+    service:
+      extensions: [health_check]
+      pipelines:
+        traces:
+          receivers: [otlp]
+          processors: [memory_limiter, batch]
+          exporters: [otlp/tempo]
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "medical-chatbot.fullname" . }}-grafana-provisioning
+  labels:
+    {{- include "medical-chatbot.labels" . | nindent 4 }}
+data:
+  datasources.yml: |
+    apiVersion: 1
+    datasources:
+      - name: Prometheus
+        uid: prometheus
+        type: prometheus
+        access: proxy
+        url: http://{{ include "medical-chatbot.fullname" . }}-prometheus:9090
+        isDefault: true
+        editable: false
+        jsonData:
+          httpMethod: POST
+          exemplarTraceIdDestinations:
+            - { datasourceUid: tempo, name: trace_id }
+      - name: Tempo
+        uid: tempo
+        type: tempo
+        access: proxy
+        url: http://{{ include "medical-chatbot.fullname" . }}-tempo:3200
+        editable: false
+        jsonData:
+          nodeGraph: { enabled: true }
+          serviceMap: { datasourceUid: prometheus }
+          tracesToMetrics:
+            datasourceUid: prometheus
+            spanStartTimeShift: -2m
+            spanEndTimeShift: 2m
+            tags:
+              - { key: service.name, value: service }
+  dashboards.yml: |
+    apiVersion: 1
+    providers:
+      - name: duocthu
+        folder: Dược Thư
+        type: file
+        disableDeletion: false
+        updateIntervalSeconds: 30
+        options: { path: /var/lib/grafana/dashboards }
+  dashboard.json: |
+    {
+      "uid": "duocthu-observability",
+      "title": "Dược Thư — Request path observability",
+      "tags": ["duocthu", "rag", "opentelemetry"],
+      "schemaVersion": 39,
+      "refresh": "10s",
+      "time": {"from": "now-1h", "to": "now"},
+      "panels": [
+        {
+          "id": 1, "type": "timeseries", "title": "Request p50/p95 — exemplars open Tempo",
+          "gridPos": {"h": 9, "w": 12, "x": 0, "y": 0},
+          "datasource": {"type": "prometheus", "uid": "prometheus"},
+          "targets": [
+            {"refId": "A", "expr": "histogram_quantile(0.50, sum by (le) (rate(duocthu_request_duration_seconds_bucket{route=\"/v1/rag/query\"}[5m])))", "legendFormat": "p50"},
+            {"refId": "B", "expr": "histogram_quantile(0.95, sum by (le) (rate(duocthu_request_duration_seconds_bucket{route=\"/v1/rag/query\"}[5m])))", "legendFormat": "p95"}
+          ],
+          "fieldConfig": {"defaults": {"unit": "s"}, "overrides": []}
+        },
+        {
+          "id": 2, "type": "timeseries", "title": "Stage p95 latency",
+          "gridPos": {"h": 9, "w": 12, "x": 12, "y": 0},
+          "datasource": {"type": "prometheus", "uid": "prometheus"},
+          "targets": [{"refId": "A", "expr": "histogram_quantile(0.95, sum by (stage, le) (rate(duocthu_stage_duration_seconds_bucket[5m])))", "legendFormat": "{{`{{stage}}`}}"}],
+          "fieldConfig": {"defaults": {"unit": "s"}, "overrides": []}
+        },
+        {
+          "id": 3, "type": "timeseries", "title": "Decision / reason",
+          "gridPos": {"h": 9, "w": 12, "x": 0, "y": 9},
+          "datasource": {"type": "prometheus", "uid": "prometheus"},
+          "targets": [{"refId": "A", "expr": "sum by (decision, reason) (rate(duocthu_decision_total[5m]))", "legendFormat": "{{`{{decision}}`}} · {{`{{reason}}`}}"}]
+        },
+        {
+          "id": 4, "type": "timeseries", "title": "Provider and trace-write failures",
+          "gridPos": {"h": 9, "w": 12, "x": 12, "y": 9},
+          "datasource": {"type": "prometheus", "uid": "prometheus"},
+          "targets": [
+            {"refId": "A", "expr": "sum by (provider, operation, reason) (rate(duocthu_provider_failure_total[5m]))", "legendFormat": "{{`{{provider}}`}} · {{`{{operation}}`}} · {{`{{reason}}`}}"},
+            {"refId": "B", "expr": "rate(duocthu_trace_write_failed_total[5m])", "legendFormat": "trace write"}
+          ]
+        }
+      ]
+    }
+{{- end }}

infra/helm/medical-chatbot/templates/observability-workloads.yaml

@@ -0,0 +1,292 @@
+{{- if .Values.observability.enabled }}
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: {{ include "medical-chatbot.fullname" . }}-prometheus
+  labels:
+    {{- include "medical-chatbot.labels" . | nindent 4 }}
+spec:
+  accessModes: [ReadWriteOnce]
+  resources:
+    requests: { storage: {{ .Values.observability.prometheus.storage }} }
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "medical-chatbot.fullname" . }}-prometheus
+  labels:
+    {{- include "medical-chatbot.labels" . | nindent 4 }}
+    app.kubernetes.io/component: prometheus
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/instance: {{ .Release.Name }}
+      app.kubernetes.io/component: prometheus
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/instance: {{ .Release.Name }}
+        app.kubernetes.io/component: prometheus
+    spec:
+      securityContext: { fsGroup: 65534 }
+      containers:
+        - name: prometheus
+          image: {{ .Values.observability.prometheus.image }}
+          args:
+            - --config.file=/etc/prometheus/prometheus.yml
+            - --storage.tsdb.path=/prometheus
+            - --storage.tsdb.retention.time={{ .Values.observability.prometheus.retention }}
+            - --enable-feature=exemplar-storage
+          ports:
+            - { name: http, containerPort: 9090 }
+          readinessProbe:
+            httpGet: { path: /-/ready, port: http }
+          livenessProbe:
+            httpGet: { path: /-/healthy, port: http }
+            initialDelaySeconds: 15
+          resources:
+            {{- toYaml .Values.observability.prometheus.resources | nindent 12 }}
+          volumeMounts:
+            - { name: config, mountPath: /etc/prometheus }
+            - { name: data, mountPath: /prometheus }
+      volumes:
+        - name: config
+          configMap: { name: {{ include "medical-chatbot.fullname" . }}-prometheus-config }
+        - name: data
+          persistentVolumeClaim: { claimName: {{ include "medical-chatbot.fullname" . }}-prometheus }
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "medical-chatbot.fullname" . }}-prometheus
+  labels:
+    {{- include "medical-chatbot.labels" . | nindent 4 }}
+spec:
+  selector:
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/component: prometheus
+  ports:
+    - { name: http, port: 9090, targetPort: http }
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: {{ include "medical-chatbot.fullname" . }}-tempo
+  labels:
+    {{- include "medical-chatbot.labels" . | nindent 4 }}
+spec:
+  accessModes: [ReadWriteOnce]
+  resources:
+    requests: { storage: {{ .Values.observability.tempo.storage }} }
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "medical-chatbot.fullname" . }}-tempo
+  labels:
+    {{- include "medical-chatbot.labels" . | nindent 4 }}
+    app.kubernetes.io/component: tempo
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/instance: {{ .Release.Name }}
+      app.kubernetes.io/component: tempo
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/instance: {{ .Release.Name }}
+        app.kubernetes.io/component: tempo
+    spec:
+      securityContext: { fsGroup: 10001 }
+      containers:
+        - name: tempo
+          image: {{ .Values.observability.tempo.image }}
+          args: ["-config.file=/etc/tempo/tempo.yml"]
+          ports:
+            - { name: http, containerPort: 3200 }
+            - { name: otlp-grpc, containerPort: 4317 }
+          readinessProbe:
+            httpGet: { path: /ready, port: http }
+            initialDelaySeconds: 5
+          livenessProbe:
+            httpGet: { path: /ready, port: http }
+            initialDelaySeconds: 20
+          resources:
+            {{- toYaml .Values.observability.tempo.resources | nindent 12 }}
+          volumeMounts:
+            - { name: config, mountPath: /etc/tempo }
+            - { name: data, mountPath: /var/tempo }
+      volumes:
+        - name: config
+          configMap: { name: {{ include "medical-chatbot.fullname" . }}-tempo-config }
+        - name: data
+          persistentVolumeClaim: { claimName: {{ include "medical-chatbot.fullname" . }}-tempo }
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "medical-chatbot.fullname" . }}-tempo
+  labels:
+    {{- include "medical-chatbot.labels" . | nindent 4 }}
+spec:
+  selector:
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/component: tempo
+  ports:
+    - { name: http, port: 3200, targetPort: http }
+    - { name: otlp-grpc, port: 4317, targetPort: otlp-grpc }
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "medical-chatbot.fullname" . }}-otel-collector
+  labels:
+    {{- include "medical-chatbot.labels" . | nindent 4 }}
+    app.kubernetes.io/component: otel-collector
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/instance: {{ .Release.Name }}
+      app.kubernetes.io/component: otel-collector
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/instance: {{ .Release.Name }}
+        app.kubernetes.io/component: otel-collector
+    spec:
+      containers:
+        - name: otel-collector
+          image: {{ .Values.observability.collector.image }}
+          args: ["--config=/etc/otelcol/collector.yml"]
+          ports:
+            - { name: otlp-grpc, containerPort: 4317 }
+            - { name: otlp-http, containerPort: 4318 }
+            - { name: health, containerPort: 13133 }
+          readinessProbe:
+            httpGet: { path: /, port: health }
+          livenessProbe:
+            httpGet: { path: /, port: health }
+            initialDelaySeconds: 10
+          resources:
+            {{- toYaml .Values.observability.collector.resources | nindent 12 }}
+          volumeMounts:
+            - { name: config, mountPath: /etc/otelcol }
+      volumes:
+        - name: config
+          configMap: { name: {{ include "medical-chatbot.fullname" . }}-otel-config }
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "medical-chatbot.fullname" . }}-otel-collector
+  labels:
+    {{- include "medical-chatbot.labels" . | nindent 4 }}
+spec:
+  selector:
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/component: otel-collector
+  ports:
+    - { name: otlp-grpc, port: 4317, targetPort: otlp-grpc }
+    - { name: otlp-http, port: 4318, targetPort: otlp-http }
+    - { name: health, port: 13133, targetPort: health }
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: {{ include "medical-chatbot.fullname" . }}-grafana
+  labels:
+    {{- include "medical-chatbot.labels" . | nindent 4 }}
+spec:
+  accessModes: [ReadWriteOnce]
+  resources:
+    requests: { storage: {{ .Values.observability.grafana.storage }} }
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "medical-chatbot.fullname" . }}-grafana
+  labels:
+    {{- include "medical-chatbot.labels" . | nindent 4 }}
+    app.kubernetes.io/component: grafana
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/instance: {{ .Release.Name }}
+      app.kubernetes.io/component: grafana
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/instance: {{ .Release.Name }}
+        app.kubernetes.io/component: grafana
+    spec:
+      securityContext: { fsGroup: 472 }
+      containers:
+        - name: grafana
+          image: {{ .Values.observability.grafana.image }}
+          ports:
+            - { name: http, containerPort: 3000 }
+          env:
+            - { name: GF_SECURITY_ADMIN_USER, value: admin }
+            - name: GF_SECURITY_ADMIN_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: {{ include "medical-chatbot.secretName" . }}
+                  key: grafana-admin-password
+            - name: GF_AUTH_ANONYMOUS_ENABLED
+              value: {{ .Values.observability.grafana.anonymousAdmin | quote }}
+            - name: GF_AUTH_ANONYMOUS_ORG_ROLE
+              value: Admin
+            - name: GF_AUTH_DISABLE_LOGIN_FORM
+              value: {{ .Values.observability.grafana.anonymousAdmin | quote }}
+          readinessProbe:
+            httpGet: { path: /api/health, port: http }
+            initialDelaySeconds: 10
+          livenessProbe:
+            httpGet: { path: /api/health, port: http }
+            initialDelaySeconds: 30
+          resources:
+            {{- toYaml .Values.observability.grafana.resources | nindent 12 }}
+          volumeMounts:
+            - { name: datasource, mountPath: /etc/grafana/provisioning/datasources }
+            - { name: dashboard-provider, mountPath: /etc/grafana/provisioning/dashboards }
+            - { name: dashboards, mountPath: /var/lib/grafana/dashboards }
+            - { name: data, mountPath: /var/lib/grafana }
+      volumes:
+        - name: datasource
+          configMap:
+            name: {{ include "medical-chatbot.fullname" . }}-grafana-provisioning
+            items: [{ key: datasources.yml, path: datasources.yml }]
+        - name: dashboard-provider
+          configMap:
+            name: {{ include "medical-chatbot.fullname" . }}-grafana-provisioning
+            items: [{ key: dashboards.yml, path: dashboards.yml }]
+        - name: dashboards
+          configMap:
+            name: {{ include "medical-chatbot.fullname" . }}-grafana-provisioning
+            items: [{ key: dashboard.json, path: dashboard.json }]
+        - name: data
+          persistentVolumeClaim: { claimName: {{ include "medical-chatbot.fullname" . }}-grafana }
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "medical-chatbot.fullname" . }}-grafana
+  labels:
+    {{- include "medical-chatbot.labels" . | nindent 4 }}
+spec:
+  type: {{ .Values.observability.grafana.service.type }}
+  selector:
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/component: grafana
+  ports:
+    - name: http
+      port: {{ .Values.observability.grafana.service.port }}
+      targetPort: http
+      {{- if and (eq .Values.observability.grafana.service.type "NodePort") .Values.observability.grafana.service.nodePort }}
+      nodePort: {{ .Values.observability.grafana.service.nodePort }}
+      {{- end }}
+{{- end }}

infra/helm/medical-chatbot/templates/secret.yaml

@@ -0,0 +1,13 @@
+{{- if .Values.secret.create }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ include "medical-chatbot.secretName" . }}
+  labels:
+    {{- include "medical-chatbot.labels" . | nindent 4 }}
+type: Opaque
+stringData:
+  postgres-password: {{ .Values.secret.postgresPassword | quote }}
+  postgres-dsn: {{ printf "postgresql://duoc_thu:%s@%s-postgres:5432/duoc_thu" .Values.secret.postgresPassword (include "medical-chatbot.fullname" .) | quote }}
+  grafana-admin-password: {{ .Values.secret.grafanaAdminPassword | quote }}
+{{- end }}

infra/helm/medical-chatbot/templates/serviceaccount.yaml

@@ -0,0 +1,10 @@
+{{- if .Values.serviceAccount.create }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "medical-chatbot.serviceAccountName" . }}
+  labels:
+    {{- include "medical-chatbot.labels" . | nindent 4 }}
+  annotations:
+    {{- toYaml .Values.serviceAccount.annotations | nindent 4 }}
+{{- end }}

infra/helm/medical-chatbot/templates/servicemonitor.yaml

@@ -0,0 +1,18 @@
+{{- if .Values.serviceMonitor.enabled }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ include "medical-chatbot.fullname" . }}-ai-service
+  labels:
+    {{- include "medical-chatbot.labels" . | nindent 4 }}
+    {{- toYaml .Values.serviceMonitor.additionalLabels | nindent 4 }}
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/instance: {{ .Release.Name }}
+      app.kubernetes.io/component: ai-service
+  endpoints:
+    - port: http
+      path: /metrics
+      interval: {{ .Values.serviceMonitor.interval }}
+{{- end }}

infra/helm/medical-chatbot/templates/web.yaml

@@ -0,0 +1,60 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "medical-chatbot.fullname" . }}-web
+  labels:
+    {{- include "medical-chatbot.labels" . | nindent 4 }}
+    app.kubernetes.io/component: web
+spec:
+  replicas: {{ .Values.web.replicaCount }}
+  selector:
+    matchLabels:
+      app.kubernetes.io/instance: {{ .Release.Name }}
+      app.kubernetes.io/component: web
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/instance: {{ .Release.Name }}
+        app.kubernetes.io/component: web
+    spec:
+      imagePullSecrets:
+        {{- toYaml .Values.global.imagePullSecrets | nindent 8 }}
+      containers:
+        - name: web
+          image: "{{ .Values.web.image.repository }}:{{ .Values.web.image.tag }}"
+          imagePullPolicy: {{ .Values.web.image.pullPolicy }}
+          env:
+            - name: AI_SERVICE_URL
+              value: {{ printf "http://%s-ai-service:%v" (include "medical-chatbot.fullname" .) .Values.aiService.service.port | quote }}
+          ports:
+            - { name: http, containerPort: 3000 }
+          readinessProbe:
+            httpGet: { path: /, port: http }
+            initialDelaySeconds: 5
+            periodSeconds: 10
+          livenessProbe:
+            httpGet: { path: /, port: http }
+            initialDelaySeconds: 15
+            periodSeconds: 20
+          resources:
+            {{- toYaml .Values.web.resources | nindent 12 }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "medical-chatbot.fullname" . }}-web
+  labels:
+    {{- include "medical-chatbot.labels" . | nindent 4 }}
+    app.kubernetes.io/component: web
+spec:
+  type: {{ .Values.web.service.type }}
+  selector:
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/component: web
+  ports:
+    - name: http
+      port: {{ .Values.web.service.port }}
+      targetPort: http
+      {{- if and (eq .Values.web.service.type "NodePort") .Values.web.service.nodePort }}
+      nodePort: {{ .Values.web.service.nodePort }}
+      {{- end }}