Verify exact production traces and record rollout

.github/workflows/deploy.yml

@@ -3,6 +3,7 @@ name: Deploy to production
 on:
   push:
     branches: [master]
+  workflow_dispatch:
 
 jobs:
   deploy:
@@ -10,18 +11,68 @@ jobs:
     steps:
       - name: Deploy over SSH
         uses: appleboy/ssh-action@v1.0.3
+        env:
+          GRAFANA_ADMIN_PASSWORD: ${{ secrets.GRAFANA_ADMIN_PASSWORD }}
         with:
           host: ${{ secrets.EC2_HOST }}
           username: ubuntu
           key: ${{ secrets.EC2_SSH_KEY }}
+          envs: GRAFANA_ADMIN_PASSWORD
           script: |
             set -e
+            test -n "${GRAFANA_ADMIN_PASSWORD:-}"
+            export GRAFANA_ADMIN_PASSWORD
             cd ~/app
             git fetch origin master
             git reset --hard origin/master
             cd infra/docker
-            sudo docker compose -f docker-compose.prod.yml up -d --build ai-service web
+            sudo -E docker compose \
+              -f docker-compose.prod.yml \
+              -f docker-compose.observability.yml \
+              up -d --build \
+              ai-service web prometheus tempo otel-collector grafana
             sudo docker exec docker-ai-service-1 python -m migrate
-            sleep 5
+            sleep 10
             sudo docker run --rm --network docker_default curlimages/curl -sf http://ai-service:8000/health
+            sudo docker run --rm --network docker_default curlimages/curl -sf http://ai-service:8000/ready
             sudo docker run --rm --network docker_default curlimages/curl -sf -o /dev/null http://web:3000
+            sudo docker run --rm --network docker_default curlimages/curl -sf http://prometheus:9090/-/ready
+            for attempt in $(seq 1 12); do
+              if sudo docker run --rm --network docker_default curlimages/curl -sf http://tempo:3200/ready; then
+                break
+              fi
+              if [ "${attempt}" -eq 12 ]; then
+                sudo docker logs --tail 100 docker-tempo-1
+                exit 1
+              fi
+              sleep 5
+            done
+            sudo docker run --rm --network docker_default curlimages/curl -sf http://grafana:3000/api/health
+            sudo docker run --rm --network docker_default curlimages/curl -sf \
+              -u "admin:${GRAFANA_ADMIN_PASSWORD}" \
+              http://grafana:3000/api/datasources/uid/prometheus > /dev/null
+
+            correlation_id="observability-deploy-$(date +%s)"
+            response_headers=$(sudo docker run --rm --network docker_default curlimages/curl -sf \
+              -D - -o /dev/null \
+              -X POST http://ai-service:8000/v1/rag/query \
+              -H 'Content-Type: application/json' \
+              -H "X-Correlation-ID: ${correlation_id}" \
+              --data '{"query":"Paracetamol là thuốc gì?","subject_scope":"unknown","intent":"unknown"}')
+            trace_id=$(printf '%s\n' "${response_headers}" | tr -d '\r' | awk -F ': ' 'tolower($1) == "x-trace-id" { print $2 }' | tail -n 1)
+            printf '%s' "${trace_id}" | grep -Eq '^[0-9a-f]{32}$'
+
+            sleep 20
+            sudo docker run --rm --network docker_default curlimages/curl -sfG \
+              --data-urlencode 'query=duocthu_requests_total' \
+              http://prometheus:9090/api/v1/query | grep -q '"__name__":"duocthu_requests_total"'
+            for attempt in $(seq 1 12); do
+              if sudo docker run --rm --network docker_default curlimages/curl -sf \
+                -o /dev/null "http://tempo:3200/api/traces/${trace_id}"; then
+                break
+              fi
+              if [ "${attempt}" -eq 12 ]; then
+                exit 1
+              fi
+              sleep 5
+            done