duocthu/.github/workflows/deploy.yml

name: Deploy to production

on:
  push:
    branches: [master]
    paths:
      # Everything the two Dockerfiles actually COPY into their images.
      - apps/ai-service/**
      - apps/web/**
      - packages/**
      - ingestion/data/verified/drug_entities.json
      # docker-compose.prod.yml, docker-compose.observability.yml,
      # Caddyfile, and the grafana/otel/prometheus/tempo configs it mounts.
      - infra/docker/**
      # Changes to the deploy logic itself should still be exercised.
      - .github/workflows/deploy.yml
  workflow_dispatch:

jobs:
  deploy:
    runs-on: ubuntu-latest
    steps:
      - name: Deploy over SSH
        uses: appleboy/ssh-action@v1.0.3
        env:
          GRAFANA_ADMIN_PASSWORD: ${{ secrets.GRAFANA_ADMIN_PASSWORD }}
        with:
          host: ${{ secrets.EC2_HOST }}
          username: ubuntu
          key: ${{ secrets.EC2_SSH_KEY }}
          envs: GRAFANA_ADMIN_PASSWORD
          script: |
            set -e
            test -n "${GRAFANA_ADMIN_PASSWORD:-}"
            export GRAFANA_ADMIN_PASSWORD
            cd ~/app
            git fetch origin master
            git reset --hard origin/master
            cd infra/docker
            sudo -E docker compose \
              -f docker-compose.prod.yml \
              -f docker-compose.observability.yml \
              up -d --build \
              ai-service web prometheus tempo otel-collector grafana caddy
            sudo docker exec docker-caddy-1 caddy validate --config /etc/caddy/Caddyfile --adapter caddyfile
            sudo docker exec docker-caddy-1 caddy reload --config /etc/caddy/Caddyfile --adapter caddyfile
            sudo docker exec docker-ai-service-1 python -m migrate
            sleep 10
            sudo docker run --rm --network docker_default curlimages/curl -sf http://ai-service:8000/health
            sudo docker run --rm --network docker_default curlimages/curl -sf http://ai-service:8000/ready
            sudo docker run --rm --network docker_default curlimages/curl -sf -o /dev/null http://web:3000
            if ! condition_response=$(sudo docker run --rm --network docker_default curlimages/curl -sf \
              -X POST http://ai-service:8000/v1/rag/query \
              -H 'Content-Type: application/json' \
              --data '{"query":"Đợt gout cấp có thuốc nào được Dược thư ghi chỉ định?","subject_scope":"human","intent":"fact_lookup","conversation_id":"deploy-condition-smoke"}'); then
              sudo docker logs --tail 200 docker-ai-service-1
              exit 1
            fi
            if ! printf '%s' "${condition_response}" | grep -q '"decision":"answerable"'; then
              printf '%s\n' "${condition_response}"
              sudo docker logs --tail 200 docker-ai-service-1
              exit 1
            fi
            printf '%s' "${condition_response}" | grep -q '"section_key":"chi_dinh"'
            sudo docker run --rm --network docker_default curlimages/curl -sf http://prometheus:9090/-/ready
            for attempt in $(seq 1 12); do
              if sudo docker run --rm --network docker_default curlimages/curl -sf http://tempo:3200/ready; then
                break
              fi
              if [ "${attempt}" -eq 12 ]; then
                sudo docker logs --tail 100 docker-tempo-1
                exit 1
              fi
              sleep 5
            done
            sudo docker run --rm --network docker_default curlimages/curl -sf http://grafana:3000/api/health
            sudo docker run --rm --network docker_default curlimages/curl -sf \
              -u "admin:${GRAFANA_ADMIN_PASSWORD}" \
              http://grafana:3000/api/datasources/uid/prometheus > /dev/null
            sudo docker run --rm --network docker_default curlimages/curl -sf \
              -u "admin:${GRAFANA_ADMIN_PASSWORD}" \
              http://grafana:3000/api/datasources/uid/tempo > /dev/null
            sudo docker run --rm --network docker_default curlimages/curl -sf \
              -u "admin:${GRAFANA_ADMIN_PASSWORD}" \
              http://grafana:3000/api/dashboards/uid/duocthu-observability \
              | grep -q '"uid":"duocthu-observability"'
            sudo docker run --rm --network docker_default curlimages/curl -sf \
              -o /dev/null https://realvuxbaro.me/grafana/login

            correlation_id="observability-deploy-$(date +%s)"
            response_headers=$(sudo docker run --rm --network docker_default curlimages/curl -sf \
              -D - -o /dev/null \
              -X POST http://ai-service:8000/v1/rag/query \
              -H 'Content-Type: application/json' \
              -H "X-Correlation-ID: ${correlation_id}" \
              --data '{"query":"Paracetamol là thuốc gì?","subject_scope":"unknown","intent":"unknown"}')
            trace_id=$(printf '%s\n' "${response_headers}" | tr -d '\r' | awk -F ': ' 'tolower($1) == "x-trace-id" { print $2 }' | tail -n 1)
            printf '%s' "${trace_id}" | grep -Eq '^[0-9a-f]{32}$'

            sleep 20
            sudo docker run --rm --network docker_default curlimages/curl -sfG \
              --data-urlencode 'query=duocthu_requests_total' \
              http://prometheus:9090/api/v1/query | grep -q '"__name__":"duocthu_requests_total"'
            for attempt in $(seq 1 12); do
              if sudo docker run --rm --network docker_default curlimages/curl -sf \
                -o /dev/null "http://tempo:3200/api/traces/${trace_id}"; then
                break
              fi
              if [ "${attempt}" -eq 12 ]; then
                exit 1
              fi
              sleep 5
            done