79 lines
3.4 KiB
YAML
79 lines
3.4 KiB
YAML
name: Deploy to production
|
|
|
|
on:
|
|
push:
|
|
branches: [master]
|
|
workflow_dispatch:
|
|
|
|
jobs:
|
|
deploy:
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- name: Deploy over SSH
|
|
uses: appleboy/ssh-action@v1.0.3
|
|
env:
|
|
GRAFANA_ADMIN_PASSWORD: ${{ secrets.GRAFANA_ADMIN_PASSWORD }}
|
|
with:
|
|
host: ${{ secrets.EC2_HOST }}
|
|
username: ubuntu
|
|
key: ${{ secrets.EC2_SSH_KEY }}
|
|
envs: GRAFANA_ADMIN_PASSWORD
|
|
script: |
|
|
set -e
|
|
test -n "${GRAFANA_ADMIN_PASSWORD:-}"
|
|
export GRAFANA_ADMIN_PASSWORD
|
|
cd ~/app
|
|
git fetch origin master
|
|
git reset --hard origin/master
|
|
cd infra/docker
|
|
sudo -E docker compose \
|
|
-f docker-compose.prod.yml \
|
|
-f docker-compose.observability.yml \
|
|
up -d --build \
|
|
ai-service web prometheus tempo otel-collector grafana
|
|
sudo docker exec docker-ai-service-1 python -m migrate
|
|
sleep 10
|
|
sudo docker run --rm --network docker_default curlimages/curl -sf http://ai-service:8000/health
|
|
sudo docker run --rm --network docker_default curlimages/curl -sf http://ai-service:8000/ready
|
|
sudo docker run --rm --network docker_default curlimages/curl -sf -o /dev/null http://web:3000
|
|
sudo docker run --rm --network docker_default curlimages/curl -sf http://prometheus:9090/-/ready
|
|
for attempt in $(seq 1 12); do
|
|
if sudo docker run --rm --network docker_default curlimages/curl -sf http://tempo:3200/ready; then
|
|
break
|
|
fi
|
|
if [ "${attempt}" -eq 12 ]; then
|
|
sudo docker logs --tail 100 docker-tempo-1
|
|
exit 1
|
|
fi
|
|
sleep 5
|
|
done
|
|
sudo docker run --rm --network docker_default curlimages/curl -sf http://grafana:3000/api/health
|
|
sudo docker run --rm --network docker_default curlimages/curl -sf \
|
|
-u "admin:${GRAFANA_ADMIN_PASSWORD}" \
|
|
http://grafana:3000/api/datasources/uid/prometheus > /dev/null
|
|
|
|
correlation_id="observability-deploy-$(date +%s)"
|
|
response_headers=$(sudo docker run --rm --network docker_default curlimages/curl -sf \
|
|
-D - -o /dev/null \
|
|
-X POST http://ai-service:8000/v1/rag/query \
|
|
-H 'Content-Type: application/json' \
|
|
-H "X-Correlation-ID: ${correlation_id}" \
|
|
--data '{"query":"Paracetamol là thuốc gì?","subject_scope":"unknown","intent":"unknown"}')
|
|
trace_id=$(printf '%s\n' "${response_headers}" | tr -d '\r' | awk -F ': ' 'tolower($1) == "x-trace-id" { print $2 }' | tail -n 1)
|
|
printf '%s' "${trace_id}" | grep -Eq '^[0-9a-f]{32}$'
|
|
|
|
sleep 20
|
|
sudo docker run --rm --network docker_default curlimages/curl -sfG \
|
|
--data-urlencode 'query=duocthu_requests_total' \
|
|
http://prometheus:9090/api/v1/query | grep -q '"__name__":"duocthu_requests_total"'
|
|
for attempt in $(seq 1 12); do
|
|
if sudo docker run --rm --network docker_default curlimages/curl -sf \
|
|
-o /dev/null "http://tempo:3200/api/traces/${trace_id}"; then
|
|
break
|
|
fi
|
|
if [ "${attempt}" -eq 12 ]; then
|
|
exit 1
|
|
fi
|
|
sleep 5
|
|
done
|