Stop mislabeling the Compose box as production in read-only audits
This commit is contained in:
@@ -1,4 +1,10 @@
|
||||
name: Audit production runtime (read-only)
|
||||
name: Audit Compose rollback box (read-only)
|
||||
|
||||
# `realvuxbaro.me` has run on k3s since the 2026-08-17 cutover; this workflow
|
||||
# still SSHes into secrets.EC2_HOST, which is the retired Compose EC2 kept
|
||||
# only as a manual DNS fallback. Useful for confirming that box is still
|
||||
# healthy and on a known commit before relying on it as a fallback — it does
|
||||
# NOT reflect what real production is currently running.
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
@@ -14,7 +20,7 @@ jobs:
|
||||
audit:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Inspect production over SSH
|
||||
- name: Inspect the Compose rollback box over SSH
|
||||
uses: appleboy/ssh-action@v1.0.3
|
||||
with:
|
||||
host: ${{ secrets.EC2_HOST }}
|
||||
|
||||
@@ -1,13 +1,15 @@
|
||||
name: Audit production Qdrant corpus (read-only)
|
||||
name: Audit Compose rollback Qdrant corpus (read-only)
|
||||
|
||||
# Answers "is the practice corpus the same corpus production serves?" with
|
||||
# `realvuxbaro.me` has run on k3s since the 2026-08-17 cutover; this workflow
|
||||
# still SSHes into secrets.EC2_HOST, which is the retired Compose EC2 kept
|
||||
# only as a manual DNS fallback. It fingerprints that box's Qdrant with
|
||||
# content hashes rather than a point count, which two different corpora can
|
||||
# share. The identical script runs against the k3s cluster over SSH, so the
|
||||
# two fingerprints are directly comparable.
|
||||
# share — the same script can be run against the k3s side (over SSH, or via
|
||||
# `docker exec` on its ai-service pod) to check the fallback still matches
|
||||
# real production before ever relying on it.
|
||||
#
|
||||
# Read-only: it scrolls points and reads collection info. It changes nothing on
|
||||
# production, and its path is not in deploy.yml's filters, so merging it cannot
|
||||
# restart the Compose stack.
|
||||
# Read-only: it scrolls points and reads collection info. It changes nothing,
|
||||
# on this box or any other, and cannot trigger a rebuild of it.
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
@@ -30,7 +32,7 @@ jobs:
|
||||
- name: Encode fingerprint script
|
||||
run: echo "SCRIPT_B64=$(base64 -w0 scripts/qdrant_fingerprint.py)" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Fingerprint production corpus over SSH
|
||||
- name: Fingerprint the Compose rollback corpus over SSH
|
||||
uses: appleboy/ssh-action@v1.0.3
|
||||
env:
|
||||
SCRIPT_B64: ${{ env.SCRIPT_B64 }}
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
name: Build and sync k3s practice images
|
||||
name: Build and sync k3s images
|
||||
|
||||
# Practice-cluster only (readytochat.realvuxbaro.me, ArgoCD-managed on the
|
||||
# self-hosted k3s box). Does not touch deploy.yml or the production
|
||||
# EC2/Compose stack — production never pulls a GHCR image and isn't
|
||||
# ArgoCD-managed at all, so this workflow has no path to affect it.
|
||||
# This IS the production deploy path. `medical-chatbot-app` (ArgoCD, k3s) is
|
||||
# the same release behind both realvuxbaro.me and readytochat.realvuxbaro.me
|
||||
# since the 2026-08-17 cutover — there is no longer a separate "practice"
|
||||
# Application this workflow avoids touching. The Compose EC2 is unaffected
|
||||
# only because it has no CI/CD path left at all (deploy.yml/rollback.yml were
|
||||
# removed); it is a manual DNS fallback, not a deploy target.
|
||||
#
|
||||
# ArgoCD's Applications already autosync (syncPolicy.automated) — the gap
|
||||
# this closes is that the image tag they deploy was a static string
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
name: CI
|
||||
|
||||
# Runs on every push and every pull request. `deploy.yml` triggers
|
||||
# independently on push to master; until it is made to depend on this job, a
|
||||
# red CI does NOT block a deploy — see docs/22-ci-cd.md.
|
||||
# Runs on every push and every pull request. `build-practice-images.yml`
|
||||
# (the k3s/ArgoCD production deploy path) triggers independently on push to
|
||||
# master; until it is made to depend on this job, a red CI does NOT block a
|
||||
# deploy — see docs/operations.md.
|
||||
on:
|
||||
push:
|
||||
pull_request:
|
||||
|
||||
@@ -1,113 +0,0 @@
|
||||
name: Deploy to production
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [master]
|
||||
paths:
|
||||
# Everything the two Dockerfiles actually COPY into their images.
|
||||
- apps/ai-service/**
|
||||
- apps/web/**
|
||||
- packages/**
|
||||
- ingestion/data/verified/drug_entities.json
|
||||
# docker-compose.prod.yml, docker-compose.observability.yml,
|
||||
# Caddyfile, and the grafana/otel/prometheus/tempo configs it mounts.
|
||||
- infra/docker/**
|
||||
# Changes to the deploy logic itself should still be exercised.
|
||||
- .github/workflows/deploy.yml
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
deploy:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Deploy over SSH
|
||||
uses: appleboy/ssh-action@v1.0.3
|
||||
env:
|
||||
GRAFANA_ADMIN_PASSWORD: ${{ secrets.GRAFANA_ADMIN_PASSWORD }}
|
||||
with:
|
||||
host: ${{ secrets.EC2_HOST }}
|
||||
username: ubuntu
|
||||
key: ${{ secrets.EC2_SSH_KEY }}
|
||||
envs: GRAFANA_ADMIN_PASSWORD
|
||||
script: |
|
||||
set -e
|
||||
test -n "${GRAFANA_ADMIN_PASSWORD:-}"
|
||||
export GRAFANA_ADMIN_PASSWORD
|
||||
cd ~/app
|
||||
git fetch origin master
|
||||
git reset --hard origin/master
|
||||
cd infra/docker
|
||||
sudo -E docker compose \
|
||||
-f docker-compose.prod.yml \
|
||||
-f docker-compose.observability.yml \
|
||||
up -d --build \
|
||||
ai-service web prometheus tempo otel-collector grafana caddy
|
||||
sudo docker exec docker-caddy-1 caddy validate --config /etc/caddy/Caddyfile --adapter caddyfile
|
||||
sudo docker exec docker-caddy-1 caddy reload --config /etc/caddy/Caddyfile --adapter caddyfile
|
||||
sudo docker exec docker-ai-service-1 python -m migrate
|
||||
sleep 10
|
||||
sudo docker run --rm --network docker_default curlimages/curl -sf http://ai-service:8000/health
|
||||
sudo docker run --rm --network docker_default curlimages/curl -sf http://ai-service:8000/ready
|
||||
sudo docker run --rm --network docker_default curlimages/curl -sf -o /dev/null http://web:3000
|
||||
if ! condition_response=$(sudo docker run --rm --network docker_default curlimages/curl -sf \
|
||||
-X POST http://ai-service:8000/v1/rag/query \
|
||||
-H 'Content-Type: application/json' \
|
||||
--data '{"query":"Đợt gout cấp có thuốc nào được Dược thư ghi chỉ định?","subject_scope":"human","intent":"fact_lookup","conversation_id":"deploy-condition-smoke"}'); then
|
||||
sudo docker logs --tail 200 docker-ai-service-1
|
||||
exit 1
|
||||
fi
|
||||
if ! printf '%s' "${condition_response}" | grep -q '"decision":"answerable"'; then
|
||||
printf '%s\n' "${condition_response}"
|
||||
sudo docker logs --tail 200 docker-ai-service-1
|
||||
exit 1
|
||||
fi
|
||||
printf '%s' "${condition_response}" | grep -q '"section_key":"chi_dinh"'
|
||||
sudo docker run --rm --network docker_default curlimages/curl -sf http://prometheus:9090/-/ready
|
||||
for attempt in $(seq 1 12); do
|
||||
if sudo docker run --rm --network docker_default curlimages/curl -sf http://tempo:3200/ready; then
|
||||
break
|
||||
fi
|
||||
if [ "${attempt}" -eq 12 ]; then
|
||||
sudo docker logs --tail 100 docker-tempo-1
|
||||
exit 1
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
sudo docker run --rm --network docker_default curlimages/curl -sf http://grafana:3000/api/health
|
||||
sudo docker run --rm --network docker_default curlimages/curl -sf \
|
||||
-u "admin:${GRAFANA_ADMIN_PASSWORD}" \
|
||||
http://grafana:3000/api/datasources/uid/prometheus > /dev/null
|
||||
sudo docker run --rm --network docker_default curlimages/curl -sf \
|
||||
-u "admin:${GRAFANA_ADMIN_PASSWORD}" \
|
||||
http://grafana:3000/api/datasources/uid/tempo > /dev/null
|
||||
sudo docker run --rm --network docker_default curlimages/curl -sf \
|
||||
-u "admin:${GRAFANA_ADMIN_PASSWORD}" \
|
||||
http://grafana:3000/api/dashboards/uid/duocthu-observability \
|
||||
| grep -q '"uid":"duocthu-observability"'
|
||||
sudo docker run --rm --network docker_default curlimages/curl -sf \
|
||||
-o /dev/null https://realvuxbaro.me/grafana/login
|
||||
|
||||
correlation_id="observability-deploy-$(date +%s)"
|
||||
response_headers=$(sudo docker run --rm --network docker_default curlimages/curl -sf \
|
||||
-D - -o /dev/null \
|
||||
-X POST http://ai-service:8000/v1/rag/query \
|
||||
-H 'Content-Type: application/json' \
|
||||
-H "X-Correlation-ID: ${correlation_id}" \
|
||||
--data '{"query":"Paracetamol là thuốc gì?","subject_scope":"unknown","intent":"unknown"}')
|
||||
trace_id=$(printf '%s\n' "${response_headers}" | tr -d '\r' | awk -F ': ' 'tolower($1) == "x-trace-id" { print $2 }' | tail -n 1)
|
||||
printf '%s' "${trace_id}" | grep -Eq '^[0-9a-f]{32}$'
|
||||
|
||||
sleep 20
|
||||
sudo docker run --rm --network docker_default curlimages/curl -sfG \
|
||||
--data-urlencode 'query=duocthu_requests_total' \
|
||||
http://prometheus:9090/api/v1/query | grep -q '"__name__":"duocthu_requests_total"'
|
||||
for attempt in $(seq 1 12); do
|
||||
if sudo docker run --rm --network docker_default curlimages/curl -sf \
|
||||
-o /dev/null "http://tempo:3200/api/traces/${trace_id}"; then
|
||||
break
|
||||
fi
|
||||
if [ "${attempt}" -eq 12 ]; then
|
||||
exit 1
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
@@ -23,45 +23,37 @@ jobs:
|
||||
version: v3.17.3
|
||||
- name: Lint chart
|
||||
run: helm lint infra/helm/medical-chatbot
|
||||
- name: Render default and production manifests
|
||||
- name: Render defaults and check the immutable-tag guard
|
||||
run: |
|
||||
helm template default infra/helm/medical-chatbot > /tmp/default.yaml
|
||||
|
||||
# values-prod.yaml leaves the image tags empty on purpose: production
|
||||
# must run an immutable commit SHA, supplied per deploy. Rendering it
|
||||
# without one has to fail rather than fall back to a development tag,
|
||||
# so assert that failure here — otherwise the guard could rot into a
|
||||
# silent default and nobody would notice until a cutover.
|
||||
if helm template production infra/helm/medical-chatbot \
|
||||
--values infra/helm/medical-chatbot/values-prod.yaml \
|
||||
> /tmp/untagged.yaml 2>/tmp/untagged.err; then
|
||||
echo "::error::production render succeeded with no image tag; the immutable-tag guard is gone"
|
||||
# The live releases carry no image tag in Git -- it is supplied per
|
||||
# deploy as a commit SHA through the ArgoCD Application. Rendering
|
||||
# with an empty tag must FAIL rather than fall back to the chart's
|
||||
# `local` development tag, so assert the failure directly; otherwise
|
||||
# the guard could rot into a silent default unnoticed.
|
||||
if helm template production infra/helm/medical-chatbot --values infra/helm/medical-chatbot/values-production.yaml --set aiService.image.tag="" --set web.image.tag="" > /tmp/untagged.yaml 2>/tmp/untagged.err; then
|
||||
echo "::error::render succeeded with no image tag; the immutable-tag guard is gone"
|
||||
exit 1
|
||||
fi
|
||||
grep -q 'image.tag must be set to an immutable tag' /tmp/untagged.err
|
||||
|
||||
helm template production infra/helm/medical-chatbot \
|
||||
--values infra/helm/medical-chatbot/values-prod.yaml \
|
||||
--set aiService.image.tag="$GITHUB_SHA" \
|
||||
--set web.image.tag="$GITHUB_SHA" \
|
||||
> /tmp/production.yaml
|
||||
grep -q 'ANSWER_MODEL_ID: "qwen.qwen3-next-80b-a3b"' /tmp/production.yaml
|
||||
grep -q 'RERANK_ENABLED: "true"' /tmp/production.yaml
|
||||
grep -q 'checksum/runtime-config:' /tmp/production.yaml
|
||||
grep -q "image: \"ghcr.io/baovu2k4/vsf-duocthu-ai-service:$GITHUB_SHA\"" /tmp/production.yaml
|
||||
grep -q "image: \"ghcr.io/baovu2k4/vsf-duocthu-web:$GITHUB_SHA\"" /tmp/production.yaml
|
||||
# ...and with a tag it must resolve the GHCR package, not the local
|
||||
# development image name.
|
||||
helm template production infra/helm/medical-chatbot --values infra/helm/medical-chatbot/values-production.yaml --set aiService.image.repository=ghcr.io/baovu2k4/vsf-duocthu-ai-service --set web.image.repository=ghcr.io/baovu2k4/vsf-duocthu-web --set aiService.image.tag="$GITHUB_SHA" --set web.image.tag="$GITHUB_SHA" > /tmp/tagged.yaml
|
||||
grep -q "image: \"ghcr.io/baovu2k4/vsf-duocthu-ai-service:$GITHUB_SHA\"" /tmp/tagged.yaml
|
||||
grep -q "image: \"ghcr.io/baovu2k4/vsf-duocthu-web:$GITHUB_SHA\"" /tmp/tagged.yaml
|
||||
|
||||
# The practice cluster is only evidence for the production migration
|
||||
# while it renders the same behavioural contract as production, so both
|
||||
# of its releases are asserted here rather than trusted by review.
|
||||
- name: Render practice manifests
|
||||
# These two releases are what realvuxbaro.me actually serves, so their
|
||||
# contract is asserted here rather than trusted by review.
|
||||
- name: Render the live production manifests
|
||||
run: |
|
||||
helm template medical-chatbot-app infra/helm/medical-chatbot \
|
||||
--values infra/helm/medical-chatbot/values-practice.yaml \
|
||||
> /tmp/practice-app.yaml
|
||||
--values infra/helm/medical-chatbot/values-production.yaml \
|
||||
> /tmp/prod-app.yaml
|
||||
helm template medical-chatbot-data infra/helm/medical-chatbot \
|
||||
--values infra/helm/medical-chatbot/values-practice-data.yaml \
|
||||
> /tmp/practice-data.yaml
|
||||
--values infra/helm/medical-chatbot/values-production-data.yaml \
|
||||
> /tmp/prod-data.yaml
|
||||
|
||||
# A bare `grep -q` fails the step with no indication of which
|
||||
# assertion broke, and `set -e` ignores a status inverted with `!`,
|
||||
@@ -83,35 +75,35 @@ jobs:
|
||||
}
|
||||
|
||||
# Behavioural parity with the audited production runtime contract.
|
||||
expect /tmp/practice-app.yaml 'ANSWER_MODEL_ID: "qwen.qwen3-next-80b-a3b"'
|
||||
expect /tmp/practice-app.yaml 'ANSWER_PROVIDER: "bedrock-converse"'
|
||||
expect /tmp/practice-app.yaml 'EMBEDDING_PROVIDER: "cohere-v4"'
|
||||
expect /tmp/practice-app.yaml 'EMBEDDING_DIMENSIONS: "1024"'
|
||||
expect /tmp/practice-app.yaml 'EVIDENCE_MINIMUM_SCORE: "0.12"'
|
||||
expect /tmp/practice-app.yaml 'RERANK_ENABLED: "true"'
|
||||
expect /tmp/practice-app.yaml 'AWS_REGION: "us-east-1"'
|
||||
expect /tmp/practice-app.yaml 'checksum/runtime-config:'
|
||||
expect /tmp/practice-app.yaml '- host: "readytochat.realvuxbaro.me"'
|
||||
expect /tmp/prod-app.yaml 'ANSWER_MODEL_ID: "qwen.qwen3-next-80b-a3b"'
|
||||
expect /tmp/prod-app.yaml 'ANSWER_PROVIDER: "bedrock-converse"'
|
||||
expect /tmp/prod-app.yaml 'EMBEDDING_PROVIDER: "cohere-v4"'
|
||||
expect /tmp/prod-app.yaml 'EMBEDDING_DIMENSIONS: "1024"'
|
||||
expect /tmp/prod-app.yaml 'EVIDENCE_MINIMUM_SCORE: "0.12"'
|
||||
expect /tmp/prod-app.yaml 'RERANK_ENABLED: "true"'
|
||||
expect /tmp/prod-app.yaml 'AWS_REGION: "us-east-1"'
|
||||
expect /tmp/prod-app.yaml 'checksum/runtime-config:'
|
||||
expect /tmp/prod-app.yaml '- host: "readytochat.realvuxbaro.me"'
|
||||
|
||||
# The production hostname now lives on this cluster, routed and with
|
||||
# its own certificate secret -- kept separate from the rehearsal
|
||||
# hostname's so one renewal failure cannot take both names offline.
|
||||
expect /tmp/practice-app.yaml '- host: "realvuxbaro.me"'
|
||||
expect /tmp/practice-app.yaml 'secretName: realvuxbaro-tls'
|
||||
expect /tmp/practice-app.yaml 'secretName: readytochat-tls'
|
||||
expect /tmp/prod-app.yaml '- host: "realvuxbaro.me"'
|
||||
expect /tmp/prod-app.yaml 'secretName: realvuxbaro-tls'
|
||||
expect /tmp/prod-app.yaml 'secretName: readytochat-tls'
|
||||
|
||||
# Grafana answers on that same public hostname. Anonymous access may
|
||||
# be open, but never as Admin, never with the login form disabled,
|
||||
# and its root URL must be the name users actually arrive on.
|
||||
expect /tmp/practice-app.yaml 'value: "https://realvuxbaro.me/grafana/"'
|
||||
refute /tmp/practice-app.yaml 'value: "Admin"'
|
||||
expect /tmp/prod-app.yaml 'value: "https://realvuxbaro.me/grafana/"'
|
||||
refute /tmp/prod-app.yaml 'value: "Admin"'
|
||||
|
||||
# grep is line-oriented, so read the value on the line after each
|
||||
# flag rather than trying to match the pair as one pattern.
|
||||
for check in "GF_AUTH_ANONYMOUS_ORG_ROLE:Viewer" "GF_AUTH_DISABLE_LOGIN_FORM:false"; do
|
||||
flag=${check%%:*}
|
||||
want=${check#*:}
|
||||
got=$(grep -A1 -- "$flag" /tmp/practice-app.yaml | grep -- 'value:' | tr -d ' "' | cut -d: -f2)
|
||||
got=$(grep -A1 -- "$flag" /tmp/prod-app.yaml | grep -- 'value:' | tr -d ' "' | cut -d: -f2)
|
||||
if [ "$got" != "$want" ]; then
|
||||
echo "::error::$flag rendered as '$got', expected '$want'"
|
||||
exit 1
|
||||
@@ -123,10 +115,10 @@ jobs:
|
||||
# prune can never delete the corpus or the query history. Only those
|
||||
# two use volumeClaimTemplates — the observability PVCs are the app
|
||||
# release's own and are expected here.
|
||||
refute /tmp/practice-app.yaml 'volumeClaimTemplates'
|
||||
expect /tmp/practice-app.yaml 'medical-chatbot-data-medical-chatbot-qdrant'
|
||||
refute /tmp/prod-app.yaml 'volumeClaimTemplates'
|
||||
expect /tmp/prod-app.yaml 'medical-chatbot-data-medical-chatbot-qdrant'
|
||||
|
||||
# ...and the data release must own nothing else.
|
||||
refute /tmp/practice-data.yaml 'medical-chatbot-data-medical-chatbot-ai-service'
|
||||
refute /tmp/practice-data.yaml 'kind: Ingress'
|
||||
expect /tmp/practice-data.yaml 'volumeClaimTemplates'
|
||||
refute /tmp/prod-data.yaml 'medical-chatbot-data-medical-chatbot-ai-service'
|
||||
refute /tmp/prod-data.yaml 'kind: Ingress'
|
||||
expect /tmp/prod-data.yaml 'volumeClaimTemplates'
|
||||
|
||||
@@ -1,66 +0,0 @@
|
||||
name: Migrate Qdrant snapshot to practice cluster
|
||||
|
||||
# One-off, manual (workflow_dispatch only) bridge: snapshots the production
|
||||
# Qdrant collection (a live, non-disruptive Qdrant operation — this is how
|
||||
# the original prod migration was done, just in reverse) and relays the
|
||||
# snapshot files to the isolated k3s practice EC2. Uses the SAME EC2_SSH_KEY
|
||||
# deploy.yml already has (never exposed to the operator) plus a new
|
||||
# PRACTICE_SSH_KEY scoped only to the practice box. Delete this workflow
|
||||
# file once the one-time migration is done — it is not part of the regular
|
||||
# deploy path.
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
migrate:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Set up SSH key
|
||||
run: |
|
||||
mkdir -p ~/.ssh
|
||||
printf '%s\n' "${{ secrets.EC2_SSH_KEY }}" > ~/.ssh/prod.pem
|
||||
chmod 600 ~/.ssh/prod.pem
|
||||
ssh-keyscan -H "${{ secrets.EC2_HOST }}" >> ~/.ssh/known_hosts 2>/dev/null
|
||||
|
||||
- name: Report production Qdrant version
|
||||
run: |
|
||||
ssh -i ~/.ssh/prod.pem "ubuntu@${{ secrets.EC2_HOST }}" '
|
||||
sudo docker run --rm --network docker_default curlimages/curl -sf http://qdrant:6333/
|
||||
'
|
||||
|
||||
- name: Snapshot Qdrant collections on production
|
||||
run: |
|
||||
ssh -i ~/.ssh/prod.pem "ubuntu@${{ secrets.EC2_HOST }}" '
|
||||
set -e
|
||||
snap() {
|
||||
collection="$1"; outfile="$2"
|
||||
name=$(sudo docker run --rm --network docker_default curlimages/curl -sf -X POST "http://qdrant:6333/collections/${collection}/snapshots" | grep -oP "\"name\":\"\K[^\"]+")
|
||||
test -n "$name"
|
||||
sudo docker run --rm --network docker_default curlimages/curl -sf "http://qdrant:6333/collections/${collection}/snapshots/${name}" > "$outfile"
|
||||
}
|
||||
snap duocthu_v1 /tmp/duocthu_v1.snapshot
|
||||
snap duocthu_v1__manifest /tmp/duocthu_v1__manifest.snapshot
|
||||
sudo chown ubuntu:ubuntu /tmp/duocthu_v1.snapshot /tmp/duocthu_v1__manifest.snapshot
|
||||
ls -la /tmp/*.snapshot
|
||||
'
|
||||
|
||||
- name: Pull snapshots to the runner
|
||||
run: |
|
||||
scp -i ~/.ssh/prod.pem "ubuntu@${{ secrets.EC2_HOST }}:/tmp/duocthu_v1.snapshot" ./duocthu_v1.snapshot
|
||||
scp -i ~/.ssh/prod.pem "ubuntu@${{ secrets.EC2_HOST }}:/tmp/duocthu_v1__manifest.snapshot" ./duocthu_v1__manifest.snapshot
|
||||
ls -la ./*.snapshot
|
||||
|
||||
- name: Upload snapshots as a workflow artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: qdrant-snapshots
|
||||
path: |
|
||||
duocthu_v1.snapshot
|
||||
duocthu_v1__manifest.snapshot
|
||||
retention-days: 1
|
||||
|
||||
- name: Clean up temp files on production
|
||||
if: always()
|
||||
run: |
|
||||
ssh -i ~/.ssh/prod.pem "ubuntu@${{ secrets.EC2_HOST }}" 'rm -f /tmp/duocthu_v1.snapshot /tmp/duocthu_v1__manifest.snapshot' || true
|
||||
@@ -1,58 +0,0 @@
|
||||
name: Rollback production
|
||||
|
||||
# Manual escape hatch for deploy.yml. deploy.yml has NO automatic rollback:
|
||||
# it runs `git reset --hard origin/master`, rebuilds and runs migrations
|
||||
# BEFORE its health checks, so a deploy that fails those checks leaves the
|
||||
# server on the bad commit with no automatic recovery. This workflow points
|
||||
# the same reset+rebuild+health-check sequence at an earlier commit instead.
|
||||
#
|
||||
# Migrations are forward-only (apps/ai-service/migrate.py, no down scripts)
|
||||
# but every migration so far uses IF NOT EXISTS / ADD COLUMN IF NOT EXISTS,
|
||||
# so re-running them against an older commit is a no-op, not an error. A
|
||||
# future non-idempotent migration would break this guarantee.
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
target_sha:
|
||||
description: "Commit SHA or tag to roll back to (e.g. the last known-good commit from a previous successful 'Deploy to production' run)"
|
||||
required: true
|
||||
|
||||
jobs:
|
||||
rollback:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Rollback over SSH
|
||||
uses: appleboy/ssh-action@v1.0.3
|
||||
env:
|
||||
GRAFANA_ADMIN_PASSWORD: ${{ secrets.GRAFANA_ADMIN_PASSWORD }}
|
||||
TARGET_SHA: ${{ inputs.target_sha }}
|
||||
with:
|
||||
host: ${{ secrets.EC2_HOST }}
|
||||
username: ubuntu
|
||||
key: ${{ secrets.EC2_SSH_KEY }}
|
||||
envs: GRAFANA_ADMIN_PASSWORD,TARGET_SHA
|
||||
script: |
|
||||
set -e
|
||||
test -n "${GRAFANA_ADMIN_PASSWORD:-}"
|
||||
export GRAFANA_ADMIN_PASSWORD
|
||||
cd ~/app
|
||||
git fetch origin
|
||||
git rev-parse --verify "${TARGET_SHA}^{commit}"
|
||||
git reset --hard "${TARGET_SHA}"
|
||||
echo "Rolled back to $(git rev-parse HEAD) — $(git log -1 --format=%s)"
|
||||
cd infra/docker
|
||||
sudo -E docker compose \
|
||||
-f docker-compose.prod.yml \
|
||||
-f docker-compose.observability.yml \
|
||||
up -d --build \
|
||||
ai-service web prometheus tempo otel-collector grafana caddy
|
||||
sudo docker exec docker-caddy-1 caddy validate --config /etc/caddy/Caddyfile --adapter caddyfile
|
||||
sudo docker exec docker-caddy-1 caddy reload --config /etc/caddy/Caddyfile --adapter caddyfile
|
||||
sudo docker exec docker-ai-service-1 python -m migrate
|
||||
sleep 10
|
||||
sudo docker run --rm --network docker_default curlimages/curl -sf http://ai-service:8000/health
|
||||
sudo docker run --rm --network docker_default curlimages/curl -sf http://ai-service:8000/ready
|
||||
sudo docker run --rm --network docker_default curlimages/curl -sf -o /dev/null http://web:3000
|
||||
sudo docker run --rm --network docker_default curlimages/curl -sf -o /dev/null https://realvuxbaro.me/grafana/login
|
||||
echo "Rollback to ${TARGET_SHA} verified healthy."
|
||||
Reference in New Issue
Block a user