Fix migration workflow: upload as artifact instead of scp to practice EC2

2026-08-13 11:14:25 +07:00
parent 7ebbe1f309
commit a4819b8653
51 changed files with 6830 additions and 8 deletions
@@ -0,0 +1,58 @@
+name: Rollback production
+
+# Manual escape hatch for deploy.yml. deploy.yml has NO automatic rollback:
+# it runs `git reset --hard origin/master`, rebuilds and runs migrations
+# BEFORE its health checks, so a deploy that fails those checks leaves the
+# server on the bad commit with no automatic recovery. This workflow points
+# the same reset+rebuild+health-check sequence at an earlier commit instead.
+#
+# Migrations are forward-only (apps/ai-service/migrate.py, no down scripts)
+# but every migration so far uses IF NOT EXISTS / ADD COLUMN IF NOT EXISTS,
+# so re-running them against an older commit is a no-op, not an error. A
+# future non-idempotent migration would break this guarantee.
+
+on:
+  workflow_dispatch:
+    inputs:
+      target_sha:
+        description: "Commit SHA or tag to roll back to (e.g. the last known-good commit from a previous successful 'Deploy to production' run)"
+        required: true
+
+jobs:
+  rollback:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Rollback over SSH
+        uses: appleboy/ssh-action@v1.0.3
+        env:
+          GRAFANA_ADMIN_PASSWORD: ${{ secrets.GRAFANA_ADMIN_PASSWORD }}
+          TARGET_SHA: ${{ inputs.target_sha }}
+        with:
+          host: ${{ secrets.EC2_HOST }}
+          username: ubuntu
+          key: ${{ secrets.EC2_SSH_KEY }}
+          envs: GRAFANA_ADMIN_PASSWORD,TARGET_SHA
+          script: |
+            set -e
+            test -n "${GRAFANA_ADMIN_PASSWORD:-}"
+            export GRAFANA_ADMIN_PASSWORD
+            cd ~/app
+            git fetch origin
+            git rev-parse --verify "${TARGET_SHA}^{commit}"
+            git reset --hard "${TARGET_SHA}"
+            echo "Rolled back to $(git rev-parse HEAD) — $(git log -1 --format=%s)"
+            cd infra/docker
+            sudo -E docker compose \
+              -f docker-compose.prod.yml \
+              -f docker-compose.observability.yml \
+              up -d --build \
+              ai-service web prometheus tempo otel-collector grafana caddy
+            sudo docker exec docker-caddy-1 caddy validate --config /etc/caddy/Caddyfile --adapter caddyfile
+            sudo docker exec docker-caddy-1 caddy reload --config /etc/caddy/Caddyfile --adapter caddyfile
+            sudo docker exec docker-ai-service-1 python -m migrate
+            sleep 10
+            sudo docker run --rm --network docker_default curlimages/curl -sf http://ai-service:8000/health
+            sudo docker run --rm --network docker_default curlimages/curl -sf http://ai-service:8000/ready
+            sudo docker run --rm --network docker_default curlimages/curl -sf -o /dev/null http://web:3000
+            sudo docker run --rm --network docker_default curlimages/curl -sf -o /dev/null https://realvuxbaro.me/grafana/login
+            echo "Rollback to ${TARGET_SHA} verified healthy."