name: Rollback production # Manual escape hatch for deploy.yml. deploy.yml has NO automatic rollback: # it runs `git reset --hard origin/master`, rebuilds and runs migrations # BEFORE its health checks, so a deploy that fails those checks leaves the # server on the bad commit with no automatic recovery. This workflow points # the same reset+rebuild+health-check sequence at an earlier commit instead. # # Migrations are forward-only (apps/ai-service/migrate.py, no down scripts) # but every migration so far uses IF NOT EXISTS / ADD COLUMN IF NOT EXISTS, # so re-running them against an older commit is a no-op, not an error. A # future non-idempotent migration would break this guarantee. on: workflow_dispatch: inputs: target_sha: description: "Commit SHA or tag to roll back to (e.g. the last known-good commit from a previous successful 'Deploy to production' run)" required: true jobs: rollback: runs-on: ubuntu-latest steps: - name: Rollback over SSH uses: appleboy/ssh-action@v1.0.3 env: GRAFANA_ADMIN_PASSWORD: ${{ secrets.GRAFANA_ADMIN_PASSWORD }} TARGET_SHA: ${{ inputs.target_sha }} with: host: ${{ secrets.EC2_HOST }} username: ubuntu key: ${{ secrets.EC2_SSH_KEY }} envs: GRAFANA_ADMIN_PASSWORD,TARGET_SHA script: | set -e test -n "${GRAFANA_ADMIN_PASSWORD:-}" export GRAFANA_ADMIN_PASSWORD cd ~/app git fetch origin git rev-parse --verify "${TARGET_SHA}^{commit}" git reset --hard "${TARGET_SHA}" echo "Rolled back to $(git rev-parse HEAD) — $(git log -1 --format=%s)" cd infra/docker sudo -E docker compose \ -f docker-compose.prod.yml \ -f docker-compose.observability.yml \ up -d --build \ ai-service web prometheus tempo otel-collector grafana caddy sudo docker exec docker-caddy-1 caddy validate --config /etc/caddy/Caddyfile --adapter caddyfile sudo docker exec docker-caddy-1 caddy reload --config /etc/caddy/Caddyfile --adapter caddyfile sudo docker exec docker-ai-service-1 python -m migrate sleep 10 sudo docker run --rm --network docker_default curlimages/curl -sf http://ai-service:8000/health sudo docker run --rm --network docker_default curlimages/curl -sf http://ai-service:8000/ready sudo docker run --rm --network docker_default curlimages/curl -sf -o /dev/null http://web:3000 sudo docker run --rm --network docker_default curlimages/curl -sf -o /dev/null https://realvuxbaro.me/grafana/login echo "Rollback to ${TARGET_SHA} verified healthy."