59 lines
2.7 KiB
YAML
59 lines
2.7 KiB
YAML
name: Rollback production
|
|
|
|
# Manual escape hatch for deploy.yml. deploy.yml has NO automatic rollback:
|
|
# it runs `git reset --hard origin/master`, rebuilds and runs migrations
|
|
# BEFORE its health checks, so a deploy that fails those checks leaves the
|
|
# server on the bad commit with no automatic recovery. This workflow points
|
|
# the same reset+rebuild+health-check sequence at an earlier commit instead.
|
|
#
|
|
# Migrations are forward-only (apps/ai-service/migrate.py, no down scripts)
|
|
# but every migration so far uses IF NOT EXISTS / ADD COLUMN IF NOT EXISTS,
|
|
# so re-running them against an older commit is a no-op, not an error. A
|
|
# future non-idempotent migration would break this guarantee.
|
|
|
|
on:
|
|
workflow_dispatch:
|
|
inputs:
|
|
target_sha:
|
|
description: "Commit SHA or tag to roll back to (e.g. the last known-good commit from a previous successful 'Deploy to production' run)"
|
|
required: true
|
|
|
|
jobs:
|
|
rollback:
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- name: Rollback over SSH
|
|
uses: appleboy/ssh-action@v1.0.3
|
|
env:
|
|
GRAFANA_ADMIN_PASSWORD: ${{ secrets.GRAFANA_ADMIN_PASSWORD }}
|
|
TARGET_SHA: ${{ inputs.target_sha }}
|
|
with:
|
|
host: ${{ secrets.EC2_HOST }}
|
|
username: ubuntu
|
|
key: ${{ secrets.EC2_SSH_KEY }}
|
|
envs: GRAFANA_ADMIN_PASSWORD,TARGET_SHA
|
|
script: |
|
|
set -e
|
|
test -n "${GRAFANA_ADMIN_PASSWORD:-}"
|
|
export GRAFANA_ADMIN_PASSWORD
|
|
cd ~/app
|
|
git fetch origin
|
|
git rev-parse --verify "${TARGET_SHA}^{commit}"
|
|
git reset --hard "${TARGET_SHA}"
|
|
echo "Rolled back to $(git rev-parse HEAD) — $(git log -1 --format=%s)"
|
|
cd infra/docker
|
|
sudo -E docker compose \
|
|
-f docker-compose.prod.yml \
|
|
-f docker-compose.observability.yml \
|
|
up -d --build \
|
|
ai-service web prometheus tempo otel-collector grafana caddy
|
|
sudo docker exec docker-caddy-1 caddy validate --config /etc/caddy/Caddyfile --adapter caddyfile
|
|
sudo docker exec docker-caddy-1 caddy reload --config /etc/caddy/Caddyfile --adapter caddyfile
|
|
sudo docker exec docker-ai-service-1 python -m migrate
|
|
sleep 10
|
|
sudo docker run --rm --network docker_default curlimages/curl -sf http://ai-service:8000/health
|
|
sudo docker run --rm --network docker_default curlimages/curl -sf http://ai-service:8000/ready
|
|
sudo docker run --rm --network docker_default curlimages/curl -sf -o /dev/null http://web:3000
|
|
sudo docker run --rm --network docker_default curlimages/curl -sf -o /dev/null https://realvuxbaro.me/grafana/login
|
|
echo "Rollback to ${TARGET_SHA} verified healthy."
|