Fix migration workflow: upload as artifact instead of scp to practice EC2

.github/workflows/ci.yml

@@ -0,0 +1,103 @@
+name: CI
+
+# Runs on every push and every pull request. `deploy.yml` triggers
+# independently on push to master; until it is made to depend on this job, a
+# red CI does NOT block a deploy — see docs/22-ci-cd.md.
+on:
+  push:
+  pull_request:
+
+concurrency:
+  group: ci-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  ai-service:
+    name: ai-service — ruff + pytest
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      # No lockfile exists for either Python project (docs/27-technical-debt.md
+      # D-07), so this mirrors apps/ai-service/Dockerfile's inline install. When
+      # a lockfile lands, replace this with an install from it.
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install \
+            "fastapi>=0.115,<1" \
+            "httpx>=0.27,<1" \
+            "psycopg[binary]>=3.2,<4" \
+            "pydantic-settings>=2.6,<3" \
+            "qdrant-client>=1.7,<2" \
+            "uvicorn[standard]>=0.30,<1" \
+            "prometheus-client>=0.20,<1" \
+            "opentelemetry-api>=1.27,<2" \
+            "opentelemetry-sdk>=1.27,<2" \
+            "opentelemetry-exporter-otlp-proto-http>=1.27,<2" \
+            "anthropic>=0.112,<1" \
+            "boto3" \
+            "pytest>=7.4,<9" \
+            "ruff"
+
+      - name: ruff
+        working-directory: apps/ai-service
+        run: ruff check .
+
+      # `tests/conftest.py` forces EMBEDDING_PROVIDER=disabled, because
+      # `main.py` builds the whole runtime at import time and would otherwise
+      # try to reach Qdrant during collection. No test needs a live datastore;
+      # `tests/test_live_datastores.py` gates itself behind RUN_INTEGRATION=1.
+      - name: pytest
+        working-directory: apps/ai-service
+        run: pytest tests -q
+
+  ingestion:
+    name: ingestion — pytest
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      # `ruff check` is not run here: `ingestion/pyproject.toml` declares no
+      # [tool.ruff] section, so ruff would apply its full default rule set and
+      # report ~426 pre-existing findings. Adding the same lint config
+      # apps/ai-service uses is tracked as follow-up work, not silenced here.
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e "./ingestion[dev]"
+
+      - name: pytest
+        working-directory: ingestion
+        run: pytest tests -q
+
+  web:
+    name: web — lint + build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "20"
+
+      # Same toolchain the production image uses (apps/web/Dockerfile).
+      - name: Enable pnpm
+        run: corepack enable
+
+      - name: Install
+        run: pnpm install --frozen-lockfile
+
+      - name: Lint
+        run: pnpm --filter @duoc-thu/web lint
+
+      - name: Build
+        run: pnpm --filter @duoc-thu/web build

.github/workflows/migrate-qdrant-snapshot.yml

@@ -0,0 +1,59 @@
+name: Migrate Qdrant snapshot to practice cluster
+
+# One-off, manual (workflow_dispatch only) bridge: snapshots the production
+# Qdrant collection (a live, non-disruptive Qdrant operation — this is how
+# the original prod migration was done, just in reverse) and relays the
+# snapshot files to the isolated k3s practice EC2. Uses the SAME EC2_SSH_KEY
+# deploy.yml already has (never exposed to the operator) plus a new
+# PRACTICE_SSH_KEY scoped only to the practice box. Delete this workflow
+# file once the one-time migration is done — it is not part of the regular
+# deploy path.
+
+on:
+  workflow_dispatch:
+
+jobs:
+  migrate:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up SSH key
+        run: |
+          mkdir -p ~/.ssh
+          printf '%s\n' "${{ secrets.EC2_SSH_KEY }}" > ~/.ssh/prod.pem
+          chmod 600 ~/.ssh/prod.pem
+          ssh-keyscan -H "${{ secrets.EC2_HOST }}" >> ~/.ssh/known_hosts 2>/dev/null
+
+      - name: Snapshot Qdrant collections on production
+        run: |
+          ssh -i ~/.ssh/prod.pem "ubuntu@${{ secrets.EC2_HOST }}" '
+            set -e
+            sudo docker run --rm --network docker_default curlimages/curl -sf -X POST http://qdrant:6333/collections/duocthu_v1/snapshots > /dev/null
+            sudo docker run --rm --network docker_default curlimages/curl -sf -X POST http://qdrant:6333/collections/duocthu_v1__manifest/snapshots > /dev/null
+            sleep 3
+            SNAP=$(sudo docker exec docker-qdrant-1 ls -t /qdrant/storage/snapshots/duocthu_v1/ | head -1)
+            SNAPM=$(sudo docker exec docker-qdrant-1 ls -t /qdrant/storage/snapshots/duocthu_v1__manifest/ | head -1)
+            sudo docker cp "docker-qdrant-1:/qdrant/storage/snapshots/duocthu_v1/${SNAP}" /tmp/duocthu_v1.snapshot
+            sudo docker cp "docker-qdrant-1:/qdrant/storage/snapshots/duocthu_v1__manifest/${SNAPM}" /tmp/duocthu_v1__manifest.snapshot
+            sudo chown ubuntu:ubuntu /tmp/duocthu_v1.snapshot /tmp/duocthu_v1__manifest.snapshot
+            ls -la /tmp/*.snapshot
+          '
+
+      - name: Pull snapshots to the runner
+        run: |
+          scp -i ~/.ssh/prod.pem "ubuntu@${{ secrets.EC2_HOST }}:/tmp/duocthu_v1.snapshot" ./duocthu_v1.snapshot
+          scp -i ~/.ssh/prod.pem "ubuntu@${{ secrets.EC2_HOST }}:/tmp/duocthu_v1__manifest.snapshot" ./duocthu_v1__manifest.snapshot
+          ls -la ./*.snapshot
+
+      - name: Upload snapshots as a workflow artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: qdrant-snapshots
+          path: |
+            duocthu_v1.snapshot
+            duocthu_v1__manifest.snapshot
+          retention-days: 1
+
+      - name: Clean up temp files on production
+        if: always()
+        run: |
+          ssh -i ~/.ssh/prod.pem "ubuntu@${{ secrets.EC2_HOST }}" 'rm -f /tmp/duocthu_v1.snapshot /tmp/duocthu_v1__manifest.snapshot' || true

.github/workflows/rollback.yml

@@ -0,0 +1,58 @@
+name: Rollback production
+
+# Manual escape hatch for deploy.yml. deploy.yml has NO automatic rollback:
+# it runs `git reset --hard origin/master`, rebuilds and runs migrations
+# BEFORE its health checks, so a deploy that fails those checks leaves the
+# server on the bad commit with no automatic recovery. This workflow points
+# the same reset+rebuild+health-check sequence at an earlier commit instead.
+#
+# Migrations are forward-only (apps/ai-service/migrate.py, no down scripts)
+# but every migration so far uses IF NOT EXISTS / ADD COLUMN IF NOT EXISTS,
+# so re-running them against an older commit is a no-op, not an error. A
+# future non-idempotent migration would break this guarantee.
+
+on:
+  workflow_dispatch:
+    inputs:
+      target_sha:
+        description: "Commit SHA or tag to roll back to (e.g. the last known-good commit from a previous successful 'Deploy to production' run)"
+        required: true
+
+jobs:
+  rollback:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Rollback over SSH
+        uses: appleboy/ssh-action@v1.0.3
+        env:
+          GRAFANA_ADMIN_PASSWORD: ${{ secrets.GRAFANA_ADMIN_PASSWORD }}
+          TARGET_SHA: ${{ inputs.target_sha }}
+        with:
+          host: ${{ secrets.EC2_HOST }}
+          username: ubuntu
+          key: ${{ secrets.EC2_SSH_KEY }}
+          envs: GRAFANA_ADMIN_PASSWORD,TARGET_SHA
+          script: |
+            set -e
+            test -n "${GRAFANA_ADMIN_PASSWORD:-}"
+            export GRAFANA_ADMIN_PASSWORD
+            cd ~/app
+            git fetch origin
+            git rev-parse --verify "${TARGET_SHA}^{commit}"
+            git reset --hard "${TARGET_SHA}"
+            echo "Rolled back to $(git rev-parse HEAD) — $(git log -1 --format=%s)"
+            cd infra/docker
+            sudo -E docker compose \
+              -f docker-compose.prod.yml \
+              -f docker-compose.observability.yml \
+              up -d --build \
+              ai-service web prometheus tempo otel-collector grafana caddy
+            sudo docker exec docker-caddy-1 caddy validate --config /etc/caddy/Caddyfile --adapter caddyfile
+            sudo docker exec docker-caddy-1 caddy reload --config /etc/caddy/Caddyfile --adapter caddyfile
+            sudo docker exec docker-ai-service-1 python -m migrate
+            sleep 10
+            sudo docker run --rm --network docker_default curlimages/curl -sf http://ai-service:8000/health
+            sudo docker run --rm --network docker_default curlimages/curl -sf http://ai-service:8000/ready
+            sudo docker run --rm --network docker_default curlimages/curl -sf -o /dev/null http://web:3000
+            sudo docker run --rm --network docker_default curlimages/curl -sf -o /dev/null https://realvuxbaro.me/grafana/login
+            echo "Rollback to ${TARGET_SHA} verified healthy."