Add read-only production runtime audit

2026-08-17 11:17:40 +07:00
parent 057d4ed9dc
commit a1de4715a4
106 changed files with 6869 additions and 1782 deletions
@@ -0,0 +1,82 @@
+"""Point the k3s practice cluster's ArgoCD Application at a freshly-built
+image tag, then trigger an immediate sync.
+
+Only touches `medical-chatbot-app` on the practice cluster
+(argocd.realvuxbaro.me). Never touches production — the EC2 Compose
+deployment isn't ArgoCD-managed at all.
+
+Required env: ARGOCD_PRACTICE_URL, ARGOCD_PRACTICE_PASSWORD, IMAGE_TAG.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import sys
+import urllib.error
+import urllib.request
+
+APP_NAME = "medical-chatbot-app"
+IMAGES = ("vsf-duocthu-ai-service", "vsf-duocthu-web")
+
+
+def call(base: str, method: str, path: str, token: str | None = None, body=None):
+    req = urllib.request.Request(
+        f"{base}{path}",
+        data=json.dumps(body).encode() if body is not None else None,
+        method=method,
+        headers={"Content-Type": "application/json"},
+    )
+    if token:
+        req.add_header("Authorization", f"Bearer {token}")
+    try:
+        with urllib.request.urlopen(req, timeout=30) as resp:
+            raw = resp.read()
+            return json.loads(raw) if raw else {}
+    except urllib.error.HTTPError as exc:
+        print(f"{method} {path} -> {exc.code}: {exc.read().decode(errors='replace')}", file=sys.stderr)
+        raise
+
+
+def main() -> int:
+    base = os.environ["ARGOCD_PRACTICE_URL"].rstrip("/")
+    password = os.environ["ARGOCD_PRACTICE_PASSWORD"]
+    tag = os.environ["IMAGE_TAG"]
+
+    session = call(base, "POST", "/api/v1/session", body={"username": "admin", "password": password})
+    token = session["token"]
+
+    app = call(base, "GET", f"/api/v1/applications/{APP_NAME}", token=token)
+    values = app["spec"]["source"]["helm"]["values"]
+
+    for image in IMAGES:
+        pattern = re.compile(
+            rf"(repository:\s*ghcr\.io/baovu2k4/{re.escape(image)}\s*\n\s*tag:\s*)\S+"
+        )
+        values, count = pattern.subn(rf"\g<1>{tag}", values)
+        if count != 1:
+            print(f"Expected exactly one tag: line after {image}'s repository line, found {count}", file=sys.stderr)
+            return 1
+
+    app["spec"]["source"]["helm"]["values"] = values
+    call(base, "PUT", f"/api/v1/applications/{APP_NAME}", token=token, body=app)
+
+    # selfHeal (syncPolicy.automated) reacts to the PUT above on its own —
+    # often before this explicit call lands, which then 400s with "another
+    # operation is already in progress". That race means the sync we wanted
+    # is already happening; only a genuinely different failure is fatal.
+    try:
+        call(base, "POST", f"/api/v1/applications/{APP_NAME}/sync", token=token, body={})
+    except urllib.error.HTTPError as exc:
+        if exc.code == 400:
+            print(f"Explicit sync raced with autosync (expected under selfHeal) — continuing.")
+        else:
+            raise
+
+    print(f"{APP_NAME} pointed at tag {tag}; sync in progress.")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
@@ -0,0 +1,122 @@
+name: Audit production runtime (read-only)
+
+on:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: audit-production-runtime
+  cancel-in-progress: false
+
+jobs:
+  audit:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Inspect production over SSH
+        uses: appleboy/ssh-action@v1.0.3
+        with:
+          host: ${{ secrets.EC2_HOST }}
+          username: ubuntu
+          key: ${{ secrets.EC2_SSH_KEY }}
+          command_timeout: 10m
+          script: |
+            set -eu
+            cd ~/app
+
+            printf '%s\n' '=== source ==='
+            printf 'git_sha='
+            git rev-parse HEAD
+            printf 'git_branch='
+            git branch --show-current
+
+            cd infra/docker
+            ai_id=$(sudo docker compose -f docker-compose.prod.yml ps -q ai-service)
+            web_id=$(sudo docker compose -f docker-compose.prod.yml ps -q web)
+            postgres_id=$(sudo docker compose -f docker-compose.prod.yml ps -q postgres)
+            qdrant_id=$(sudo docker compose -f docker-compose.prod.yml ps -q qdrant)
+            test -n "$ai_id"
+            test -n "$web_id"
+            test -n "$postgres_id"
+            test -n "$qdrant_id"
+
+            printf '%s\n' '=== containers ==='
+            for entry in "ai-service:$ai_id" "web:$web_id" "postgres:$postgres_id" "qdrant:$qdrant_id"; do
+              service=${entry%%:*}
+              container=${entry#*:}
+              state=$(sudo docker inspect --format '{{.State.Status}}' "$container")
+              image_id=$(sudo docker inspect --format '{{.Image}}' "$container")
+              printf '%s state=%s image_id=%s\n' "$service" "$state" "$image_id"
+            done
+
+            printf '%s\n' '=== ai_runtime_contract ==='
+            sudo docker exec -i "$ai_id" python - <<'PY'
+            import json
+
+            from config import Settings
+
+            settings = Settings()
+            safe_fields = (
+                "app_name",
+                "environment",
+                "qdrant_url",
+                "qdrant_collection",
+                "embedding_provider",
+                "embedding_dimensions",
+                "evidence_minimum_score",
+                "aws_region",
+                "answer_provider",
+                "answer_model_id",
+                "rerank_enabled",
+                "metrics_enabled",
+                "otel_enabled",
+                "otel_service_name",
+                "otel_exporter_otlp_endpoint",
+                "otel_sample_ratio",
+                "entities_path",
+                "max_wall_clock_ms",
+                "max_llm_calls_per_turn",
+            )
+            contract = {name: str(getattr(settings, name)) for name in safe_fields}
+            print(json.dumps(contract, ensure_ascii=True, sort_keys=True))
+            PY
+
+            printf '%s\n' '=== persistent_mounts ==='
+            for entry in "postgres:$postgres_id" "qdrant:$qdrant_id"; do
+              service=${entry%%:*}
+              container=${entry#*:}
+              sudo docker inspect --format \
+                "$service {{range .Mounts}}{{.Type}}:{{.Name}}:{{.Destination}} {{end}}" \
+                "$container"
+            done
+
+            printf '%s\n' '=== datastore_identity ==='
+            sudo docker exec -i "$ai_id" python - <<'PY'
+            import json
+            import urllib.request
+
+            from config import Settings
+
+            settings = Settings()
+            url = settings.qdrant_url.rstrip("/") + "/collections/" + settings.qdrant_collection
+            with urllib.request.urlopen(url, timeout=10) as response:
+                payload = json.load(response)
+            result = payload.get("result", {})
+            config = result.get("config", {}).get("params", {}).get("vectors", {})
+            print(json.dumps({
+                "collection": settings.qdrant_collection,
+                "points_count": result.get("points_count"),
+                "status": result.get("status"),
+                "vector_config": config,
+            }, ensure_ascii=True, sort_keys=True))
+            PY
+
+            printf '%s\n' '=== health ==='
+            sudo docker exec -i "$ai_id" python - <<'PY'
+            import urllib.request
+
+            for path in ("/health", "/ready"):
+                with urllib.request.urlopen("http://127.0.0.1:8000" + path, timeout=10) as response:
+                    print(path, response.status)
+            PY
@@ -0,0 +1,86 @@
+name: Build and sync k3s practice images
+
+# Practice-cluster only (readytochat.realvuxbaro.me, ArgoCD-managed on the
+# self-hosted k3s box). Does not touch deploy.yml or the production
+# EC2/Compose stack — production never pulls a GHCR image and isn't
+# ArgoCD-managed at all, so this workflow has no path to affect it.
+#
+# ArgoCD's Applications already autosync (syncPolicy.automated) — the gap
+# this closes is that the image tag they deploy was a static string
+# (`:practice`) that nothing ever rebuilt. This tags every build with the
+# commit SHA and repoints the Application at it.
+
+on:
+  push:
+    branches: [master]
+    paths:
+      - apps/ai-service/**
+      - apps/web/**
+      - packages/**
+      - ingestion/data/verified/drug_entities.json
+      - .github/workflows/build-practice-images.yml
+      - .github/scripts/sync_practice_argocd.py
+  workflow_dispatch:
+
+concurrency:
+  group: practice-images
+  cancel-in-progress: false
+
+jobs:
+  build-and-sync:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: docker/setup-buildx-action@v3
+
+      - name: Log in to GHCR
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push ai-service
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: apps/ai-service/Dockerfile
+          push: true
+          tags: ghcr.io/baovu2k4/vsf-duocthu-ai-service:${{ github.sha }}
+          cache-from: type=gha,scope=practice-ai-service
+          cache-to: type=gha,mode=max,scope=practice-ai-service
+
+      - name: Build and push web
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: apps/web/Dockerfile
+          push: true
+          tags: ghcr.io/baovu2k4/vsf-duocthu-web:${{ github.sha }}
+          cache-from: type=gha,scope=practice-web
+          cache-to: type=gha,mode=max,scope=practice-web
+
+      - name: Point the practice ArgoCD Application at the new images
+        env:
+          ARGOCD_PRACTICE_URL: ${{ secrets.ARGOCD_PRACTICE_URL }}
+          ARGOCD_PRACTICE_PASSWORD: ${{ secrets.ARGOCD_PRACTICE_PASSWORD }}
+          IMAGE_TAG: ${{ github.sha }}
+        run: python3 .github/scripts/sync_practice_argocd.py
+
+      - name: Confirm readytochat is serving the new build
+        run: |
+          for attempt in $(seq 1 18); do
+            code=$(curl -s -o /dev/null -w '%{http_code}' \
+              "https://readytochat.realvuxbaro.me/api/history?conversation_id=ci-smoke-${{ github.sha }}")
+            if [ "$code" = "200" ]; then
+              echo "readytochat.realvuxbaro.me is live on ${{ github.sha }}"
+              exit 0
+            fi
+            sleep 10
+          done
+          echo "readytochat.realvuxbaro.me did not pick up ${{ github.sha }} within 3 minutes"
+          exit 1