Fingerprint the Qdrant corpus by content, not by count

2026-08-17 14:31:56 +07:00
parent c778f7e9cc
commit d1c9933b9e
8 changed files with 391 additions and 43 deletions
@@ -0,0 +1,53 @@
+name: Audit production Qdrant corpus (read-only)
+
+# Answers "is the practice corpus the same corpus production serves?" with
+# content hashes rather than a point count, which two different corpora can
+# share. The identical script runs against the k3s cluster over SSH, so the
+# two fingerprints are directly comparable.
+#
+# Read-only: it scrolls points and reads collection info. It changes nothing on
+# production, and its path is not in deploy.yml's filters, so merging it cannot
+# restart the Compose stack.
+
+on:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: audit-qdrant-corpus
+  cancel-in-progress: false
+
+jobs:
+  audit:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      # Ship the script rather than inlining it, so production and practice
+      # provably run the same bytes instead of two copies that can drift.
+      - name: Encode fingerprint script
+        run: echo "SCRIPT_B64=$(base64 -w0 scripts/qdrant_fingerprint.py)" >> "$GITHUB_ENV"
+
+      - name: Fingerprint production corpus over SSH
+        uses: appleboy/ssh-action@v1.0.3
+        env:
+          SCRIPT_B64: ${{ env.SCRIPT_B64 }}
+        with:
+          host: ${{ secrets.EC2_HOST }}
+          username: ubuntu
+          key: ${{ secrets.EC2_SSH_KEY }}
+          command_timeout: 30m
+          envs: SCRIPT_B64
+          script: |
+            set -eu
+            cd ~/app/infra/docker
+            ai_id=$(sudo docker compose -f docker-compose.prod.yml ps -q ai-service)
+            test -n "$ai_id"
+
+            printf '%s\n' '=== qdrant_version ==='
+            sudo docker compose -f docker-compose.prod.yml images qdrant
+
+            printf '%s\n' '=== corpus_fingerprint ==='
+            printf '%s' "$SCRIPT_B64" | base64 -d | sudo docker exec -i "$ai_id" python -
@@ -26,12 +26,30 @@ jobs:
      - name: Render default and production manifests
        run: |
          helm template default infra/helm/medical-chatbot > /tmp/default.yaml
+
+          # values-prod.yaml leaves the image tags empty on purpose: production
+          # must run an immutable commit SHA, supplied per deploy. Rendering it
+          # without one has to fail rather than fall back to a development tag,
+          # so assert that failure here — otherwise the guard could rot into a
+          # silent default and nobody would notice until a cutover.
+          if helm template production infra/helm/medical-chatbot \
+               --values infra/helm/medical-chatbot/values-prod.yaml \
+               > /tmp/untagged.yaml 2>/tmp/untagged.err; then
+            echo "::error::production render succeeded with no image tag; the immutable-tag guard is gone"
+            exit 1
+          fi
+          grep -q 'image.tag must be set to an immutable tag' /tmp/untagged.err
+
          helm template production infra/helm/medical-chatbot \
            --values infra/helm/medical-chatbot/values-prod.yaml \
+            --set aiService.image.tag="$GITHUB_SHA" \
+            --set web.image.tag="$GITHUB_SHA" \
            > /tmp/production.yaml
          grep -q 'ANSWER_MODEL_ID: "qwen.qwen3-next-80b-a3b"' /tmp/production.yaml
          grep -q 'RERANK_ENABLED: "true"' /tmp/production.yaml
          grep -q 'checksum/runtime-config:' /tmp/production.yaml
+          grep -q "image: \"ghcr.io/baovu2k4/vsf-duocthu-ai-service:$GITHUB_SHA\"" /tmp/production.yaml
+          grep -q "image: \"ghcr.io/baovu2k4/vsf-duocthu-web:$GITHUB_SHA\"" /tmp/production.yaml

      # The practice cluster is only evidence for the production migration
      # while it renders the same behavioural contract as production, so both
@@ -45,36 +63,45 @@ jobs:
            --values infra/helm/medical-chatbot/values-practice-data.yaml \
            > /tmp/practice-data.yaml

+          # A bare `grep -q` fails the step with no indication of which
+          # assertion broke, and `set -e` ignores a status inverted with `!`,
+          # so a `! grep -q` assertion can never fail at all. Both directions
+          # go through helpers that name the pattern and exit explicitly.
+          # `--` matters: a YAML list item pattern starts with `-`, which grep
+          # would otherwise parse as an option bundle.
+          expect() {
+            if ! grep -q -- "$2" "$1"; then
+              echo "::error::$1 is missing: $2"
+              exit 1
+            fi
+          }
+          refute() {
+            if grep -q -- "$2" "$1"; then
+              echo "::error::$1 must not contain: $2"
+              exit 1
+            fi
+          }
+
          # Behavioural parity with the audited production runtime contract.
-          grep -q 'ANSWER_MODEL_ID: "qwen.qwen3-next-80b-a3b"' /tmp/practice-app.yaml
-          grep -q 'ANSWER_PROVIDER: "bedrock-converse"' /tmp/practice-app.yaml
-          grep -q 'EMBEDDING_PROVIDER: "cohere-v4"' /tmp/practice-app.yaml
-          grep -q 'EMBEDDING_DIMENSIONS: "1024"' /tmp/practice-app.yaml
-          grep -q 'EVIDENCE_MINIMUM_SCORE: "0.12"' /tmp/practice-app.yaml
-          grep -q 'RERANK_ENABLED: "true"' /tmp/practice-app.yaml
-          grep -q 'AWS_REGION: "us-east-1"' /tmp/practice-app.yaml
-          grep -q 'checksum/runtime-config:' /tmp/practice-app.yaml
-          grep -q 'host: readytochat.realvuxbaro.me' /tmp/practice-app.yaml
+          expect /tmp/practice-app.yaml 'ANSWER_MODEL_ID: "qwen.qwen3-next-80b-a3b"'
+          expect /tmp/practice-app.yaml 'ANSWER_PROVIDER: "bedrock-converse"'
+          expect /tmp/practice-app.yaml 'EMBEDDING_PROVIDER: "cohere-v4"'
+          expect /tmp/practice-app.yaml 'EMBEDDING_DIMENSIONS: "1024"'
+          expect /tmp/practice-app.yaml 'EVIDENCE_MINIMUM_SCORE: "0.12"'
+          expect /tmp/practice-app.yaml 'RERANK_ENABLED: "true"'
+          expect /tmp/practice-app.yaml 'AWS_REGION: "us-east-1"'
+          expect /tmp/practice-app.yaml 'checksum/runtime-config:'
+          expect /tmp/practice-app.yaml '- host: "readytochat.realvuxbaro.me"'

          # The app release must own neither data StatefulSet: PostgreSQL and
          # Qdrant belong to the data release, so an app-side sync failure or
          # prune can never delete the corpus or the query history. Only those
          # two use volumeClaimTemplates — the observability PVCs are the app
          # release's own and are expected here.
-          #
-          # `set -e` ignores a command whose status is inverted with `!`, so
-          # every must-NOT-contain assertion is written as an explicit exit.
-          refute() {
-            if grep -q "$2" "$1"; then
-              echo "::error::$1 must not contain: $2"
-              exit 1
-            fi
-          }
-
          refute /tmp/practice-app.yaml 'volumeClaimTemplates'
-          grep -q 'medical-chatbot-data-medical-chatbot-qdrant' /tmp/practice-app.yaml
+          expect /tmp/practice-app.yaml 'medical-chatbot-data-medical-chatbot-qdrant'

          # ...and the data release must own nothing else.
          refute /tmp/practice-data.yaml 'medical-chatbot-data-medical-chatbot-ai-service'
          refute /tmp/practice-data.yaml 'kind: Ingress'
-          grep -q 'volumeClaimTemplates' /tmp/practice-data.yaml
+          expect /tmp/practice-data.yaml 'volumeClaimTemplates'
@@ -213,14 +213,97 @@ the absolute approximately 12-second response time would be a separate RAG
 architecture change because a normal turn currently performs understanding,
 generation and entailment as sequential Qwen calls.

+## GitOps source-of-truth remediation applied (Claude, 2026-08-17 afternoon)
+
+Both practice Applications now render from tracked values files instead of
+untracked inline values. This closes what was open risk 1.
+
+- `infra/helm/medical-chatbot/values-practice.yaml` and
+  `values-practice-data.yaml` hold every stable non-secret setting of the two
+  releases. Pushed to `master` at `a5d1b86`.
+- Only the `aiService.image` / `web.image` blocks remain inline on
+  `medical-chatbot-app`, because `.github/scripts/sync_practice_argocd.py`
+  regex-rewrites those tags on every push; a tag committed to Git would be
+  stale by design. The rewrite contract (exactly one `repository:`/`tag:` pair
+  per image) was asserted against the new inline block before the PUT.
+  `medical-chatbot-data` now carries no inline values at all.
+- Nothing secret was inline, so nothing secret moved. Bedrock still
+  authenticates through the node's instance role.
+- `helm-chart.yml` now renders both practice releases and asserts the
+  production behavioural contract on them, plus that the app release owns
+  neither data StatefulSet — so an app-side sync or prune cannot delete the
+  corpus or the query history.
+
+Evidence that the refactor changed nothing:
+
+- The tracked files were checked to parse to structures identical to the live
+  inline values minus the image blocks, before any Application was edited.
+- ArgoCD effective manifests were captured before and after. Both releases are
+  **byte-for-byte identical** (23 and 6 manifests). Both converged
+  Synced/Healthy at `a5d1b86`. The unchanged `checksum/runtime-config` means
+  no Pod was rolled.
+
+Independent interleaved re-measurement, five *different* clinical questions
+(Codex's controlled sample repeated one Zolpidem query), fresh conversation
+each, alternating which environment ran first:
+
+| Scope | Practice | Production |
+| --- | ---: | ---: |
+| Average | 12.37 s | 13.06 s |
+| Median | 11.90 s | 12.77 s |
+| Maximum | 14.27 s | 16.25 s |
+| Errors/timeouts | 0 | 0 |
+
+All ten answers were `answerable` with at least one citation, and all five
+question pairs agreed on decision. This corroborates the latency-parity verdict
+across question variety rather than a single repeated query. Answer depth is
+still not deterministic between environments: Q1 returned 271 characters with
+one citation on practice against 769 characters with two on production, while
+Q2 and Q5 were character-identical. Decision parity is established; content
+parity is not, and needs a golden-set comparison before cutover.
+
+## Immutable production images applied (Claude, 2026-08-17 afternoon)
+
+`values-prod.yaml` was not deployable as written, beyond the known `latest`
+problem: it pinned no `image.repository`, so both images fell back to the
+chart's local development names (`duocthu-ai-service`, `duocthu-web`) and would
+never have resolved the GHCR packages, and it carried no `imagePullSecrets` for
+those private packages.
+
+Fixed on `master` at `b66c7e5`:
+
+- Both GHCR repositories pinned, `ghcr-pull-secret` declared.
+- Both tags left empty on purpose. Production must run an immutable commit SHA
+  supplied per deploy — the same shape practice already uses — because a
+  `latest` tag makes "which commit is production running?" unanswerable and
+  silently breaks rollback when the tag later points at different content.
+- A new `medical-chatbot.image` helper demands the tag explicitly, so an unset
+  tag fails the render instead of degrading into the chart's `local` default.
+- CI asserts that failure directly (renders `values-prod.yaml` with no tag and
+  requires a non-zero exit plus the guard's message), so the guard cannot rot
+  into a silent default unnoticed.
+
+Still open before this file can be used: the production ArgoCD Application does
+not exist yet, and the `medical-chatbot-prod` Secret it expects
+(`secret.create: false`) must be created in the target namespace first.
+
+Verification that neither afternoon change disturbed the running cluster: after
+both merges, ArgoCD converged Synced/Healthy at `b66c7e5` and the effective
+manifests of both releases remained **byte-for-byte identical to the original
+pre-refactor baseline**. A live query returned `answerable` with one citation
+and character-identical content on both environments (11.36 s practice,
+11.66 s production).
+
 ## Migration risks currently open

-1. The live practice Applications use untracked inline Helm values. This is not
-   yet a complete Git source of truth.
-2. `values-prod.yaml` still uses mutable `latest` image tags; it is not safe for
-   cutover as written even though its behavior settings now match production.
-3. PostgreSQL/Qdrant snapshot, restore, rollback, ingress/TLS/DNS, secrets,
+1. PostgreSQL/Qdrant snapshot, restore, rollback, ingress/TLS/DNS, secrets,
   resource limits, and failure recovery still need explicit rehearsal gates.
+3. Cluster-level inspection is currently blocked from this machine: the
+   practice security group scopes port 6443 to the operator's own outbound IP,
+   which has changed again, so `kubectl` hangs. ArgoCD's API and the public
+   ingress were used instead. Re-open the SG rule before any step that needs
+   direct Pod/exec access.
+4. Answer-content parity between environments is unverified (see above).

 ## Workspace ownership and current edits

@@ -228,23 +311,25 @@ generation and entailment as sequential Qwen calls.
  pre-existing uncommitted Claude/user changes in three Helm files concerning
  optional AWS static credentials plus untracked slide/material files. Do not
  stage, overwrite, or discard them accidentally.
- Codex created clean worktree `D:\VSF-DUOCTHU-codex-argocd`, branch
-  `agent/argocd-prod-migration`, from `origin/master` to isolate migration work.
- No Helm migration change has been made in that clean worktree yet at this
-  checkpoint.
+- Codex worktree `D:\VSF-DUOCTHU-codex-argocd`, branch
+  `agent/argocd-prod-migration`. Its migration work landed on `master` through
+  `5a1a600`..`1684e10`; the worktree is clean with nothing in flight.
+- Claude worktree `D:\VSF-DUOCTHU-claude-gitops`, branch
+  `agent/gitops-tracked-values`, holds the GitOps source-of-truth work above.
+  Merged fast-forward into `master` at `a5d1b86`. See
+  `CLAUDE_CLAIM_2026-08-17.md`.
 - A stale Claude worktree under `.claude/worktrees/agent-a1f5e73fc2ce814e8`
  contains unrelated, uncommitted table-reconstruction work from 2026-08-05.

 ## Next safe execution order

-1. Move the practice Application's stable non-secret config out of inline Helm
-   values and into a tracked values file; keep only dynamic image tags and
-   secret references outside Git.
-2. Verify the actual running Pod environment, image digest, Qdrant identity,
-   health, readiness and traces through a cluster-level inspection path.
-3. Run a larger interleaved production/practice benchmark and compare
-   p50/p95/p99,
-   timeout rate, answer decisions and citations.
+1. Create the production ArgoCD Application against `values-prod.yaml`, and the
+   `medical-chatbot-prod` Secret it expects, without pointing DNS at it yet.
+2. Restore the security-group rule for the operator's current IP, then verify
+   the running Pod environment, image digest, Qdrant identity, health,
+   readiness and traces through a cluster-level inspection path.
+3. Run the golden set against both environments and compare answer content, not
+   just decision and latency.
 4. Rehearse state restore and DNS rollback. Do not repoint `realvuxbaro.me`
   until the gates and rollback path pass.

@@ -10,6 +10,23 @@
 {{- end -}}
 {{- end -}}

+{{/*
+Container image reference.
+
+The tag is demanded explicitly rather than defaulted so that a values file
+which deliberately leaves it unset fails the render instead of silently
+inheriting the chart's `local` development tag. Production and practice both
+supply an immutable commit SHA per deploy — practice through the ArgoCD
+Application's inline values, rewritten by sync_practice_argocd.py — and a
+chart that guessed a tag here would deploy something nobody asked for.
+
+Usage: {{ include "medical-chatbot.image" (dict "image" .Values.web.image "name" "web") }}
+*/}}
+{{- define "medical-chatbot.image" -}}
+{{- $tag := required (printf "%s.image.tag must be set to an immutable tag (a commit SHA); the chart will not guess one" .name) .image.tag -}}
+{{- printf "%s:%s" (required (printf "%s.image.repository must be set" .name) .image.repository) $tag -}}
+{{- end -}}
+
 {{- define "medical-chatbot.labels" -}}
 app.kubernetes.io/name: {{ include "medical-chatbot.name" . }}
 app.kubernetes.io/instance: {{ .Release.Name }}
@@ -55,7 +55,7 @@ spec:
      {{- if .Values.aiService.migration.enabled }}
      initContainers:
        - name: migrate
-          image: "{{ .Values.aiService.image.repository }}:{{ .Values.aiService.image.tag }}"
+          image: {{ include "medical-chatbot.image" (dict "image" .Values.aiService.image "name" "aiService") | quote }}
          imagePullPolicy: {{ .Values.aiService.image.pullPolicy }}
          command: ["python", "migrate.py"]
          envFrom:
@@ -69,7 +69,7 @@ spec:
      {{- end }}
      containers:
        - name: ai-service
-          image: "{{ .Values.aiService.image.repository }}:{{ .Values.aiService.image.tag }}"
+          image: {{ include "medical-chatbot.image" (dict "image" .Values.aiService.image "name" "aiService") | quote }}
          imagePullPolicy: {{ .Values.aiService.image.pullPolicy }}
          ports:
            - { name: http, containerPort: 8000 }
@@ -22,7 +22,7 @@ spec:
        {{- toYaml .Values.global.imagePullSecrets | nindent 8 }}
      containers:
        - name: web
-          image: "{{ .Values.web.image.repository }}:{{ .Values.web.image.tag }}"
+          image: {{ include "medical-chatbot.image" (dict "image" .Values.web.image "name" "web") | quote }}
          imagePullPolicy: {{ .Values.web.image.pullPolicy }}
          env:
            - name: AI_SERVICE_URL
@@ -1,10 +1,33 @@
+# Production values for the eventual ArgoCD cutover of realvuxbaro.me.
+#
+# Not live yet: production still runs Docker Compose on its own EC2, which is
+# the DNS-level rollback for the migration. This file is what the production
+# ArgoCD Application will render from once the rehearsal gates pass.
+#
+# The behavioural settings mirror the production runtime contract audited on
+# 2026-08-17 — see coordination/ARGOCD_PRODUCTION_MIGRATION_STATE_2026-08-17.md.
+
 global:
  environment: production
+  # The GHCR packages are private, same as on the practice cluster. The Secret
+  # must exist in the target namespace before the first sync.
+  imagePullSecrets:
+    - name: ghcr-pull-secret

 aiService:
  replicaCount: 2
  image:
-    tag: latest
+    repository: ghcr.io/baovu2k4/vsf-duocthu-ai-service
+    # Deliberately empty. Production must run an immutable, verifiable image,
+    # so the tag is supplied per deploy as a commit SHA — through the ArgoCD
+    # Application's inline values, exactly as the practice cluster does. A
+    # `latest` here would make "which code is production running?" unanswerable
+    # and would break rollback, since the same tag would point at new content.
+    #
+    # `medical-chatbot.image` turns this empty value into a hard render error
+    # rather than a silent fallback to the chart's `local` development tag.
+    tag: ""
+    pullPolicy: Always
  config:
    embeddingProvider: cohere-v4
    embeddingDimensions: 1024
@@ -18,7 +41,9 @@ aiService:
 web:
  replicaCount: 2
  image:
-    tag: latest
+    repository: ghcr.io/baovu2k4/vsf-duocthu-web
+    tag: ""
+    pullPolicy: Always

 ingress:
  enabled: true
@@ -0,0 +1,141 @@
+"""Content fingerprint of a Qdrant collection, for comparing two environments.
+
+Point count and collection status say nothing about whether two collections
+hold the *same corpus*: the same 15,100 points could carry different text, be
+embedded by a different model, or be a stale re-ingest. This walks every point
+and reduces it to hashes that only match when the content matches.
+
+Payload and vectors are hashed separately on purpose. If the payload hash
+matches but the vector hash does not, the same source text was embedded
+differently — a different embedding model or dimension — which is exactly the
+failure a migration can introduce silently.
+
+Per-point digests are sorted before the final hash, so scroll order cannot
+change the result. Vectors are rounded before hashing because float formatting
+is not guaranteed identical across versions; 6 decimals is far finer than any
+meaningful embedding difference.
+
+Reads QDRANT_URL and QDRANT_COLLECTION from the environment, so it runs
+unchanged inside the Compose ai-service container and the k3s ai-service pod.
+Read-only: it issues nothing but scroll and collection-info requests.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import sys
+import urllib.request
+
+BATCH = 256
+VECTOR_PRECISION = 6
+
+
+def post(url: str, body: dict) -> dict:
+    req = urllib.request.Request(
+        url,
+        data=json.dumps(body).encode(),
+        method="POST",
+        headers={"Content-Type": "application/json"},
+    )
+    with urllib.request.urlopen(req, timeout=120) as resp:
+        return json.load(resp)
+
+
+def get(url: str) -> dict:
+    with urllib.request.urlopen(url, timeout=60) as resp:
+        return json.load(resp)
+
+
+def canonical(value) -> str:
+    return json.dumps(value, sort_keys=True, ensure_ascii=True, separators=(",", ":"))
+
+
+def main() -> int:
+    base = os.environ["QDRANT_URL"].rstrip("/")
+    collection = os.environ.get("QDRANT_COLLECTION", "duocthu_v1")
+
+    info = get(f"{base}/collections/{collection}")["result"]
+    vectors_config = info.get("config", {}).get("params", {}).get("vectors", {})
+
+    payload_digests: list[str] = []
+    vector_digests: list[str] = []
+    drug_ids: set[str] = set()
+    section_keys: set[str] = set()
+    pages: list[int] = []
+    missing_vectors = 0
+
+    offset = None
+    seen = 0
+    while True:
+        body = {"limit": BATCH, "with_payload": True, "with_vector": True}
+        if offset is not None:
+            body["offset"] = offset
+        result = post(f"{base}/collections/{collection}/points/scroll", body)["result"]
+        points = result.get("points", [])
+        if not points:
+            break
+
+        for point in points:
+            pid = str(point.get("id"))
+            payload = point.get("payload") or {}
+            payload_digests.append(
+                hashlib.sha256((pid + "|" + canonical(payload)).encode()).hexdigest()
+            )
+
+            vector = point.get("vector")
+            if isinstance(vector, dict):  # named vectors
+                vector = canonical(
+                    {k: [round(float(x), VECTOR_PRECISION) for x in v] for k, v in vector.items()}
+                )
+            elif isinstance(vector, list):
+                vector = canonical([round(float(x), VECTOR_PRECISION) for x in vector])
+            else:
+                missing_vectors += 1
+                vector = "null"
+            vector_digests.append(hashlib.sha256((pid + "|" + vector).encode()).hexdigest())
+
+            if payload.get("drug_id"):
+                drug_ids.add(str(payload["drug_id"]))
+            if payload.get("section_key"):
+                section_keys.add(str(payload["section_key"]))
+            for key in ("printed_page_start", "printed_page_end"):
+                value = payload.get(key)
+                if isinstance(value, int):
+                    pages.append(value)
+
+        seen += len(points)
+        print(f"...scrolled {seen}", file=sys.stderr, flush=True)
+
+        offset = result.get("next_page_offset")
+        if offset is None:
+            break
+
+    payload_digests.sort()
+    vector_digests.sort()
+
+    print(
+        json.dumps(
+            {
+                "collection": collection,
+                "points_scrolled": seen,
+                "points_count_reported": info.get("points_count"),
+                "status": info.get("status"),
+                "vectors_config": vectors_config,
+                "payload_hash": hashlib.sha256("".join(payload_digests).encode()).hexdigest(),
+                "vector_hash": hashlib.sha256("".join(vector_digests).encode()).hexdigest(),
+                "missing_vectors": missing_vectors,
+                "distinct_drug_ids": len(drug_ids),
+                "distinct_section_keys": len(section_keys),
+                "printed_page_min": min(pages) if pages else None,
+                "printed_page_max": max(pages) if pages else None,
+            },
+            sort_keys=True,
+        )
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())