Resolve ai-service directly from the RAG routes, never the gateway

.github/scripts/repair_argocd_inline_values.py

@@ -0,0 +1,157 @@
+"""Repair the corrupted inline `spec.source.helm.values` on medical-chatbot-app.
+
+## What broke
+
+Yesterday's manual ArgoCD UI edit saved the Application with a FOLDED block
+scalar (`values: >`) rather than a literal one (`values: |`). A folded scalar
+joins consecutive same-indent lines into one, so these four lines:
+
+    # Only the image blocks stay here: .github/scripts/sync_practice_argocd.py
+    # regex-rewrites these tags on every push. Every other setting is tracked in
+    # infra/helm/medical-chatbot/values-practice.yaml.
+    aiService:
+
+collapsed into a SINGLE line -- putting `aiService:` inside the comment. The
+stored string now starts:
+
+    0: '# Only the image blocks stay here: ... values-practice.yaml. aiService:'
+    1: '  image:'
+    ...
+    5: 'web:'
+
+which is not parseable YAML: line 1 is indented 2, so it would open the root
+mapping at indent 2, and `web:` at indent 0 then sits *outside* it.
+
+That one broken line explains every symptom together:
+  - the GHCR image overrides never apply, so Deployments fall back to the
+    chart default `duocthu-*:local`, which exists in no registry
+    (ErrImagePull -> stuck ReplicaSets -> App health Degraded);
+  - `authService`/`apiGateway` `enabled: true` never applies either, so those
+    Deployments were never created at all despite the UI showing them set;
+  - the App still reads "Synced" because ArgoCD's last SUCCESSFUL render was
+    16 hours ago -- it has been serving a stale comparison ever since.
+
+The old pods keep serving traffic (Kubernetes will not retire them until a
+replacement is Ready), which is why the site stayed up throughout.
+
+## The repair
+
+Rebuild the values with `aiService:` on its own line and drop the comment
+block entirely. Dropping it is deliberate, not laziness:
+
+  - a base-indent comment directly above a base-indent key is exactly what
+    the folded scalar destroys, so re-adding it re-arms the same trap for the
+    next person who edits this in the UI;
+  - it is stale anyway -- it points at `values-practice.yaml`, renamed to
+    `values-production.yaml` on 2026-08-17.
+
+The same explanation now lives in infra/argocd/applications/medical-chatbot-app.yaml,
+which is version-controlled and cannot be mangled by a UI text box.
+
+Every other line is preserved byte-for-byte, secrets included -- they are read
+from the live object and written straight back, never logged.
+
+Refuses to write unless the live values match the exact corruption described
+above, so a rerun (or a differently-broken Application) is a no-op rather than
+a second guess at what the content should be.
+
+Required env: ARGOCD_PRACTICE_URL, ARGOCD_PRACTICE_PASSWORD.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import sys
+import time
+import urllib.error
+import urllib.request
+
+APP_NAME = "medical-chatbot-app"
+MANGLED_PREFIX = "# Only the image blocks stay here:"
+MANGLED_SUFFIX = "aiService:"
+POLL_SECONDS = 20
+POLL_ROUNDS = 9  # ~3 minutes
+
+
+def call(base: str, method: str, path: str, token: str | None = None, body=None):
+    req = urllib.request.Request(
+        f"{base}{path}",
+        data=json.dumps(body).encode() if body is not None else None,
+        method=method,
+        headers={"Content-Type": "application/json"},
+    )
+    if token:
+        req.add_header("Authorization", f"Bearer {token}")
+    with urllib.request.urlopen(req, timeout=30) as resp:
+        raw = resp.read()
+        return json.loads(raw) if raw else {}
+
+
+def report(app: dict) -> None:
+    status = app.get("status", {})
+    print(f"  sync={status.get('sync', {}).get('status')} health={status.get('health', {}).get('status')}")
+    for r in status.get("resources", []):
+        health = (r.get("health") or {}).get("status")
+        if r.get("kind") == "Deployment":
+            print(f"    {r.get('kind'):<11} {r.get('name'):<48} health={health}")
+
+
+def main() -> int:
+    base = os.environ["ARGOCD_PRACTICE_URL"].rstrip("/")
+    password = os.environ["ARGOCD_PRACTICE_PASSWORD"]
+
+    session = call(base, "POST", "/api/v1/session", body={"username": "admin", "password": password})
+    token = session["token"]
+
+    app = call(base, "GET", f"/api/v1/applications/{APP_NAME}", token=token)
+    values = app["spec"]["source"]["helm"].get("values", "")
+    lines = values.splitlines()
+
+    if not lines:
+        print("Inline values are empty -- nothing to repair, and nothing safe to guess.", file=sys.stderr)
+        return 1
+
+    first = lines[0]
+    if not (first.startswith(MANGLED_PREFIX) and first.rstrip().endswith(MANGLED_SUFFIX)):
+        print(
+            "Line 0 is not the known folded-comment corruption; refusing to rewrite.\n"
+            f"  line 0 = {first!r}",
+            file=sys.stderr,
+        )
+        return 1
+
+    # Drop the mangled comment line, keep `aiService:` that it swallowed, and
+    # leave every remaining line untouched.
+    repaired = "\n".join(["aiService:"] + lines[1:]) + "\n"
+
+    print(f"Repairing line 0: {len(values)} chars -> {len(repaired)} chars")
+    print("Structural keys after repair:")
+    for line in repaired.splitlines():
+        if line and not line[0].isspace():
+            print(f"  {line.split(':')[0]}:")
+
+    app["spec"]["source"]["helm"]["values"] = repaired
+    call(base, "PUT", f"/api/v1/applications/{APP_NAME}", token=token, body=app)
+    print("PUT accepted.")
+
+    try:
+        call(base, "POST", f"/api/v1/applications/{APP_NAME}/sync", token=token, body={})
+        print("Sync triggered.")
+    except urllib.error.HTTPError as exc:
+        if exc.code == 400:
+            print("Explicit sync raced with autosync (expected under selfHeal) -- continuing.")
+        else:
+            raise
+
+    for i in range(POLL_ROUNDS):
+        time.sleep(POLL_SECONDS)
+        app = call(base, "GET", f"/api/v1/applications/{APP_NAME}", token=token)
+        print(f"--- poll {i + 1}/{POLL_ROUNDS} (+{(i + 1) * POLL_SECONDS}s) ---")
+        report(app)
+
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())