Resolve ai-service directly from the RAG routes, never the gateway

.github/scripts/disable_api_gateway_live.py

@@ -0,0 +1,139 @@
+"""Turn apiGateway back off on the live Application to restore the chat.
+
+## Why
+
+Enabling apiGateway makes the Helm chart set `API_GATEWAY_URL` on the web pod
+(infra/helm/medical-chatbot/templates/web.yaml). Every one of web's BFF routes
+prefers that variable over `AI_SERVICE_URL`:
+
+    apps/web/app/api/chat/route.ts:7
+    apps/web/app/api/suggest/route.ts:5
+    apps/web/app/api/history/route.ts:5
+    apps/web/app/api/sections/route.ts:5
+    apps/web/app/api/section-text/route.ts:5
+    apps/web/app/api/feedback/route.ts:5
+
+        process.env.API_GATEWAY_URL ?? process.env.AI_SERVICE_URL ?? ...
+
+but the gateway only proxies `/auth/*` (apps/api-gateway/src/proxy/ has a
+single AuthProxyController). So all six start posting to a service with no
+such route, and the UI shows "Dịch vụ AI Service đang khởi động hoặc gặp sự
+cố tạm thời."
+
+The bug shipped in PR #27 yesterday and sat dormant: the configuration that
+triggers it was never actually rendered until the 2026-08-19 repair of the
+folded-scalar corruption deployed it for the first time.
+
+## What this does
+
+Flips `enabled` to false under the `apiGateway` key only -- `authService`'s
+own `enabled: true` is left alone -- so `API_GATEWAY_URL` stops being set and
+all six routes fall back to `AI_SERVICE_URL`, exactly the configuration that
+served traffic before today.
+
+Cost: login stops working again (apps/web/app/api/auth/{login,me}/route.ts
+read API_GATEWAY_URL with no fallback). That is the deliberate trade -- chat
+is the product and affects every visitor; login is a day-old addition that
+was not reachable before today anyway. The real fix is to stop routing RAG
+through the gateway at all, which needs an image rebuild.
+
+Refuses to act unless it finds `apiGateway:` followed by `enabled: true`, so
+it cannot silently do something else.
+
+Required env: ARGOCD_PRACTICE_URL, ARGOCD_PRACTICE_PASSWORD.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import sys
+import time
+import urllib.error
+import urllib.request
+
+APP_NAME = "medical-chatbot-app"
+POLL_SECONDS = 15
+POLL_ROUNDS = 8
+
+
+def call(base: str, method: str, path: str, token: str | None = None, body=None):
+    req = urllib.request.Request(
+        f"{base}{path}",
+        data=json.dumps(body).encode() if body is not None else None,
+        method=method,
+        headers={"Content-Type": "application/json"},
+    )
+    if token:
+        req.add_header("Authorization", f"Bearer {token}")
+    with urllib.request.urlopen(req, timeout=30) as resp:
+        raw = resp.read()
+        return json.loads(raw) if raw else {}
+
+
+def disable_api_gateway(values: str) -> str:
+    """Set `enabled: false` under the apiGateway key, and nowhere else."""
+    lines = values.splitlines()
+    out: list[str] = []
+    in_gateway = False
+    changed = 0
+
+    for line in lines:
+        stripped = line.strip()
+        # A non-indented key ends whatever block we were in.
+        if line and not line[0].isspace():
+            in_gateway = stripped == "apiGateway:"
+        elif in_gateway and stripped == "enabled: true":
+            indent = line[: len(line) - len(line.lstrip())]
+            line = f"{indent}enabled: false"
+            changed += 1
+        out.append(line)
+
+    if changed != 1:
+        raise SystemExit(
+            f"Expected exactly one `enabled: true` under apiGateway, changed {changed}. "
+            "Refusing to write."
+        )
+    return "\n".join(out) + "\n"
+
+
+def main() -> int:
+    base = os.environ["ARGOCD_PRACTICE_URL"].rstrip("/")
+    password = os.environ["ARGOCD_PRACTICE_PASSWORD"]
+
+    session = call(base, "POST", "/api/v1/session", body={"username": "admin", "password": password})
+    token = session["token"]
+
+    app = call(base, "GET", f"/api/v1/applications/{APP_NAME}", token=token)
+    values = app["spec"]["source"]["helm"].get("values", "")
+
+    repaired = disable_api_gateway(values)
+    print("apiGateway.enabled -> false (authService untouched)")
+
+    app["spec"]["source"]["helm"]["values"] = repaired
+    call(base, "PUT", f"/api/v1/applications/{APP_NAME}", token=token, body=app)
+    print("PUT accepted.")
+
+    try:
+        call(base, "POST", f"/api/v1/applications/{APP_NAME}/sync", token=token, body={})
+        print("Sync triggered.")
+    except urllib.error.HTTPError as exc:
+        if exc.code == 400:
+            print("Explicit sync raced with autosync -- continuing.")
+        else:
+            raise
+
+    for i in range(POLL_ROUNDS):
+        time.sleep(POLL_SECONDS)
+        app = call(base, "GET", f"/api/v1/applications/{APP_NAME}", token=token)
+        status = app.get("status", {})
+        print(
+            f"poll {i + 1}/{POLL_ROUNDS}: sync={status.get('sync', {}).get('status')} "
+            f"health={status.get('health', {}).get('status')}"
+        )
+
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

.github/scripts/inspect_argocd_app.py

@@ -58,6 +58,18 @@ def main() -> int:
 
     print(f"sync.status={app.get('status', {}).get('sync', {}).get('status')}")
     print(f"health.status={app.get('status', {}).get('health', {}).get('status')}")
+
+    # The raw string, escaped -- a manual UI edit used a folded (`values: >`)
+    # block instead of a literal one, which silently joins same-indent lines.
+    # Only an escaped dump shows where the newlines really are; the pretty
+    # print below looks almost right and hides it.
+    print("--- spec.source.helm.values RAW (first line + newline positions) ---")
+    raw = app["spec"]["source"]["helm"].get("values", "")
+    for i, line in enumerate(raw.splitlines()):
+        safe = re.sub(r"(?i)(password|secret|jwt)[^\s]*:.*", r"\1<redacted>", line)
+        print(f"{i:>3}: {safe!r}")
+    print("--- end raw ---")
+
     print("--- spec.source.helm.values (secrets redacted) ---")
     for line in values.splitlines():
         print(redact(line))
@@ -78,6 +90,19 @@ def main() -> int:
         print(f"{c.get('type')}: {c.get('message')}")
     print("--- end conditions ---")
 
+    op = app.get("status", {}).get("operationState", {})
+    print("--- status.operationState (last sync operation) ---")
+    print(f"phase={op.get('phase')}")
+    print(f"message={op.get('message')}")
+    print(f"startedAt={op.get('startedAt')} finishedAt={op.get('finishedAt')}")
+    sync_res = op.get("syncResult") or {}
+    for r in sync_res.get("resources", []):
+        print(
+            f"  {r.get('kind'):<12} {r.get('name'):<45} "
+            f"status={r.get('status')} hookPhase={r.get('hookPhase')} message={r.get('message')}"
+        )
+    print("--- end operationState ---")
+
     print("--- resource tree (nodes with non-empty health/status) ---")
     try:
         tree = call(base, "GET", f"/api/v1/applications/{APP_NAME}/resource-tree", token=token)

.github/scripts/repair_argocd_inline_values.py

@@ -0,0 +1,157 @@
+"""Repair the corrupted inline `spec.source.helm.values` on medical-chatbot-app.
+
+## What broke
+
+Yesterday's manual ArgoCD UI edit saved the Application with a FOLDED block
+scalar (`values: >`) rather than a literal one (`values: |`). A folded scalar
+joins consecutive same-indent lines into one, so these four lines:
+
+    # Only the image blocks stay here: .github/scripts/sync_practice_argocd.py
+    # regex-rewrites these tags on every push. Every other setting is tracked in
+    # infra/helm/medical-chatbot/values-practice.yaml.
+    aiService:
+
+collapsed into a SINGLE line -- putting `aiService:` inside the comment. The
+stored string now starts:
+
+    0: '# Only the image blocks stay here: ... values-practice.yaml. aiService:'
+    1: '  image:'
+    ...
+    5: 'web:'
+
+which is not parseable YAML: line 1 is indented 2, so it would open the root
+mapping at indent 2, and `web:` at indent 0 then sits *outside* it.
+
+That one broken line explains every symptom together:
+  - the GHCR image overrides never apply, so Deployments fall back to the
+    chart default `duocthu-*:local`, which exists in no registry
+    (ErrImagePull -> stuck ReplicaSets -> App health Degraded);
+  - `authService`/`apiGateway` `enabled: true` never applies either, so those
+    Deployments were never created at all despite the UI showing them set;
+  - the App still reads "Synced" because ArgoCD's last SUCCESSFUL render was
+    16 hours ago -- it has been serving a stale comparison ever since.
+
+The old pods keep serving traffic (Kubernetes will not retire them until a
+replacement is Ready), which is why the site stayed up throughout.
+
+## The repair
+
+Rebuild the values with `aiService:` on its own line and drop the comment
+block entirely. Dropping it is deliberate, not laziness:
+
+  - a base-indent comment directly above a base-indent key is exactly what
+    the folded scalar destroys, so re-adding it re-arms the same trap for the
+    next person who edits this in the UI;
+  - it is stale anyway -- it points at `values-practice.yaml`, renamed to
+    `values-production.yaml` on 2026-08-17.
+
+The same explanation now lives in infra/argocd/applications/medical-chatbot-app.yaml,
+which is version-controlled and cannot be mangled by a UI text box.
+
+Every other line is preserved byte-for-byte, secrets included -- they are read
+from the live object and written straight back, never logged.
+
+Refuses to write unless the live values match the exact corruption described
+above, so a rerun (or a differently-broken Application) is a no-op rather than
+a second guess at what the content should be.
+
+Required env: ARGOCD_PRACTICE_URL, ARGOCD_PRACTICE_PASSWORD.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import sys
+import time
+import urllib.error
+import urllib.request
+
+APP_NAME = "medical-chatbot-app"
+MANGLED_PREFIX = "# Only the image blocks stay here:"
+MANGLED_SUFFIX = "aiService:"
+POLL_SECONDS = 20
+POLL_ROUNDS = 9  # ~3 minutes
+
+
+def call(base: str, method: str, path: str, token: str | None = None, body=None):
+    req = urllib.request.Request(
+        f"{base}{path}",
+        data=json.dumps(body).encode() if body is not None else None,
+        method=method,
+        headers={"Content-Type": "application/json"},
+    )
+    if token:
+        req.add_header("Authorization", f"Bearer {token}")
+    with urllib.request.urlopen(req, timeout=30) as resp:
+        raw = resp.read()
+        return json.loads(raw) if raw else {}
+
+
+def report(app: dict) -> None:
+    status = app.get("status", {})
+    print(f"  sync={status.get('sync', {}).get('status')} health={status.get('health', {}).get('status')}")
+    for r in status.get("resources", []):
+        health = (r.get("health") or {}).get("status")
+        if r.get("kind") == "Deployment":
+            print(f"    {r.get('kind'):<11} {r.get('name'):<48} health={health}")
+
+
+def main() -> int:
+    base = os.environ["ARGOCD_PRACTICE_URL"].rstrip("/")
+    password = os.environ["ARGOCD_PRACTICE_PASSWORD"]
+
+    session = call(base, "POST", "/api/v1/session", body={"username": "admin", "password": password})
+    token = session["token"]
+
+    app = call(base, "GET", f"/api/v1/applications/{APP_NAME}", token=token)
+    values = app["spec"]["source"]["helm"].get("values", "")
+    lines = values.splitlines()
+
+    if not lines:
+        print("Inline values are empty -- nothing to repair, and nothing safe to guess.", file=sys.stderr)
+        return 1
+
+    first = lines[0]
+    if not (first.startswith(MANGLED_PREFIX) and first.rstrip().endswith(MANGLED_SUFFIX)):
+        print(
+            "Line 0 is not the known folded-comment corruption; refusing to rewrite.\n"
+            f"  line 0 = {first!r}",
+            file=sys.stderr,
+        )
+        return 1
+
+    # Drop the mangled comment line, keep `aiService:` that it swallowed, and
+    # leave every remaining line untouched.
+    repaired = "\n".join(["aiService:"] + lines[1:]) + "\n"
+
+    print(f"Repairing line 0: {len(values)} chars -> {len(repaired)} chars")
+    print("Structural keys after repair:")
+    for line in repaired.splitlines():
+        if line and not line[0].isspace():
+            print(f"  {line.split(':')[0]}:")
+
+    app["spec"]["source"]["helm"]["values"] = repaired
+    call(base, "PUT", f"/api/v1/applications/{APP_NAME}", token=token, body=app)
+    print("PUT accepted.")
+
+    try:
+        call(base, "POST", f"/api/v1/applications/{APP_NAME}/sync", token=token, body={})
+        print("Sync triggered.")
+    except urllib.error.HTTPError as exc:
+        if exc.code == 400:
+            print("Explicit sync raced with autosync (expected under selfHeal) -- continuing.")
+        else:
+            raise
+
+    for i in range(POLL_ROUNDS):
+        time.sleep(POLL_SECONDS)
+        app = call(base, "GET", f"/api/v1/applications/{APP_NAME}", token=token)
+        print(f"--- poll {i + 1}/{POLL_ROUNDS} (+{(i + 1) * POLL_SECONDS}s) ---")
+        report(app)
+
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

.github/workflows/ci.yml

@@ -97,6 +97,26 @@ jobs:
       - name: Install
         run: pnpm install --frozen-lockfile
 
+      # api-gateway proxies /auth/* and nothing else (a single
+      # AuthProxyController). Preferring API_GATEWAY_URL in a RAG route
+      # therefore breaks chat, suggest, history, sections, section-text and
+      # feedback the moment apiGateway is enabled -- which is exactly what
+      # shipped in PR #27 and stayed invisible until the config was first
+      # rendered on 2026-08-19. Nothing else in CI would have caught it: the
+      # code compiles and lints fine, and it only misbehaves once a specific
+      # Helm value is set. Assert the boundary directly.
+      - name: Assert RAG routes never resolve through api-gateway
+        run: |
+          offenders=$(grep -rln 'API_GATEWAY_URL'             apps/web/app/api/chat             apps/web/app/api/suggest             apps/web/app/api/history             apps/web/app/api/sections             apps/web/app/api/section-text             apps/web/app/api/feedback || true)
+          if [ -n "$offenders" ]; then
+            echo "::error::RAG routes must use AI_SERVICE_URL, not API_GATEWAY_URL: $offenders"
+            exit 1
+          fi
+          # ...and auth must keep using it, or login silently talks to the
+          # wrong service instead.
+          grep -q 'API_GATEWAY_URL' apps/web/app/api/auth/login/route.ts
+          grep -q 'API_GATEWAY_URL' apps/web/app/api/auth/me/route.ts
+
       - name: Lint
         run: pnpm --filter @duoc-thu/web lint
 

.github/workflows/disable-api-gateway.yml

@@ -0,0 +1,22 @@
+name: Disable apiGateway on production (restore chat)
+
+# Emergency: enabling apiGateway sets API_GATEWAY_URL on the web pod, which
+# every BFF route prefers over AI_SERVICE_URL -- but the gateway only proxies
+# /auth/*, so chat/suggest/history/sections/section-text/feedback all break.
+# See the script docstring for the full trace.
+
+on:
+  workflow_dispatch: {}
+
+jobs:
+  disable:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+      - name: Disable apiGateway and sync
+        env:
+          ARGOCD_PRACTICE_URL: ${{ secrets.ARGOCD_PRACTICE_URL }}
+          ARGOCD_PRACTICE_PASSWORD: ${{ secrets.ARGOCD_PRACTICE_PASSWORD }}
+        run: python3 .github/scripts/disable_api_gateway_live.py

.github/workflows/repair-argocd-inline-values.yml

@@ -0,0 +1,23 @@
+name: Repair ArgoCD inline values
+
+# One-off repair for the folded-block-scalar corruption a manual UI edit left
+# on medical-chatbot-app's inline helm values on 2026-08-18 -- see the module
+# docstring in .github/scripts/repair_argocd_inline_values.py for the full
+# diagnosis. The script refuses to write unless it finds that exact
+# corruption, so running it once the Application is healthy is a no-op.
+
+on:
+  workflow_dispatch: {}
+
+jobs:
+  repair:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+      - name: Repair and sync
+        env:
+          ARGOCD_PRACTICE_URL: ${{ secrets.ARGOCD_PRACTICE_URL }}
+          ARGOCD_PRACTICE_PASSWORD: ${{ secrets.ARGOCD_PRACTICE_PASSWORD }}
+        run: python3 .github/scripts/repair_argocd_inline_values.py

apps/web/app/api/chat/route.ts

@@ -4,7 +4,13 @@ import type { AnswerBlock, AnswerPlan, Citation, SendMessageResponse } from "@du
 
 export const runtime = "nodejs";
 
-const API_GATEWAY_URL = process.env.API_GATEWAY_URL ?? process.env.AI_SERVICE_URL ?? "http://localhost:8000";
+// RAG never goes through api-gateway. The gateway proxies `/auth/*` only --
+// apps/api-gateway/src/proxy/ holds a single AuthProxyController -- so
+// preferring AI_SERVICE_URL here pointed every RAG call at a service with no
+// such route the moment apiGateway was enabled, breaking chat, suggest,
+// history, sections, section-text and feedback at once. Resolve ai-service
+// directly and let AI_SERVICE_URL mean what its name says: auth only.
+const AI_SERVICE_URL = process.env.AI_SERVICE_URL ?? "http://localhost:8000";
 
 interface RagCitation {
   chunk_id: string;
@@ -257,9 +263,9 @@ export async function POST(request: Request) {
 
   let rag: RagResponse;
   try {
-    const targetUrl = API_GATEWAY_URL.includes("/v1/rag")
+    const targetUrl = AI_SERVICE_URL.includes("/v1/rag")
-      ? API_GATEWAY_URL
+      ? AI_SERVICE_URL
-      : `${API_GATEWAY_URL}/v1/rag/query`;
+      : `${AI_SERVICE_URL}/v1/rag/query`;
 
     const upstreamHeaders: Record<string, string> = {
       "Content-Type": "application/json",
@@ -306,7 +312,7 @@ export async function POST(request: Request) {
       trace_id: `fallback-${Date.now()}`,
       decision: "abstain",
       reason: "upstream_unreachable",
-      answer: `Không thể kết nối đến AI Service (${API_GATEWAY_URL}). Vui lòng đảm bảo AI Service đã được bật.`,
+      answer: `Không thể kết nối đến AI Service (${AI_SERVICE_URL}). Vui lòng đảm bảo AI Service đã được bật.`,
       resolved_drug_id: null,
       citations: [],
     };

apps/web/app/api/feedback/route.ts

@@ -2,8 +2,13 @@ import { NextResponse } from "next/server";
 
 export const runtime = "nodejs";
 
-const API_GATEWAY_URL =
+// RAG never goes through api-gateway. The gateway proxies `/auth/*` only --
-  process.env.API_GATEWAY_URL ?? process.env.AI_SERVICE_URL ?? "http://localhost:8000";
+// apps/api-gateway/src/proxy/ holds a single AuthProxyController -- so
+// preferring AI_SERVICE_URL here pointed every RAG call at a service with no
+// such route the moment apiGateway was enabled, breaking chat, suggest,
+// history, sections, section-text and feedback at once. Resolve ai-service
+// directly and let AI_SERVICE_URL mean what its name says: auth only.
+const AI_SERVICE_URL = process.env.AI_SERVICE_URL ?? "http://localhost:8000";
 
 export async function POST(request: Request) {
   let traceId: string;
@@ -35,7 +40,7 @@ export async function POST(request: Request) {
     return NextResponse.json({ error: "conversation_id_too_long" }, { status: 400 });
   }
 
-  const base = API_GATEWAY_URL.replace(/\/v1\/rag(?:\/query)?\/?$/, "");
+  const base = AI_SERVICE_URL.replace(/\/v1\/rag(?:\/query)?\/?$/, "");
   try {
     const upstream = await fetch(`${base}/v1/rag/feedback`, {
       method: "POST",

apps/web/app/api/history/route.ts

@@ -2,8 +2,13 @@ import { NextResponse } from "next/server";
 
 export const runtime = "nodejs";
 
-const API_GATEWAY_URL =
+// RAG never goes through api-gateway. The gateway proxies `/auth/*` only --
-  process.env.API_GATEWAY_URL ?? process.env.AI_SERVICE_URL ?? "http://localhost:8000";
+// apps/api-gateway/src/proxy/ holds a single AuthProxyController -- so
+// preferring AI_SERVICE_URL here pointed every RAG call at a service with no
+// such route the moment apiGateway was enabled, breaking chat, suggest,
+// history, sections, section-text and feedback at once. Resolve ai-service
+// directly and let AI_SERVICE_URL mean what its name says: auth only.
+const AI_SERVICE_URL = process.env.AI_SERVICE_URL ?? "http://localhost:8000";
 
 export async function GET(request: Request) {
   const { searchParams } = new URL(request.url);
@@ -14,7 +19,7 @@ export async function GET(request: Request) {
   }
 
   try {
-    const base = API_GATEWAY_URL.replace(/\/v1\/rag(?:\/query)?\/?$/, "");
+    const base = AI_SERVICE_URL.replace(/\/v1\/rag(?:\/query)?\/?$/, "");
     const targetUrl = `${base}/v1/rag/history?conversation_id=${encodeURIComponent(conversationId)}`;
 
     const upstream = await fetch(targetUrl, {

apps/web/app/api/section-text/route.ts

@@ -2,11 +2,16 @@ import { NextResponse } from "next/server";
 
 export const runtime = "nodejs";
 
-const API_GATEWAY_URL =
+// RAG never goes through api-gateway. The gateway proxies `/auth/*` only --
-  process.env.API_GATEWAY_URL ?? process.env.AI_SERVICE_URL ?? "http://localhost:8000";
+// apps/api-gateway/src/proxy/ holds a single AuthProxyController -- so
+// preferring AI_SERVICE_URL here pointed every RAG call at a service with no
+// such route the moment apiGateway was enabled, breaking chat, suggest,
+// history, sections, section-text and feedback at once. Resolve ai-service
+// directly and let AI_SERVICE_URL mean what its name says: auth only.
+const AI_SERVICE_URL = process.env.AI_SERVICE_URL ?? "http://localhost:8000";
 
 function ragBaseUrl() {
-  return API_GATEWAY_URL.replace(/\/v1\/rag(?:\/query)?\/?$/, "");
+  return AI_SERVICE_URL.replace(/\/v1\/rag(?:\/query)?\/?$/, "");
 }
 
 export async function GET(request: Request) {

apps/web/app/api/sections/route.ts

@@ -2,11 +2,16 @@ import { NextResponse } from "next/server";
 
 export const runtime = "nodejs";
 
-const API_GATEWAY_URL =
+// RAG never goes through api-gateway. The gateway proxies `/auth/*` only --
-  process.env.API_GATEWAY_URL ?? process.env.AI_SERVICE_URL ?? "http://localhost:8000";
+// apps/api-gateway/src/proxy/ holds a single AuthProxyController -- so
+// preferring AI_SERVICE_URL here pointed every RAG call at a service with no
+// such route the moment apiGateway was enabled, breaking chat, suggest,
+// history, sections, section-text and feedback at once. Resolve ai-service
+// directly and let AI_SERVICE_URL mean what its name says: auth only.
+const AI_SERVICE_URL = process.env.AI_SERVICE_URL ?? "http://localhost:8000";
 
 function ragBaseUrl() {
-  return API_GATEWAY_URL.replace(/\/v1\/rag(?:\/query)?\/?$/, "");
+  return AI_SERVICE_URL.replace(/\/v1\/rag(?:\/query)?\/?$/, "");
 }
 
 export async function GET(request: Request) {

apps/web/app/api/suggest/route.ts

@@ -2,8 +2,13 @@ import { NextResponse } from "next/server";
 
 export const runtime = "nodejs";
 
-const API_GATEWAY_URL =
+// RAG never goes through api-gateway. The gateway proxies `/auth/*` only --
-  process.env.API_GATEWAY_URL ?? process.env.AI_SERVICE_URL ?? "http://localhost:8000";
+// apps/api-gateway/src/proxy/ holds a single AuthProxyController -- so
+// preferring AI_SERVICE_URL here pointed every RAG call at a service with no
+// such route the moment apiGateway was enabled, breaking chat, suggest,
+// history, sections, section-text and feedback at once. Resolve ai-service
+// directly and let AI_SERVICE_URL mean what its name says: auth only.
+const AI_SERVICE_URL = process.env.AI_SERVICE_URL ?? "http://localhost:8000";
 
 export async function GET(request: Request) {
   const { searchParams } = new URL(request.url);
@@ -14,9 +19,9 @@ export async function GET(request: Request) {
   }
 
   try {
-    const targetUrl = API_GATEWAY_URL.includes("/v1/rag")
+    const targetUrl = AI_SERVICE_URL.includes("/v1/rag")
-      ? `${API_GATEWAY_URL.replace(/\/query$/, "/suggest")}?q=${encodeURIComponent(q)}`
+      ? `${AI_SERVICE_URL.replace(/\/query$/, "/suggest")}?q=${encodeURIComponent(q)}`
-      : `${API_GATEWAY_URL}/v1/rag/suggest?q=${encodeURIComponent(q)}`;
+      : `${AI_SERVICE_URL}/v1/rag/suggest?q=${encodeURIComponent(q)}`;
 
     const upstream = await fetch(targetUrl, {
       method: "GET",

infra/argocd/applications/medical-chatbot-app.yaml

@@ -25,6 +25,37 @@
 # recovery. It does NOT restore the current image tag or the Grafana
 # password: set the tag afterward via `rollback-k3s.yml` (target_sha = the
 # last known-good commit) and re-enter the Grafana password by hand.
+#
+# ---------------------------------------------------------------------------
+# EDITING THE INLINE VALUES BY HAND — read this first (2026-08-18 outage)
+# ---------------------------------------------------------------------------
+# Editing `spec.source.helm.values` through the ArgoCD UI broke this
+# Application for ~16 hours. The UI saved the block as a FOLDED scalar
+# (`values: >`), and a folded scalar joins consecutive same-indent lines into
+# one. Three comment lines sat directly above `aiService:` at the same indent,
+# so all four became a single line and `aiService:` ended up *inside* the
+# comment — leaving YAML that cannot parse at all.
+#
+# It failed silently and misleadingly. ArgoCD kept reporting `Synced` (its
+# last successful render was hours stale), the UI's PARAMETERS tab still
+# showed sensible values, and the site stayed up — Kubernetes will not retire
+# working pods until a replacement goes Ready, and the replacement never
+# could. Meanwhile every override in the block was being ignored: the GHCR
+# image repositories (so Deployments fell back to the chart-default
+# `duocthu-*:local`, which exists in no registry) and `authService`/
+# `apiGateway` `enabled: true` (so neither was ever created).
+#
+# Therefore:
+#   - Do NOT put comments in the inline values. Explain things here instead;
+#     this file is version-controlled and no text box can mangle it.
+#   - Prefer the API over the UI for edits — see
+#     .github/scripts/repair_argocd_inline_values.py, which reads the live
+#     object, edits the string, and PUTs it back with the structure intact.
+#   - After ANY inline edit, verify with
+#     `.github/workflows/inspect-argocd-app.yml`: it dumps the raw string one
+#     escaped line at a time, which is the only view that reveals where the
+#     newlines actually are. The pretty-printed view looked almost correct
+#     throughout the outage.
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata: