diff --git a/.github/scripts/disable_api_gateway_live.py b/.github/scripts/disable_api_gateway_live.py new file mode 100644 index 0000000..235985a --- /dev/null +++ b/.github/scripts/disable_api_gateway_live.py @@ -0,0 +1,139 @@ +"""Turn apiGateway back off on the live Application to restore the chat. + +## Why + +Enabling apiGateway makes the Helm chart set `API_GATEWAY_URL` on the web pod +(infra/helm/medical-chatbot/templates/web.yaml). Every one of web's BFF routes +prefers that variable over `AI_SERVICE_URL`: + + apps/web/app/api/chat/route.ts:7 + apps/web/app/api/suggest/route.ts:5 + apps/web/app/api/history/route.ts:5 + apps/web/app/api/sections/route.ts:5 + apps/web/app/api/section-text/route.ts:5 + apps/web/app/api/feedback/route.ts:5 + + process.env.API_GATEWAY_URL ?? process.env.AI_SERVICE_URL ?? ... + +but the gateway only proxies `/auth/*` (apps/api-gateway/src/proxy/ has a +single AuthProxyController). So all six start posting to a service with no +such route, and the UI shows "Dịch vụ AI Service đang khởi động hoặc gặp sự +cố tạm thời." + +The bug shipped in PR #27 yesterday and sat dormant: the configuration that +triggers it was never actually rendered until the 2026-08-19 repair of the +folded-scalar corruption deployed it for the first time. + +## What this does + +Flips `enabled` to false under the `apiGateway` key only -- `authService`'s +own `enabled: true` is left alone -- so `API_GATEWAY_URL` stops being set and +all six routes fall back to `AI_SERVICE_URL`, exactly the configuration that +served traffic before today. + +Cost: login stops working again (apps/web/app/api/auth/{login,me}/route.ts +read API_GATEWAY_URL with no fallback). That is the deliberate trade -- chat +is the product and affects every visitor; login is a day-old addition that +was not reachable before today anyway. The real fix is to stop routing RAG +through the gateway at all, which needs an image rebuild. + +Refuses to act unless it finds `apiGateway:` followed by `enabled: true`, so +it cannot silently do something else. + +Required env: ARGOCD_PRACTICE_URL, ARGOCD_PRACTICE_PASSWORD. +""" + +from __future__ import annotations + +import json +import os +import sys +import time +import urllib.error +import urllib.request + +APP_NAME = "medical-chatbot-app" +POLL_SECONDS = 15 +POLL_ROUNDS = 8 + + +def call(base: str, method: str, path: str, token: str | None = None, body=None): + req = urllib.request.Request( + f"{base}{path}", + data=json.dumps(body).encode() if body is not None else None, + method=method, + headers={"Content-Type": "application/json"}, + ) + if token: + req.add_header("Authorization", f"Bearer {token}") + with urllib.request.urlopen(req, timeout=30) as resp: + raw = resp.read() + return json.loads(raw) if raw else {} + + +def disable_api_gateway(values: str) -> str: + """Set `enabled: false` under the apiGateway key, and nowhere else.""" + lines = values.splitlines() + out: list[str] = [] + in_gateway = False + changed = 0 + + for line in lines: + stripped = line.strip() + # A non-indented key ends whatever block we were in. + if line and not line[0].isspace(): + in_gateway = stripped == "apiGateway:" + elif in_gateway and stripped == "enabled: true": + indent = line[: len(line) - len(line.lstrip())] + line = f"{indent}enabled: false" + changed += 1 + out.append(line) + + if changed != 1: + raise SystemExit( + f"Expected exactly one `enabled: true` under apiGateway, changed {changed}. " + "Refusing to write." + ) + return "\n".join(out) + "\n" + + +def main() -> int: + base = os.environ["ARGOCD_PRACTICE_URL"].rstrip("/") + password = os.environ["ARGOCD_PRACTICE_PASSWORD"] + + session = call(base, "POST", "/api/v1/session", body={"username": "admin", "password": password}) + token = session["token"] + + app = call(base, "GET", f"/api/v1/applications/{APP_NAME}", token=token) + values = app["spec"]["source"]["helm"].get("values", "") + + repaired = disable_api_gateway(values) + print("apiGateway.enabled -> false (authService untouched)") + + app["spec"]["source"]["helm"]["values"] = repaired + call(base, "PUT", f"/api/v1/applications/{APP_NAME}", token=token, body=app) + print("PUT accepted.") + + try: + call(base, "POST", f"/api/v1/applications/{APP_NAME}/sync", token=token, body={}) + print("Sync triggered.") + except urllib.error.HTTPError as exc: + if exc.code == 400: + print("Explicit sync raced with autosync -- continuing.") + else: + raise + + for i in range(POLL_ROUNDS): + time.sleep(POLL_SECONDS) + app = call(base, "GET", f"/api/v1/applications/{APP_NAME}", token=token) + status = app.get("status", {}) + print( + f"poll {i + 1}/{POLL_ROUNDS}: sync={status.get('sync', {}).get('status')} " + f"health={status.get('health', {}).get('status')}" + ) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/scripts/inspect_argocd_app.py b/.github/scripts/inspect_argocd_app.py index 04c4595..d01bebb 100644 --- a/.github/scripts/inspect_argocd_app.py +++ b/.github/scripts/inspect_argocd_app.py @@ -58,6 +58,18 @@ def main() -> int: print(f"sync.status={app.get('status', {}).get('sync', {}).get('status')}") print(f"health.status={app.get('status', {}).get('health', {}).get('status')}") + + # The raw string, escaped -- a manual UI edit used a folded (`values: >`) + # block instead of a literal one, which silently joins same-indent lines. + # Only an escaped dump shows where the newlines really are; the pretty + # print below looks almost right and hides it. + print("--- spec.source.helm.values RAW (first line + newline positions) ---") + raw = app["spec"]["source"]["helm"].get("values", "") + for i, line in enumerate(raw.splitlines()): + safe = re.sub(r"(?i)(password|secret|jwt)[^\s]*:.*", r"\1", line) + print(f"{i:>3}: {safe!r}") + print("--- end raw ---") + print("--- spec.source.helm.values (secrets redacted) ---") for line in values.splitlines(): print(redact(line)) @@ -78,6 +90,19 @@ def main() -> int: print(f"{c.get('type')}: {c.get('message')}") print("--- end conditions ---") + op = app.get("status", {}).get("operationState", {}) + print("--- status.operationState (last sync operation) ---") + print(f"phase={op.get('phase')}") + print(f"message={op.get('message')}") + print(f"startedAt={op.get('startedAt')} finishedAt={op.get('finishedAt')}") + sync_res = op.get("syncResult") or {} + for r in sync_res.get("resources", []): + print( + f" {r.get('kind'):<12} {r.get('name'):<45} " + f"status={r.get('status')} hookPhase={r.get('hookPhase')} message={r.get('message')}" + ) + print("--- end operationState ---") + print("--- resource tree (nodes with non-empty health/status) ---") try: tree = call(base, "GET", f"/api/v1/applications/{APP_NAME}/resource-tree", token=token) diff --git a/.github/scripts/repair_argocd_inline_values.py b/.github/scripts/repair_argocd_inline_values.py new file mode 100644 index 0000000..01273dc --- /dev/null +++ b/.github/scripts/repair_argocd_inline_values.py @@ -0,0 +1,157 @@ +"""Repair the corrupted inline `spec.source.helm.values` on medical-chatbot-app. + +## What broke + +Yesterday's manual ArgoCD UI edit saved the Application with a FOLDED block +scalar (`values: >`) rather than a literal one (`values: |`). A folded scalar +joins consecutive same-indent lines into one, so these four lines: + + # Only the image blocks stay here: .github/scripts/sync_practice_argocd.py + # regex-rewrites these tags on every push. Every other setting is tracked in + # infra/helm/medical-chatbot/values-practice.yaml. + aiService: + +collapsed into a SINGLE line -- putting `aiService:` inside the comment. The +stored string now starts: + + 0: '# Only the image blocks stay here: ... values-practice.yaml. aiService:' + 1: ' image:' + ... + 5: 'web:' + +which is not parseable YAML: line 1 is indented 2, so it would open the root +mapping at indent 2, and `web:` at indent 0 then sits *outside* it. + +That one broken line explains every symptom together: + - the GHCR image overrides never apply, so Deployments fall back to the + chart default `duocthu-*:local`, which exists in no registry + (ErrImagePull -> stuck ReplicaSets -> App health Degraded); + - `authService`/`apiGateway` `enabled: true` never applies either, so those + Deployments were never created at all despite the UI showing them set; + - the App still reads "Synced" because ArgoCD's last SUCCESSFUL render was + 16 hours ago -- it has been serving a stale comparison ever since. + +The old pods keep serving traffic (Kubernetes will not retire them until a +replacement is Ready), which is why the site stayed up throughout. + +## The repair + +Rebuild the values with `aiService:` on its own line and drop the comment +block entirely. Dropping it is deliberate, not laziness: + + - a base-indent comment directly above a base-indent key is exactly what + the folded scalar destroys, so re-adding it re-arms the same trap for the + next person who edits this in the UI; + - it is stale anyway -- it points at `values-practice.yaml`, renamed to + `values-production.yaml` on 2026-08-17. + +The same explanation now lives in infra/argocd/applications/medical-chatbot-app.yaml, +which is version-controlled and cannot be mangled by a UI text box. + +Every other line is preserved byte-for-byte, secrets included -- they are read +from the live object and written straight back, never logged. + +Refuses to write unless the live values match the exact corruption described +above, so a rerun (or a differently-broken Application) is a no-op rather than +a second guess at what the content should be. + +Required env: ARGOCD_PRACTICE_URL, ARGOCD_PRACTICE_PASSWORD. +""" + +from __future__ import annotations + +import json +import os +import sys +import time +import urllib.error +import urllib.request + +APP_NAME = "medical-chatbot-app" +MANGLED_PREFIX = "# Only the image blocks stay here:" +MANGLED_SUFFIX = "aiService:" +POLL_SECONDS = 20 +POLL_ROUNDS = 9 # ~3 minutes + + +def call(base: str, method: str, path: str, token: str | None = None, body=None): + req = urllib.request.Request( + f"{base}{path}", + data=json.dumps(body).encode() if body is not None else None, + method=method, + headers={"Content-Type": "application/json"}, + ) + if token: + req.add_header("Authorization", f"Bearer {token}") + with urllib.request.urlopen(req, timeout=30) as resp: + raw = resp.read() + return json.loads(raw) if raw else {} + + +def report(app: dict) -> None: + status = app.get("status", {}) + print(f" sync={status.get('sync', {}).get('status')} health={status.get('health', {}).get('status')}") + for r in status.get("resources", []): + health = (r.get("health") or {}).get("status") + if r.get("kind") == "Deployment": + print(f" {r.get('kind'):<11} {r.get('name'):<48} health={health}") + + +def main() -> int: + base = os.environ["ARGOCD_PRACTICE_URL"].rstrip("/") + password = os.environ["ARGOCD_PRACTICE_PASSWORD"] + + session = call(base, "POST", "/api/v1/session", body={"username": "admin", "password": password}) + token = session["token"] + + app = call(base, "GET", f"/api/v1/applications/{APP_NAME}", token=token) + values = app["spec"]["source"]["helm"].get("values", "") + lines = values.splitlines() + + if not lines: + print("Inline values are empty -- nothing to repair, and nothing safe to guess.", file=sys.stderr) + return 1 + + first = lines[0] + if not (first.startswith(MANGLED_PREFIX) and first.rstrip().endswith(MANGLED_SUFFIX)): + print( + "Line 0 is not the known folded-comment corruption; refusing to rewrite.\n" + f" line 0 = {first!r}", + file=sys.stderr, + ) + return 1 + + # Drop the mangled comment line, keep `aiService:` that it swallowed, and + # leave every remaining line untouched. + repaired = "\n".join(["aiService:"] + lines[1:]) + "\n" + + print(f"Repairing line 0: {len(values)} chars -> {len(repaired)} chars") + print("Structural keys after repair:") + for line in repaired.splitlines(): + if line and not line[0].isspace(): + print(f" {line.split(':')[0]}:") + + app["spec"]["source"]["helm"]["values"] = repaired + call(base, "PUT", f"/api/v1/applications/{APP_NAME}", token=token, body=app) + print("PUT accepted.") + + try: + call(base, "POST", f"/api/v1/applications/{APP_NAME}/sync", token=token, body={}) + print("Sync triggered.") + except urllib.error.HTTPError as exc: + if exc.code == 400: + print("Explicit sync raced with autosync (expected under selfHeal) -- continuing.") + else: + raise + + for i in range(POLL_ROUNDS): + time.sleep(POLL_SECONDS) + app = call(base, "GET", f"/api/v1/applications/{APP_NAME}", token=token) + print(f"--- poll {i + 1}/{POLL_ROUNDS} (+{(i + 1) * POLL_SECONDS}s) ---") + report(app) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3424d8d..d578826 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -97,6 +97,26 @@ jobs: - name: Install run: pnpm install --frozen-lockfile + # api-gateway proxies /auth/* and nothing else (a single + # AuthProxyController). Preferring API_GATEWAY_URL in a RAG route + # therefore breaks chat, suggest, history, sections, section-text and + # feedback the moment apiGateway is enabled -- which is exactly what + # shipped in PR #27 and stayed invisible until the config was first + # rendered on 2026-08-19. Nothing else in CI would have caught it: the + # code compiles and lints fine, and it only misbehaves once a specific + # Helm value is set. Assert the boundary directly. + - name: Assert RAG routes never resolve through api-gateway + run: | + offenders=$(grep -rln 'API_GATEWAY_URL' apps/web/app/api/chat apps/web/app/api/suggest apps/web/app/api/history apps/web/app/api/sections apps/web/app/api/section-text apps/web/app/api/feedback || true) + if [ -n "$offenders" ]; then + echo "::error::RAG routes must use AI_SERVICE_URL, not API_GATEWAY_URL: $offenders" + exit 1 + fi + # ...and auth must keep using it, or login silently talks to the + # wrong service instead. + grep -q 'API_GATEWAY_URL' apps/web/app/api/auth/login/route.ts + grep -q 'API_GATEWAY_URL' apps/web/app/api/auth/me/route.ts + - name: Lint run: pnpm --filter @duoc-thu/web lint diff --git a/.github/workflows/disable-api-gateway.yml b/.github/workflows/disable-api-gateway.yml new file mode 100644 index 0000000..d8fc82b --- /dev/null +++ b/.github/workflows/disable-api-gateway.yml @@ -0,0 +1,22 @@ +name: Disable apiGateway on production (restore chat) + +# Emergency: enabling apiGateway sets API_GATEWAY_URL on the web pod, which +# every BFF route prefers over AI_SERVICE_URL -- but the gateway only proxies +# /auth/*, so chat/suggest/history/sections/section-text/feedback all break. +# See the script docstring for the full trace. + +on: + workflow_dispatch: {} + +jobs: + disable: + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + - name: Disable apiGateway and sync + env: + ARGOCD_PRACTICE_URL: ${{ secrets.ARGOCD_PRACTICE_URL }} + ARGOCD_PRACTICE_PASSWORD: ${{ secrets.ARGOCD_PRACTICE_PASSWORD }} + run: python3 .github/scripts/disable_api_gateway_live.py diff --git a/.github/workflows/repair-argocd-inline-values.yml b/.github/workflows/repair-argocd-inline-values.yml new file mode 100644 index 0000000..d669670 --- /dev/null +++ b/.github/workflows/repair-argocd-inline-values.yml @@ -0,0 +1,23 @@ +name: Repair ArgoCD inline values + +# One-off repair for the folded-block-scalar corruption a manual UI edit left +# on medical-chatbot-app's inline helm values on 2026-08-18 -- see the module +# docstring in .github/scripts/repair_argocd_inline_values.py for the full +# diagnosis. The script refuses to write unless it finds that exact +# corruption, so running it once the Application is healthy is a no-op. + +on: + workflow_dispatch: {} + +jobs: + repair: + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + - name: Repair and sync + env: + ARGOCD_PRACTICE_URL: ${{ secrets.ARGOCD_PRACTICE_URL }} + ARGOCD_PRACTICE_PASSWORD: ${{ secrets.ARGOCD_PRACTICE_PASSWORD }} + run: python3 .github/scripts/repair_argocd_inline_values.py diff --git a/apps/web/app/api/chat/route.ts b/apps/web/app/api/chat/route.ts index 4e0a6b1..e13e747 100644 --- a/apps/web/app/api/chat/route.ts +++ b/apps/web/app/api/chat/route.ts @@ -4,7 +4,13 @@ import type { AnswerBlock, AnswerPlan, Citation, SendMessageResponse } from "@du export const runtime = "nodejs"; -const API_GATEWAY_URL = process.env.API_GATEWAY_URL ?? process.env.AI_SERVICE_URL ?? "http://localhost:8000"; +// RAG never goes through api-gateway. The gateway proxies `/auth/*` only -- +// apps/api-gateway/src/proxy/ holds a single AuthProxyController -- so +// preferring AI_SERVICE_URL here pointed every RAG call at a service with no +// such route the moment apiGateway was enabled, breaking chat, suggest, +// history, sections, section-text and feedback at once. Resolve ai-service +// directly and let AI_SERVICE_URL mean what its name says: auth only. +const AI_SERVICE_URL = process.env.AI_SERVICE_URL ?? "http://localhost:8000"; interface RagCitation { chunk_id: string; @@ -257,9 +263,9 @@ export async function POST(request: Request) { let rag: RagResponse; try { - const targetUrl = API_GATEWAY_URL.includes("/v1/rag") - ? API_GATEWAY_URL - : `${API_GATEWAY_URL}/v1/rag/query`; + const targetUrl = AI_SERVICE_URL.includes("/v1/rag") + ? AI_SERVICE_URL + : `${AI_SERVICE_URL}/v1/rag/query`; const upstreamHeaders: Record = { "Content-Type": "application/json", @@ -306,7 +312,7 @@ export async function POST(request: Request) { trace_id: `fallback-${Date.now()}`, decision: "abstain", reason: "upstream_unreachable", - answer: `Không thể kết nối đến AI Service (${API_GATEWAY_URL}). Vui lòng đảm bảo AI Service đã được bật.`, + answer: `Không thể kết nối đến AI Service (${AI_SERVICE_URL}). Vui lòng đảm bảo AI Service đã được bật.`, resolved_drug_id: null, citations: [], }; diff --git a/apps/web/app/api/feedback/route.ts b/apps/web/app/api/feedback/route.ts index e127279..5d81819 100644 --- a/apps/web/app/api/feedback/route.ts +++ b/apps/web/app/api/feedback/route.ts @@ -2,8 +2,13 @@ import { NextResponse } from "next/server"; export const runtime = "nodejs"; -const API_GATEWAY_URL = - process.env.API_GATEWAY_URL ?? process.env.AI_SERVICE_URL ?? "http://localhost:8000"; +// RAG never goes through api-gateway. The gateway proxies `/auth/*` only -- +// apps/api-gateway/src/proxy/ holds a single AuthProxyController -- so +// preferring AI_SERVICE_URL here pointed every RAG call at a service with no +// such route the moment apiGateway was enabled, breaking chat, suggest, +// history, sections, section-text and feedback at once. Resolve ai-service +// directly and let AI_SERVICE_URL mean what its name says: auth only. +const AI_SERVICE_URL = process.env.AI_SERVICE_URL ?? "http://localhost:8000"; export async function POST(request: Request) { let traceId: string; @@ -35,7 +40,7 @@ export async function POST(request: Request) { return NextResponse.json({ error: "conversation_id_too_long" }, { status: 400 }); } - const base = API_GATEWAY_URL.replace(/\/v1\/rag(?:\/query)?\/?$/, ""); + const base = AI_SERVICE_URL.replace(/\/v1\/rag(?:\/query)?\/?$/, ""); try { const upstream = await fetch(`${base}/v1/rag/feedback`, { method: "POST", diff --git a/apps/web/app/api/history/route.ts b/apps/web/app/api/history/route.ts index 8c6edf8..14d2c73 100644 --- a/apps/web/app/api/history/route.ts +++ b/apps/web/app/api/history/route.ts @@ -2,8 +2,13 @@ import { NextResponse } from "next/server"; export const runtime = "nodejs"; -const API_GATEWAY_URL = - process.env.API_GATEWAY_URL ?? process.env.AI_SERVICE_URL ?? "http://localhost:8000"; +// RAG never goes through api-gateway. The gateway proxies `/auth/*` only -- +// apps/api-gateway/src/proxy/ holds a single AuthProxyController -- so +// preferring AI_SERVICE_URL here pointed every RAG call at a service with no +// such route the moment apiGateway was enabled, breaking chat, suggest, +// history, sections, section-text and feedback at once. Resolve ai-service +// directly and let AI_SERVICE_URL mean what its name says: auth only. +const AI_SERVICE_URL = process.env.AI_SERVICE_URL ?? "http://localhost:8000"; export async function GET(request: Request) { const { searchParams } = new URL(request.url); @@ -14,7 +19,7 @@ export async function GET(request: Request) { } try { - const base = API_GATEWAY_URL.replace(/\/v1\/rag(?:\/query)?\/?$/, ""); + const base = AI_SERVICE_URL.replace(/\/v1\/rag(?:\/query)?\/?$/, ""); const targetUrl = `${base}/v1/rag/history?conversation_id=${encodeURIComponent(conversationId)}`; const upstream = await fetch(targetUrl, { diff --git a/apps/web/app/api/section-text/route.ts b/apps/web/app/api/section-text/route.ts index 0a8d5e1..fbd383c 100644 --- a/apps/web/app/api/section-text/route.ts +++ b/apps/web/app/api/section-text/route.ts @@ -2,11 +2,16 @@ import { NextResponse } from "next/server"; export const runtime = "nodejs"; -const API_GATEWAY_URL = - process.env.API_GATEWAY_URL ?? process.env.AI_SERVICE_URL ?? "http://localhost:8000"; +// RAG never goes through api-gateway. The gateway proxies `/auth/*` only -- +// apps/api-gateway/src/proxy/ holds a single AuthProxyController -- so +// preferring AI_SERVICE_URL here pointed every RAG call at a service with no +// such route the moment apiGateway was enabled, breaking chat, suggest, +// history, sections, section-text and feedback at once. Resolve ai-service +// directly and let AI_SERVICE_URL mean what its name says: auth only. +const AI_SERVICE_URL = process.env.AI_SERVICE_URL ?? "http://localhost:8000"; function ragBaseUrl() { - return API_GATEWAY_URL.replace(/\/v1\/rag(?:\/query)?\/?$/, ""); + return AI_SERVICE_URL.replace(/\/v1\/rag(?:\/query)?\/?$/, ""); } export async function GET(request: Request) { diff --git a/apps/web/app/api/sections/route.ts b/apps/web/app/api/sections/route.ts index 3348971..33f871a 100644 --- a/apps/web/app/api/sections/route.ts +++ b/apps/web/app/api/sections/route.ts @@ -2,11 +2,16 @@ import { NextResponse } from "next/server"; export const runtime = "nodejs"; -const API_GATEWAY_URL = - process.env.API_GATEWAY_URL ?? process.env.AI_SERVICE_URL ?? "http://localhost:8000"; +// RAG never goes through api-gateway. The gateway proxies `/auth/*` only -- +// apps/api-gateway/src/proxy/ holds a single AuthProxyController -- so +// preferring AI_SERVICE_URL here pointed every RAG call at a service with no +// such route the moment apiGateway was enabled, breaking chat, suggest, +// history, sections, section-text and feedback at once. Resolve ai-service +// directly and let AI_SERVICE_URL mean what its name says: auth only. +const AI_SERVICE_URL = process.env.AI_SERVICE_URL ?? "http://localhost:8000"; function ragBaseUrl() { - return API_GATEWAY_URL.replace(/\/v1\/rag(?:\/query)?\/?$/, ""); + return AI_SERVICE_URL.replace(/\/v1\/rag(?:\/query)?\/?$/, ""); } export async function GET(request: Request) { diff --git a/apps/web/app/api/suggest/route.ts b/apps/web/app/api/suggest/route.ts index 66f54fe..387ba69 100644 --- a/apps/web/app/api/suggest/route.ts +++ b/apps/web/app/api/suggest/route.ts @@ -2,8 +2,13 @@ import { NextResponse } from "next/server"; export const runtime = "nodejs"; -const API_GATEWAY_URL = - process.env.API_GATEWAY_URL ?? process.env.AI_SERVICE_URL ?? "http://localhost:8000"; +// RAG never goes through api-gateway. The gateway proxies `/auth/*` only -- +// apps/api-gateway/src/proxy/ holds a single AuthProxyController -- so +// preferring AI_SERVICE_URL here pointed every RAG call at a service with no +// such route the moment apiGateway was enabled, breaking chat, suggest, +// history, sections, section-text and feedback at once. Resolve ai-service +// directly and let AI_SERVICE_URL mean what its name says: auth only. +const AI_SERVICE_URL = process.env.AI_SERVICE_URL ?? "http://localhost:8000"; export async function GET(request: Request) { const { searchParams } = new URL(request.url); @@ -14,9 +19,9 @@ export async function GET(request: Request) { } try { - const targetUrl = API_GATEWAY_URL.includes("/v1/rag") - ? `${API_GATEWAY_URL.replace(/\/query$/, "/suggest")}?q=${encodeURIComponent(q)}` - : `${API_GATEWAY_URL}/v1/rag/suggest?q=${encodeURIComponent(q)}`; + const targetUrl = AI_SERVICE_URL.includes("/v1/rag") + ? `${AI_SERVICE_URL.replace(/\/query$/, "/suggest")}?q=${encodeURIComponent(q)}` + : `${AI_SERVICE_URL}/v1/rag/suggest?q=${encodeURIComponent(q)}`; const upstream = await fetch(targetUrl, { method: "GET", diff --git a/infra/argocd/applications/medical-chatbot-app.yaml b/infra/argocd/applications/medical-chatbot-app.yaml index db1ad0c..c6f4415 100644 --- a/infra/argocd/applications/medical-chatbot-app.yaml +++ b/infra/argocd/applications/medical-chatbot-app.yaml @@ -25,6 +25,37 @@ # recovery. It does NOT restore the current image tag or the Grafana # password: set the tag afterward via `rollback-k3s.yml` (target_sha = the # last known-good commit) and re-enter the Grafana password by hand. +# +# --------------------------------------------------------------------------- +# EDITING THE INLINE VALUES BY HAND — read this first (2026-08-18 outage) +# --------------------------------------------------------------------------- +# Editing `spec.source.helm.values` through the ArgoCD UI broke this +# Application for ~16 hours. The UI saved the block as a FOLDED scalar +# (`values: >`), and a folded scalar joins consecutive same-indent lines into +# one. Three comment lines sat directly above `aiService:` at the same indent, +# so all four became a single line and `aiService:` ended up *inside* the +# comment — leaving YAML that cannot parse at all. +# +# It failed silently and misleadingly. ArgoCD kept reporting `Synced` (its +# last successful render was hours stale), the UI's PARAMETERS tab still +# showed sensible values, and the site stayed up — Kubernetes will not retire +# working pods until a replacement goes Ready, and the replacement never +# could. Meanwhile every override in the block was being ignored: the GHCR +# image repositories (so Deployments fell back to the chart-default +# `duocthu-*:local`, which exists in no registry) and `authService`/ +# `apiGateway` `enabled: true` (so neither was ever created). +# +# Therefore: +# - Do NOT put comments in the inline values. Explain things here instead; +# this file is version-controlled and no text box can mangle it. +# - Prefer the API over the UI for edits — see +# .github/scripts/repair_argocd_inline_values.py, which reads the live +# object, edits the string, and PUTs it back with the structure intact. +# - After ANY inline edit, verify with +# `.github/workflows/inspect-argocd-app.yml`: it dumps the raw string one +# escaped line at a time, which is the only view that reveals where the +# newlines actually are. The pretty-printed view looked almost correct +# throughout the outage. apiVersion: argoproj.io/v1alpha1 kind: Application metadata: