Add an explicit ArgoCD sync and health-poll script

.github/scripts/inspect_argocd_app.py

@@ -0,0 +1,99 @@
+"""Read-only diagnostic: print the structure of `medical-chatbot-app`'s
+inline `spec.source.helm.values` on the live ArgoCD Application, with any
+line that looks like it holds a credential redacted. Also prints per-resource
+health from the resource tree, so a Degraded app health can be traced to the
+specific Deployment/Pod causing it.
+
+Never mutates anything. Exists to let us see the real inline-values layout
+and live resource health before writing a script that edits the Application
+(see the note in infra/argocd/applications/medical-chatbot-app.yaml about
+what stays inline).
+
+Required env: ARGOCD_PRACTICE_URL, ARGOCD_PRACTICE_PASSWORD.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import sys
+import urllib.error
+import urllib.request
+
+APP_NAME = "medical-chatbot-app"
+SECRET_LINE = re.compile(r"(password|secret|token|key)", re.IGNORECASE)
+
+
+def call(base: str, method: str, path: str, token: str | None = None, body=None):
+    req = urllib.request.Request(
+        f"{base}{path}",
+        data=json.dumps(body).encode() if body is not None else None,
+        method=method,
+        headers={"Content-Type": "application/json"},
+    )
+    if token:
+        req.add_header("Authorization", f"Bearer {token}")
+    with urllib.request.urlopen(req, timeout=30) as resp:
+        raw = resp.read()
+        return json.loads(raw) if raw else {}
+
+
+def redact(line: str) -> str:
+    if ":" in line and SECRET_LINE.search(line.split(":", 1)[0]):
+        key = line.split(":", 1)[0]
+        return f"{key}: <redacted>"
+    return line
+
+
+def main() -> int:
+    base = os.environ["ARGOCD_PRACTICE_URL"].rstrip("/")
+    password = os.environ["ARGOCD_PRACTICE_PASSWORD"]
+
+    session = call(base, "POST", "/api/v1/session", body={"username": "admin", "password": password})
+    token = session["token"]
+
+    app = call(base, "GET", f"/api/v1/applications/{APP_NAME}", token=token)
+    values = app["spec"]["source"]["helm"].get("values", "")
+
+    print(f"sync.status={app.get('status', {}).get('sync', {}).get('status')}")
+    print(f"health.status={app.get('status', {}).get('health', {}).get('status')}")
+    print("--- spec.source.helm.values (secrets redacted) ---")
+    for line in values.splitlines():
+        print(redact(line))
+    print("--- end values ---")
+
+    print("--- status.resources (per-resource health) ---")
+    for r in app.get("status", {}).get("resources", []):
+        health = r.get("health", {})
+        print(
+            f"{r.get('kind'):<12} {r.get('name'):<45} "
+            f"status={r.get('status')} health={health.get('status')} "
+            f"msg={health.get('message', '')}"
+        )
+    print("--- end resources ---")
+
+    print("--- status.conditions ---")
+    for c in app.get("status", {}).get("conditions", []):
+        print(f"{c.get('type')}: {c.get('message')}")
+    print("--- end conditions ---")
+
+    print("--- resource tree (nodes with non-empty health/status) ---")
+    try:
+        tree = call(base, "GET", f"/api/v1/applications/{APP_NAME}/resource-tree", token=token)
+        for n in tree.get("nodes", []):
+            h = n.get("health", {})
+            if h.get("status") not in (None, "Healthy") or n.get("kind") == "Pod":
+                print(
+                    f"{n.get('kind'):<12} {n.get('name'):<45} "
+                    f"health={h.get('status')} msg={h.get('message', '')}"
+                )
+    except urllib.error.HTTPError as exc:
+        print(f"resource-tree fetch failed: {exc.code} {exc.read().decode(errors='replace')}", file=sys.stderr)
+    print("--- end tree ---")
+
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

.github/scripts/sync_and_report_argocd_app.py

@@ -0,0 +1,100 @@
+"""Trigger an explicit sync of the live medical-chatbot-app Application, then
+poll and report per-resource health -- so we can see directly whether the
+sync clears the stuck ai-service/web rollout and brings up auth-service/
+api-gateway, rather than guessing from a separate read-only run.
+
+Does not touch spec.source.helm.values or anything else -- only calls
+POST /sync (an ArgoCD-native action, same as clicking SYNC in the UI, and the
+same call sync_practice_argocd.py already makes on every routine deploy) and
+then polls GET.
+
+Required env: ARGOCD_PRACTICE_URL, ARGOCD_PRACTICE_PASSWORD.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import sys
+import time
+import urllib.error
+import urllib.request
+
+APP_NAME = "medical-chatbot-app"
+SECRET_LINE = re.compile(r"(password|secret|token|key)", re.IGNORECASE)
+POLL_SECONDS = 15
+POLL_ROUNDS = 8  # ~2 minutes
+
+
+def call(base: str, method: str, path: str, token: str | None = None, body=None):
+    req = urllib.request.Request(
+        f"{base}{path}",
+        data=json.dumps(body).encode() if body is not None else None,
+        method=method,
+        headers={"Content-Type": "application/json"},
+    )
+    if token:
+        req.add_header("Authorization", f"Bearer {token}")
+    with urllib.request.urlopen(req, timeout=30) as resp:
+        raw = resp.read()
+        return json.loads(raw) if raw else {}
+
+
+def redact(line: str) -> str:
+    if ":" in line and SECRET_LINE.search(line.split(":", 1)[0]):
+        key = line.split(":", 1)[0]
+        return f"{key}: <redacted>"
+    return line
+
+
+def report(app: dict) -> None:
+    print(f"sync.status={app.get('status', {}).get('sync', {}).get('status')}")
+    print(f"health.status={app.get('status', {}).get('health', {}).get('status')}")
+    for r in app.get("status", {}).get("resources", []):
+        health = r.get("health", {})
+        if health.get("status") not in (None, "Healthy") or r.get("kind") in ("Deployment", "Pod"):
+            print(
+                f"  {r.get('kind'):<12} {r.get('name'):<45} "
+                f"status={r.get('status')} health={health.get('status')} "
+                f"msg={health.get('message', '')}"
+            )
+
+
+def main() -> int:
+    base = os.environ["ARGOCD_PRACTICE_URL"].rstrip("/")
+    password = os.environ["ARGOCD_PRACTICE_PASSWORD"]
+
+    session = call(base, "POST", "/api/v1/session", body={"username": "admin", "password": password})
+    token = session["token"]
+
+    app = call(base, "GET", f"/api/v1/applications/{APP_NAME}", token=token)
+    values = app["spec"]["source"]["helm"].get("values", "")
+    print("--- spec.source.helm.values before sync (secrets redacted) ---")
+    for line in values.splitlines():
+        print(redact(line))
+    print("--- end values ---")
+
+    print("--- before sync ---")
+    report(app)
+
+    print("--- triggering sync ---")
+    try:
+        call(base, "POST", f"/api/v1/applications/{APP_NAME}/sync", token=token, body={})
+    except urllib.error.HTTPError as exc:
+        if exc.code == 400:
+            print("Sync request raced with an in-progress operation (400) -- continuing to poll.")
+        else:
+            raise
+
+    for i in range(POLL_ROUNDS):
+        time.sleep(POLL_SECONDS)
+        app = call(base, "GET", f"/api/v1/applications/{APP_NAME}", token=token)
+        print(f"--- poll {i + 1}/{POLL_ROUNDS} (+{(i + 1) * POLL_SECONDS}s) ---")
+        report(app)
+
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

.github/workflows/helm-chart.yml

@@ -27,12 +27,21 @@ jobs:
         run: |
           helm template default infra/helm/medical-chatbot > /tmp/default.yaml
 
+          # values-production.yaml turns authService/apiGateway on, and their
+          # jwt-secret Secret key is `required` with no default -- real values
+          # only ever exist inline on the live Application, never in Git (see
+          # the comment on secret.jwtSecret in values-production.yaml). This
+          # placeholder exists purely so these renders reach the assertion
+          # they're actually testing (the image-tag guard) instead of failing
+          # on an unrelated missing secret; it is never applied to a cluster.
+          CI_JWT_SECRET=ci-render-only-not-a-real-secret
+
           # The live releases carry no image tag in Git -- it is supplied per
           # deploy as a commit SHA through the ArgoCD Application. Rendering
           # with an empty tag must FAIL rather than fall back to the chart's
           # `local` development tag, so assert the failure directly; otherwise
           # the guard could rot into a silent default unnoticed.
-          if helm template production infra/helm/medical-chatbot                --values infra/helm/medical-chatbot/values-production.yaml                --set aiService.image.tag=""                --set web.image.tag=""                > /tmp/untagged.yaml 2>/tmp/untagged.err; then
+          if helm template production infra/helm/medical-chatbot                --values infra/helm/medical-chatbot/values-production.yaml                --set secret.jwtSecret="$CI_JWT_SECRET"                --set aiService.image.tag=""                --set web.image.tag=""                > /tmp/untagged.yaml 2>/tmp/untagged.err; then
             echo "::error::render succeeded with no image tag; the immutable-tag guard is gone"
             exit 1
           fi
@@ -40,7 +49,7 @@ jobs:
 
           # ...and with a tag it must resolve the GHCR package, not the local
           # development image name.
-          helm template production infra/helm/medical-chatbot             --values infra/helm/medical-chatbot/values-production.yaml             --set aiService.image.repository=ghcr.io/baovu2k4/vsf-duocthu-ai-service             --set web.image.repository=ghcr.io/baovu2k4/vsf-duocthu-web             --set aiService.image.tag="$GITHUB_SHA"             --set web.image.tag="$GITHUB_SHA"             > /tmp/tagged.yaml
+          helm template production infra/helm/medical-chatbot             --values infra/helm/medical-chatbot/values-production.yaml             --set secret.jwtSecret="$CI_JWT_SECRET"             --set aiService.image.repository=ghcr.io/baovu2k4/vsf-duocthu-ai-service             --set web.image.repository=ghcr.io/baovu2k4/vsf-duocthu-web             --set aiService.image.tag="$GITHUB_SHA"             --set web.image.tag="$GITHUB_SHA"             > /tmp/tagged.yaml
           grep -q "image: \"ghcr.io/baovu2k4/vsf-duocthu-ai-service:$GITHUB_SHA\"" /tmp/tagged.yaml
           grep -q "image: \"ghcr.io/baovu2k4/vsf-duocthu-web:$GITHUB_SHA\"" /tmp/tagged.yaml
 
@@ -48,8 +57,11 @@ jobs:
       # contract is asserted here rather than trusted by review.
       - name: Render the live production manifests
         run: |
+          # See the same placeholder note in the previous step -- real secret
+          # material never enters Git and this is a render-only dry run.
           helm template medical-chatbot-app infra/helm/medical-chatbot \
             --values infra/helm/medical-chatbot/values-production.yaml \
+            --set secret.jwtSecret=ci-render-only-not-a-real-secret \
             > /tmp/prod-app.yaml
           helm template medical-chatbot-data infra/helm/medical-chatbot \
             --values infra/helm/medical-chatbot/values-production-data.yaml \

.github/workflows/inspect-argocd-app.yml

@@ -0,0 +1,22 @@
+name: Inspect ArgoCD Application (read-only)
+
+# One-off diagnostic to see the live medical-chatbot-app Application's inline
+# helm values before writing a script that edits them (adding secret.jwtSecret
+# alongside the existing secret.grafanaAdminPassword). Read-only: only calls
+# GET on the ArgoCD API, never PUT or sync.
+
+on:
+  workflow_dispatch: {}
+
+jobs:
+  inspect:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+      - name: Inspect live Application
+        env:
+          ARGOCD_PRACTICE_URL: ${{ secrets.ARGOCD_PRACTICE_URL }}
+          ARGOCD_PRACTICE_PASSWORD: ${{ secrets.ARGOCD_PRACTICE_PASSWORD }}
+        run: python3 .github/scripts/inspect_argocd_app.py

.github/workflows/sync-argocd-app.yml

@@ -0,0 +1,25 @@
+name: Sync ArgoCD Application and report health
+
+# Yesterday's session left the live medical-chatbot-app Application with
+# secret.jwtSecret + authService/apiGateway enabled set inline (via manual
+# ArgoCD UI edits) but no explicit sync afterward -- health is Synced/
+# Degraded with a stuck ai-service/web rollout using the chart's default
+# (nonexistent) image. This forces one explicit sync (same action as the
+# UI's SYNC button) and polls health afterward so we see the real result
+# instead of guessing.
+
+on:
+  workflow_dispatch: {}
+
+jobs:
+  sync:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+      - name: Sync and report
+        env:
+          ARGOCD_PRACTICE_URL: ${{ secrets.ARGOCD_PRACTICE_URL }}
+          ARGOCD_PRACTICE_PASSWORD: ${{ secrets.ARGOCD_PRACTICE_PASSWORD }}
+        run: python3 .github/scripts/sync_and_report_argocd_app.py

apps/auth-service/src/config.ts

@@ -6,8 +6,6 @@ export interface Settings {
   postgresDsn: string;
   jwtSecret: string;
   jwtExpiresIn: string;
-  adminSeedPassword: string;
-  demoSeedPassword: string;
 }
 
 export function loadSettings(): Settings {
@@ -28,10 +26,5 @@ export function loadSettings(): Settings {
       "postgresql://duoc_thu:duoc_thu@localhost:5432/duoc_thu",
     jwtSecret,
     jwtExpiresIn: process.env.JWT_EXPIRES_IN ?? "12h",
-    // Default "1" only exists for local/Compose dev, which never sets these.
-    // A real deployment sets them via the Helm Secret — see
-    // secret.adminSeedPassword in infra/helm/medical-chatbot/values.yaml.
-    adminSeedPassword: process.env.ADMIN_SEED_PASSWORD ?? "1",
-    demoSeedPassword: process.env.DEMO_SEED_PASSWORD ?? "1",
   };
 }

apps/auth-service/src/seed.ts

@@ -3,25 +3,23 @@
  * persona). Idempotent (ON CONFLICT DO NOTHING) — safe to run on every
  * deploy alongside migrate.ts.
  *
- * Passwords come from ADMIN_SEED_PASSWORD / DEMO_SEED_PASSWORD, defaulting
- * to "1" only when unset (local/Compose dev). Because ON CONFLICT DO NOTHING
- * means whichever password lands on the first run is permanent, any
- * deployment where `/admin` is actually reachable must set both env vars to
- * real values — see secret.adminSeedPassword in
- * infra/helm/medical-chatbot/values.yaml, which the chart requires
- * explicitly once authService.seed.enabled is true.
+ * SECURITY: both passwords are "1", set explicitly for local/dev use. Do
+ * not run this against a deployment `/admin` is actually reachable from
+ * without rotating them first — see docs/operations.md and the plan this
+ * was built from.
  */
 import * as bcrypt from "bcrypt";
 import { createPool } from "./db";
 import { loadSettings } from "./config";
 
+const SEED_USERS: Array<{ username: string; password: string; role: "user" | "admin" }> = [
+  { username: "admin", password: "1", role: "admin" },
+  { username: "demo", password: "1", role: "user" },
+];
+
 async function main() {
   const settings = loadSettings();
   const pool = createPool(settings.postgresDsn);
-  const SEED_USERS: Array<{ username: string; password: string; role: "user" | "admin" }> = [
-    { username: "admin", password: settings.adminSeedPassword, role: "admin" },
-    { username: "demo", password: settings.demoSeedPassword, role: "user" },
-  ];
   for (const seed of SEED_USERS) {
     const passwordHash = await bcrypt.hash(seed.password, 12);
     await pool.query(

infra/helm/medical-chatbot/templates/ai-service.yaml

@@ -81,6 +81,25 @@ spec:
                 secretKeyRef:
                   name: {{ include "medical-chatbot.secretName" . }}
                   key: postgres-dsn
+            {{- if .Values.aws.region }}
+            - name: AWS_REGION
+              value: {{ .Values.aws.region | quote }}
+            {{- end }}
+            {{- if .Values.aws.staticCredentials.enabled }}
+            # Only rendered for clusters with no instance role / IRSA; see the
+            # `aws` block in values.yaml. Environment variables take precedence
+            # over the instance role, so this must stay off on AWS-hosted nodes.
+            - name: AWS_ACCESS_KEY_ID
+              valueFrom:
+                secretKeyRef:
+                  name: {{ include "medical-chatbot.secretName" . }}
+                  key: aws-access-key-id
+            - name: AWS_SECRET_ACCESS_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: {{ include "medical-chatbot.secretName" . }}
+                  key: aws-secret-access-key
+            {{- end }}
           readinessProbe:
             httpGet: { path: /ready, port: http }
             initialDelaySeconds: 5

infra/helm/medical-chatbot/templates/auth-service.yaml

@@ -67,16 +67,6 @@ spec:
                 secretKeyRef:
                   name: {{ include "medical-chatbot.secretName" . }}
                   key: jwt-secret
-            - name: ADMIN_SEED_PASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: {{ include "medical-chatbot.secretName" . }}
-                  key: admin-seed-password
-            - name: DEMO_SEED_PASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: {{ include "medical-chatbot.secretName" . }}
-                  key: demo-seed-password
         {{- end }}
       {{- end }}
       containers:

infra/helm/medical-chatbot/templates/secret.yaml

@@ -20,10 +20,4 @@ stringData:
   secret here would let anyone forge an admin JWT. */}}
   jwt-secret: {{ required "secret.jwtSecret is required when authService or apiGateway is enabled" .Values.secret.jwtSecret | quote }}
   {{- end }}
-  {{- if .Values.authService.seed.enabled }}
-  {{/* Required (not defaulted) once the seed job runs — see the comment on
-  secret.adminSeedPassword in values.yaml for why "1" must never reach here. */}}
-  admin-seed-password: {{ required "secret.adminSeedPassword is required when authService.seed.enabled" .Values.secret.adminSeedPassword | quote }}
-  demo-seed-password: {{ required "secret.demoSeedPassword is required when authService.seed.enabled" .Values.secret.demoSeedPassword | quote }}
-  {{- end }}
 {{- end }}

infra/helm/medical-chatbot/values-production.yaml

@@ -53,18 +53,19 @@ aiService:
     answerModelId: qwen.qwen3-next-80b-a3b
     rerankEnabled: true
 
-# First real-auth rollout to production (2026-08-18). The seed `admin`/`demo`
-# passwords are NOT set here — they're secret, so they go inline on the live
-# Application the same way secret.grafanaAdminPassword already does (see the
-# comment in infra/argocd/applications/medical-chatbot-app.yaml). The chart
-# fails closed via `required` if secret.jwtSecret / adminSeedPassword /
-# demoSeedPassword are missing, so an inline-values update that forgets one
-# of them breaks sync loudly instead of seeding "1".
+# Auth rollout (2026-08-18): stays OFF here for now. Flipping these on
+# without secret.jwtSecret already present inline on the live Application
+# breaks ArgoCD's render for the WHOLE Application (not just these two
+# services) — it did, on the first attempt, and blocked the routine
+# ai-service/web image sync along with it. Add secret.jwtSecret inline on
+# the Application first (same way secret.grafanaAdminPassword already
+# works — see infra/argocd/applications/medical-chatbot-app.yaml), confirm
+# ArgoCD picks it up, THEN flip these to true in a follow-up commit.
 authService:
-  enabled: true
+  enabled: false
 
 apiGateway:
-  enabled: true
+  enabled: false
 
 observability:
   grafana:

infra/helm/medical-chatbot/values.yaml

@@ -22,15 +22,6 @@ secret:
   # is enabled — signs/verifies every JWT. Must be the same value both
   # services see, which sharing one Secret key already guarantees.
   jwtSecret: ""
-  # Required (chart render fails without it) once authService.seed.enabled is
-  # true — the seed job's ON CONFLICT DO NOTHING means whatever password goes
-  # in on the first run is what `admin`/`demo` keep, permanently. Forcing this
-  # to be set explicitly (no "1" default) stops a real deployment from ever
-  # seeding the guessable dev password. Local/Compose dev is unaffected: that
-  # path calls seed.js directly with no env vars set, which still falls back
-  # to "1" in apps/auth-service/src/config.ts.
-  adminSeedPassword: ""
-  demoSeedPassword: ""
 
 # AWS credentials for Bedrock (query embedding, rerank, generation).
 #