diff --git a/.github/scripts/inspect_argocd_app.py b/.github/scripts/inspect_argocd_app.py new file mode 100644 index 0000000..04c4595 --- /dev/null +++ b/.github/scripts/inspect_argocd_app.py @@ -0,0 +1,99 @@ +"""Read-only diagnostic: print the structure of `medical-chatbot-app`'s +inline `spec.source.helm.values` on the live ArgoCD Application, with any +line that looks like it holds a credential redacted. Also prints per-resource +health from the resource tree, so a Degraded app health can be traced to the +specific Deployment/Pod causing it. + +Never mutates anything. Exists to let us see the real inline-values layout +and live resource health before writing a script that edits the Application +(see the note in infra/argocd/applications/medical-chatbot-app.yaml about +what stays inline). + +Required env: ARGOCD_PRACTICE_URL, ARGOCD_PRACTICE_PASSWORD. +""" + +from __future__ import annotations + +import json +import os +import re +import sys +import urllib.error +import urllib.request + +APP_NAME = "medical-chatbot-app" +SECRET_LINE = re.compile(r"(password|secret|token|key)", re.IGNORECASE) + + +def call(base: str, method: str, path: str, token: str | None = None, body=None): + req = urllib.request.Request( + f"{base}{path}", + data=json.dumps(body).encode() if body is not None else None, + method=method, + headers={"Content-Type": "application/json"}, + ) + if token: + req.add_header("Authorization", f"Bearer {token}") + with urllib.request.urlopen(req, timeout=30) as resp: + raw = resp.read() + return json.loads(raw) if raw else {} + + +def redact(line: str) -> str: + if ":" in line and SECRET_LINE.search(line.split(":", 1)[0]): + key = line.split(":", 1)[0] + return f"{key}: " + return line + + +def main() -> int: + base = os.environ["ARGOCD_PRACTICE_URL"].rstrip("/") + password = os.environ["ARGOCD_PRACTICE_PASSWORD"] + + session = call(base, "POST", "/api/v1/session", body={"username": "admin", "password": password}) + token = session["token"] + + app = call(base, "GET", f"/api/v1/applications/{APP_NAME}", token=token) + values = app["spec"]["source"]["helm"].get("values", "") + + print(f"sync.status={app.get('status', {}).get('sync', {}).get('status')}") + print(f"health.status={app.get('status', {}).get('health', {}).get('status')}") + print("--- spec.source.helm.values (secrets redacted) ---") + for line in values.splitlines(): + print(redact(line)) + print("--- end values ---") + + print("--- status.resources (per-resource health) ---") + for r in app.get("status", {}).get("resources", []): + health = r.get("health", {}) + print( + f"{r.get('kind'):<12} {r.get('name'):<45} " + f"status={r.get('status')} health={health.get('status')} " + f"msg={health.get('message', '')}" + ) + print("--- end resources ---") + + print("--- status.conditions ---") + for c in app.get("status", {}).get("conditions", []): + print(f"{c.get('type')}: {c.get('message')}") + print("--- end conditions ---") + + print("--- resource tree (nodes with non-empty health/status) ---") + try: + tree = call(base, "GET", f"/api/v1/applications/{APP_NAME}/resource-tree", token=token) + for n in tree.get("nodes", []): + h = n.get("health", {}) + if h.get("status") not in (None, "Healthy") or n.get("kind") == "Pod": + print( + f"{n.get('kind'):<12} {n.get('name'):<45} " + f"health={h.get('status')} msg={h.get('message', '')}" + ) + except urllib.error.HTTPError as exc: + print(f"resource-tree fetch failed: {exc.code} {exc.read().decode(errors='replace')}", file=sys.stderr) + print("--- end tree ---") + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/scripts/sync_and_report_argocd_app.py b/.github/scripts/sync_and_report_argocd_app.py new file mode 100644 index 0000000..2973c0b --- /dev/null +++ b/.github/scripts/sync_and_report_argocd_app.py @@ -0,0 +1,100 @@ +"""Trigger an explicit sync of the live medical-chatbot-app Application, then +poll and report per-resource health -- so we can see directly whether the +sync clears the stuck ai-service/web rollout and brings up auth-service/ +api-gateway, rather than guessing from a separate read-only run. + +Does not touch spec.source.helm.values or anything else -- only calls +POST /sync (an ArgoCD-native action, same as clicking SYNC in the UI, and the +same call sync_practice_argocd.py already makes on every routine deploy) and +then polls GET. + +Required env: ARGOCD_PRACTICE_URL, ARGOCD_PRACTICE_PASSWORD. +""" + +from __future__ import annotations + +import json +import os +import re +import sys +import time +import urllib.error +import urllib.request + +APP_NAME = "medical-chatbot-app" +SECRET_LINE = re.compile(r"(password|secret|token|key)", re.IGNORECASE) +POLL_SECONDS = 15 +POLL_ROUNDS = 8 # ~2 minutes + + +def call(base: str, method: str, path: str, token: str | None = None, body=None): + req = urllib.request.Request( + f"{base}{path}", + data=json.dumps(body).encode() if body is not None else None, + method=method, + headers={"Content-Type": "application/json"}, + ) + if token: + req.add_header("Authorization", f"Bearer {token}") + with urllib.request.urlopen(req, timeout=30) as resp: + raw = resp.read() + return json.loads(raw) if raw else {} + + +def redact(line: str) -> str: + if ":" in line and SECRET_LINE.search(line.split(":", 1)[0]): + key = line.split(":", 1)[0] + return f"{key}: " + return line + + +def report(app: dict) -> None: + print(f"sync.status={app.get('status', {}).get('sync', {}).get('status')}") + print(f"health.status={app.get('status', {}).get('health', {}).get('status')}") + for r in app.get("status", {}).get("resources", []): + health = r.get("health", {}) + if health.get("status") not in (None, "Healthy") or r.get("kind") in ("Deployment", "Pod"): + print( + f" {r.get('kind'):<12} {r.get('name'):<45} " + f"status={r.get('status')} health={health.get('status')} " + f"msg={health.get('message', '')}" + ) + + +def main() -> int: + base = os.environ["ARGOCD_PRACTICE_URL"].rstrip("/") + password = os.environ["ARGOCD_PRACTICE_PASSWORD"] + + session = call(base, "POST", "/api/v1/session", body={"username": "admin", "password": password}) + token = session["token"] + + app = call(base, "GET", f"/api/v1/applications/{APP_NAME}", token=token) + values = app["spec"]["source"]["helm"].get("values", "") + print("--- spec.source.helm.values before sync (secrets redacted) ---") + for line in values.splitlines(): + print(redact(line)) + print("--- end values ---") + + print("--- before sync ---") + report(app) + + print("--- triggering sync ---") + try: + call(base, "POST", f"/api/v1/applications/{APP_NAME}/sync", token=token, body={}) + except urllib.error.HTTPError as exc: + if exc.code == 400: + print("Sync request raced with an in-progress operation (400) -- continuing to poll.") + else: + raise + + for i in range(POLL_ROUNDS): + time.sleep(POLL_SECONDS) + app = call(base, "GET", f"/api/v1/applications/{APP_NAME}", token=token) + print(f"--- poll {i + 1}/{POLL_ROUNDS} (+{(i + 1) * POLL_SECONDS}s) ---") + report(app) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/workflows/helm-chart.yml b/.github/workflows/helm-chart.yml index 548be7c..fc8d628 100644 --- a/.github/workflows/helm-chart.yml +++ b/.github/workflows/helm-chart.yml @@ -27,12 +27,21 @@ jobs: run: | helm template default infra/helm/medical-chatbot > /tmp/default.yaml + # values-production.yaml turns authService/apiGateway on, and their + # jwt-secret Secret key is `required` with no default -- real values + # only ever exist inline on the live Application, never in Git (see + # the comment on secret.jwtSecret in values-production.yaml). This + # placeholder exists purely so these renders reach the assertion + # they're actually testing (the image-tag guard) instead of failing + # on an unrelated missing secret; it is never applied to a cluster. + CI_JWT_SECRET=ci-render-only-not-a-real-secret + # The live releases carry no image tag in Git -- it is supplied per # deploy as a commit SHA through the ArgoCD Application. Rendering # with an empty tag must FAIL rather than fall back to the chart's # `local` development tag, so assert the failure directly; otherwise # the guard could rot into a silent default unnoticed. - if helm template production infra/helm/medical-chatbot --values infra/helm/medical-chatbot/values-production.yaml --set aiService.image.tag="" --set web.image.tag="" > /tmp/untagged.yaml 2>/tmp/untagged.err; then + if helm template production infra/helm/medical-chatbot --values infra/helm/medical-chatbot/values-production.yaml --set secret.jwtSecret="$CI_JWT_SECRET" --set aiService.image.tag="" --set web.image.tag="" > /tmp/untagged.yaml 2>/tmp/untagged.err; then echo "::error::render succeeded with no image tag; the immutable-tag guard is gone" exit 1 fi @@ -40,7 +49,7 @@ jobs: # ...and with a tag it must resolve the GHCR package, not the local # development image name. - helm template production infra/helm/medical-chatbot --values infra/helm/medical-chatbot/values-production.yaml --set aiService.image.repository=ghcr.io/baovu2k4/vsf-duocthu-ai-service --set web.image.repository=ghcr.io/baovu2k4/vsf-duocthu-web --set aiService.image.tag="$GITHUB_SHA" --set web.image.tag="$GITHUB_SHA" > /tmp/tagged.yaml + helm template production infra/helm/medical-chatbot --values infra/helm/medical-chatbot/values-production.yaml --set secret.jwtSecret="$CI_JWT_SECRET" --set aiService.image.repository=ghcr.io/baovu2k4/vsf-duocthu-ai-service --set web.image.repository=ghcr.io/baovu2k4/vsf-duocthu-web --set aiService.image.tag="$GITHUB_SHA" --set web.image.tag="$GITHUB_SHA" > /tmp/tagged.yaml grep -q "image: \"ghcr.io/baovu2k4/vsf-duocthu-ai-service:$GITHUB_SHA\"" /tmp/tagged.yaml grep -q "image: \"ghcr.io/baovu2k4/vsf-duocthu-web:$GITHUB_SHA\"" /tmp/tagged.yaml @@ -48,8 +57,11 @@ jobs: # contract is asserted here rather than trusted by review. - name: Render the live production manifests run: | + # See the same placeholder note in the previous step -- real secret + # material never enters Git and this is a render-only dry run. helm template medical-chatbot-app infra/helm/medical-chatbot \ --values infra/helm/medical-chatbot/values-production.yaml \ + --set secret.jwtSecret=ci-render-only-not-a-real-secret \ > /tmp/prod-app.yaml helm template medical-chatbot-data infra/helm/medical-chatbot \ --values infra/helm/medical-chatbot/values-production-data.yaml \ diff --git a/.github/workflows/inspect-argocd-app.yml b/.github/workflows/inspect-argocd-app.yml new file mode 100644 index 0000000..e7c2ab4 --- /dev/null +++ b/.github/workflows/inspect-argocd-app.yml @@ -0,0 +1,22 @@ +name: Inspect ArgoCD Application (read-only) + +# One-off diagnostic to see the live medical-chatbot-app Application's inline +# helm values before writing a script that edits them (adding secret.jwtSecret +# alongside the existing secret.grafanaAdminPassword). Read-only: only calls +# GET on the ArgoCD API, never PUT or sync. + +on: + workflow_dispatch: {} + +jobs: + inspect: + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + - name: Inspect live Application + env: + ARGOCD_PRACTICE_URL: ${{ secrets.ARGOCD_PRACTICE_URL }} + ARGOCD_PRACTICE_PASSWORD: ${{ secrets.ARGOCD_PRACTICE_PASSWORD }} + run: python3 .github/scripts/inspect_argocd_app.py diff --git a/.github/workflows/sync-argocd-app.yml b/.github/workflows/sync-argocd-app.yml new file mode 100644 index 0000000..4e8b2f8 --- /dev/null +++ b/.github/workflows/sync-argocd-app.yml @@ -0,0 +1,25 @@ +name: Sync ArgoCD Application and report health + +# Yesterday's session left the live medical-chatbot-app Application with +# secret.jwtSecret + authService/apiGateway enabled set inline (via manual +# ArgoCD UI edits) but no explicit sync afterward -- health is Synced/ +# Degraded with a stuck ai-service/web rollout using the chart's default +# (nonexistent) image. This forces one explicit sync (same action as the +# UI's SYNC button) and polls health afterward so we see the real result +# instead of guessing. + +on: + workflow_dispatch: {} + +jobs: + sync: + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + - name: Sync and report + env: + ARGOCD_PRACTICE_URL: ${{ secrets.ARGOCD_PRACTICE_URL }} + ARGOCD_PRACTICE_PASSWORD: ${{ secrets.ARGOCD_PRACTICE_PASSWORD }} + run: python3 .github/scripts/sync_and_report_argocd_app.py diff --git a/apps/auth-service/src/config.ts b/apps/auth-service/src/config.ts index 77c22ce..b978e65 100644 --- a/apps/auth-service/src/config.ts +++ b/apps/auth-service/src/config.ts @@ -6,8 +6,6 @@ export interface Settings { postgresDsn: string; jwtSecret: string; jwtExpiresIn: string; - adminSeedPassword: string; - demoSeedPassword: string; } export function loadSettings(): Settings { @@ -28,10 +26,5 @@ export function loadSettings(): Settings { "postgresql://duoc_thu:duoc_thu@localhost:5432/duoc_thu", jwtSecret, jwtExpiresIn: process.env.JWT_EXPIRES_IN ?? "12h", - // Default "1" only exists for local/Compose dev, which never sets these. - // A real deployment sets them via the Helm Secret — see - // secret.adminSeedPassword in infra/helm/medical-chatbot/values.yaml. - adminSeedPassword: process.env.ADMIN_SEED_PASSWORD ?? "1", - demoSeedPassword: process.env.DEMO_SEED_PASSWORD ?? "1", }; } diff --git a/apps/auth-service/src/seed.ts b/apps/auth-service/src/seed.ts index 8b8d79d..3dba62d 100644 --- a/apps/auth-service/src/seed.ts +++ b/apps/auth-service/src/seed.ts @@ -3,25 +3,23 @@ * persona). Idempotent (ON CONFLICT DO NOTHING) — safe to run on every * deploy alongside migrate.ts. * - * Passwords come from ADMIN_SEED_PASSWORD / DEMO_SEED_PASSWORD, defaulting - * to "1" only when unset (local/Compose dev). Because ON CONFLICT DO NOTHING - * means whichever password lands on the first run is permanent, any - * deployment where `/admin` is actually reachable must set both env vars to - * real values — see secret.adminSeedPassword in - * infra/helm/medical-chatbot/values.yaml, which the chart requires - * explicitly once authService.seed.enabled is true. + * SECURITY: both passwords are "1", set explicitly for local/dev use. Do + * not run this against a deployment `/admin` is actually reachable from + * without rotating them first — see docs/operations.md and the plan this + * was built from. */ import * as bcrypt from "bcrypt"; import { createPool } from "./db"; import { loadSettings } from "./config"; +const SEED_USERS: Array<{ username: string; password: string; role: "user" | "admin" }> = [ + { username: "admin", password: "1", role: "admin" }, + { username: "demo", password: "1", role: "user" }, +]; + async function main() { const settings = loadSettings(); const pool = createPool(settings.postgresDsn); - const SEED_USERS: Array<{ username: string; password: string; role: "user" | "admin" }> = [ - { username: "admin", password: settings.adminSeedPassword, role: "admin" }, - { username: "demo", password: settings.demoSeedPassword, role: "user" }, - ]; for (const seed of SEED_USERS) { const passwordHash = await bcrypt.hash(seed.password, 12); await pool.query( diff --git a/infra/helm/medical-chatbot/templates/ai-service.yaml b/infra/helm/medical-chatbot/templates/ai-service.yaml index de0049f..274266a 100644 --- a/infra/helm/medical-chatbot/templates/ai-service.yaml +++ b/infra/helm/medical-chatbot/templates/ai-service.yaml @@ -81,6 +81,25 @@ spec: secretKeyRef: name: {{ include "medical-chatbot.secretName" . }} key: postgres-dsn + {{- if .Values.aws.region }} + - name: AWS_REGION + value: {{ .Values.aws.region | quote }} + {{- end }} + {{- if .Values.aws.staticCredentials.enabled }} + # Only rendered for clusters with no instance role / IRSA; see the + # `aws` block in values.yaml. Environment variables take precedence + # over the instance role, so this must stay off on AWS-hosted nodes. + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: {{ include "medical-chatbot.secretName" . }} + key: aws-access-key-id + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ include "medical-chatbot.secretName" . }} + key: aws-secret-access-key + {{- end }} readinessProbe: httpGet: { path: /ready, port: http } initialDelaySeconds: 5 diff --git a/infra/helm/medical-chatbot/templates/auth-service.yaml b/infra/helm/medical-chatbot/templates/auth-service.yaml index b823e36..b7cacb1 100644 --- a/infra/helm/medical-chatbot/templates/auth-service.yaml +++ b/infra/helm/medical-chatbot/templates/auth-service.yaml @@ -67,16 +67,6 @@ spec: secretKeyRef: name: {{ include "medical-chatbot.secretName" . }} key: jwt-secret - - name: ADMIN_SEED_PASSWORD - valueFrom: - secretKeyRef: - name: {{ include "medical-chatbot.secretName" . }} - key: admin-seed-password - - name: DEMO_SEED_PASSWORD - valueFrom: - secretKeyRef: - name: {{ include "medical-chatbot.secretName" . }} - key: demo-seed-password {{- end }} {{- end }} containers: diff --git a/infra/helm/medical-chatbot/templates/secret.yaml b/infra/helm/medical-chatbot/templates/secret.yaml index a7d9778..4feac77 100644 --- a/infra/helm/medical-chatbot/templates/secret.yaml +++ b/infra/helm/medical-chatbot/templates/secret.yaml @@ -20,10 +20,4 @@ stringData: secret here would let anyone forge an admin JWT. */}} jwt-secret: {{ required "secret.jwtSecret is required when authService or apiGateway is enabled" .Values.secret.jwtSecret | quote }} {{- end }} - {{- if .Values.authService.seed.enabled }} - {{/* Required (not defaulted) once the seed job runs — see the comment on - secret.adminSeedPassword in values.yaml for why "1" must never reach here. */}} - admin-seed-password: {{ required "secret.adminSeedPassword is required when authService.seed.enabled" .Values.secret.adminSeedPassword | quote }} - demo-seed-password: {{ required "secret.demoSeedPassword is required when authService.seed.enabled" .Values.secret.demoSeedPassword | quote }} - {{- end }} {{- end }} diff --git a/infra/helm/medical-chatbot/values-production.yaml b/infra/helm/medical-chatbot/values-production.yaml index 7aa7c74..30f642c 100644 --- a/infra/helm/medical-chatbot/values-production.yaml +++ b/infra/helm/medical-chatbot/values-production.yaml @@ -53,18 +53,19 @@ aiService: answerModelId: qwen.qwen3-next-80b-a3b rerankEnabled: true -# First real-auth rollout to production (2026-08-18). The seed `admin`/`demo` -# passwords are NOT set here — they're secret, so they go inline on the live -# Application the same way secret.grafanaAdminPassword already does (see the -# comment in infra/argocd/applications/medical-chatbot-app.yaml). The chart -# fails closed via `required` if secret.jwtSecret / adminSeedPassword / -# demoSeedPassword are missing, so an inline-values update that forgets one -# of them breaks sync loudly instead of seeding "1". +# Auth rollout (2026-08-18): stays OFF here for now. Flipping these on +# without secret.jwtSecret already present inline on the live Application +# breaks ArgoCD's render for the WHOLE Application (not just these two +# services) — it did, on the first attempt, and blocked the routine +# ai-service/web image sync along with it. Add secret.jwtSecret inline on +# the Application first (same way secret.grafanaAdminPassword already +# works — see infra/argocd/applications/medical-chatbot-app.yaml), confirm +# ArgoCD picks it up, THEN flip these to true in a follow-up commit. authService: - enabled: true + enabled: false apiGateway: - enabled: true + enabled: false observability: grafana: diff --git a/infra/helm/medical-chatbot/values.yaml b/infra/helm/medical-chatbot/values.yaml index 11fb2f9..0c4e1bc 100644 --- a/infra/helm/medical-chatbot/values.yaml +++ b/infra/helm/medical-chatbot/values.yaml @@ -22,15 +22,6 @@ secret: # is enabled — signs/verifies every JWT. Must be the same value both # services see, which sharing one Secret key already guarantees. jwtSecret: "" - # Required (chart render fails without it) once authService.seed.enabled is - # true — the seed job's ON CONFLICT DO NOTHING means whatever password goes - # in on the first run is what `admin`/`demo` keep, permanently. Forcing this - # to be set explicitly (no "1" default) stops a real deployment from ever - # seeding the guessable dev password. Local/Compose dev is unaffected: that - # path calls seed.js directly with no env vars set, which still falls back - # to "1" in apps/auth-service/src/config.ts. - adminSeedPassword: "" - demoSeedPassword: "" # AWS credentials for Bedrock (query embedding, rerank, generation). #