From 332cb657ee8344a6a9a9ac6848fe66180b3ab629 Mon Sep 17 00:00:00 2001 From: BaoVu2k4 Date: Mon, 17 Aug 2026 15:17:12 +0700 Subject: [PATCH] Stop serving Grafana as anonymous admin on the public hostname --- .github/workflows/helm-chart.yml | 25 +++ ...D_PRODUCTION_MIGRATION_STATE_2026-08-17.md | 199 +++++++++++++++++- .../medical-chatbot/templates/ingress.yaml | 24 ++- .../templates/observability-workloads.yaml | 14 +- .../helm/medical-chatbot/values-practice.yaml | 27 ++- infra/helm/medical-chatbot/values.yaml | 12 +- 6 files changed, 277 insertions(+), 24 deletions(-) diff --git a/.github/workflows/helm-chart.yml b/.github/workflows/helm-chart.yml index f3760fa..5890c7d 100644 --- a/.github/workflows/helm-chart.yml +++ b/.github/workflows/helm-chart.yml @@ -93,6 +93,31 @@ jobs: expect /tmp/practice-app.yaml 'checksum/runtime-config:' expect /tmp/practice-app.yaml '- host: "readytochat.realvuxbaro.me"' + # The production hostname now lives on this cluster, routed and with + # its own certificate secret -- kept separate from the rehearsal + # hostname's so one renewal failure cannot take both names offline. + expect /tmp/practice-app.yaml '- host: "realvuxbaro.me"' + expect /tmp/practice-app.yaml 'secretName: realvuxbaro-tls' + expect /tmp/practice-app.yaml 'secretName: readytochat-tls' + + # Grafana answers on that same public hostname. Anonymous access may + # be open, but never as Admin, never with the login form disabled, + # and its root URL must be the name users actually arrive on. + expect /tmp/practice-app.yaml 'value: "https://realvuxbaro.me/grafana/"' + refute /tmp/practice-app.yaml 'value: "Admin"' + + # grep is line-oriented, so read the value on the line after each + # flag rather than trying to match the pair as one pattern. + for check in "GF_AUTH_ANONYMOUS_ORG_ROLE:Viewer" "GF_AUTH_DISABLE_LOGIN_FORM:false"; do + flag=${check%%:*} + want=${check#*:} + got=$(grep -A1 -- "$flag" /tmp/practice-app.yaml | grep -- 'value:' | tr -d ' "' | cut -d: -f2) + if [ "$got" != "$want" ]; then + echo "::error::$flag rendered as '$got', expected '$want'" + exit 1 + fi + done + # The app release must own neither data StatefulSet: PostgreSQL and # Qdrant belong to the data release, so an app-side sync failure or # prune can never delete the corpus or the query history. Only those diff --git a/coordination/ARGOCD_PRODUCTION_MIGRATION_STATE_2026-08-17.md b/coordination/ARGOCD_PRODUCTION_MIGRATION_STATE_2026-08-17.md index aec2064..07d086d 100644 --- a/coordination/ARGOCD_PRODUCTION_MIGRATION_STATE_2026-08-17.md +++ b/coordination/ARGOCD_PRODUCTION_MIGRATION_STATE_2026-08-17.md @@ -294,10 +294,183 @@ pre-refactor baseline**. A live query returned `answerable` with one citation and character-identical content on both environments (11.36 s practice, 11.66 s production). +## Corpus content verified identical (Claude, 2026-08-17 afternoon) + +Owner ruled PostgreSQL history/feedback out of scope: the deployment has no +real users yet, so that state does not need migrating. Qdrant content, the +ingress path and rollback were called out as the things that do need checking. + +`scripts/qdrant_fingerprint.py` walks every point and hashes payload and +vectors separately, sorting per-point digests so scroll order cannot affect +the result. The same file runs on both sides — shipped to production +base64-encoded by `audit-qdrant-corpus.yml` — so the two runs are directly +comparable rather than two reimplementations. + +| Field | Production | Practice | +| --- | --- | --- | +| `payload_hash` | `f8c364ee…59b1a9` | `f8c364ee…59b1a9` | +| `vector_hash` | `5cc240ea…9bec13d` | `5cc240ea…9bec13d` | +| Points scrolled | 15,100 | 15,100 | +| Distinct `drug_id` | 684 | 684 | +| Distinct `section_key` | 19 | 19 | +| Points missing a vector | 0 | 0 | +| Vector config | Cosine / 1024 | Cosine / 1024 | + +Both hashes match exactly. The two collections hold the same text embedded by +the same model — not merely the same number of points. Production run +`32006244220`. This closes the corpus half of the data gate; no Qdrant +snapshot/restore is required for cutover. + +Engine versions do **not** match in the same way, and this is a new finding: +practice pins `qdrant/qdrant:v1.19.0` (confirmed running 1.19.0), while +production runs the mutable tag `qdrant/qdrant:latest`, pulled 12 days ago. +Production's engine can therefore change under it on any pull. Pin production +to an explicit version before cutover. + +## Ingress, TLS and DNS (Claude, 2026-08-17 afternoon) + +Current state, read from the live cluster and public DNS: + +- `letsencrypt-prod` ClusterIssuer is Ready and solves **HTTP-01** only + (`{"http01":{"ingress":{"ingressClassName":"traefik"}}}`). +- Live certificates are Ready for `readytochat.realvuxbaro.me` and + `argocd.realvuxbaro.me`. There is none for `realvuxbaro.me`. +- The k3s Ingress serves `readytochat.realvuxbaro.me` only; `realvuxbaro.me` + has no rule on the cluster at all. +- DNS: `realvuxbaro.me` → `52.0.158.61` (Compose production), + `readytochat`/`argocd` → `44.206.194.195` (k3s). All at **TTL 1799 s**. +- Authoritative nameservers are Namecheap (`dns1.registrar-servers.com`), not + Route 53, so a cert-manager DNS-01 solver would need a third-party webhook + and Namecheap API credentials. + +The blocking consequence: **HTTP-01 cannot issue a certificate for +`realvuxbaro.me` until that name already resolves to the k3s node.** Adding the +Ingress rule ahead of time does not help — the challenge would keep failing. +Cutting DNS over first therefore exposes users to TLS errors for as long as +issuance takes, on top of propagation. + +Two ways to close it, in preference order: + +1. Pre-seed the `realvuxbaro.me` TLS Secret in the cluster from the certificate + Caddy already holds on the Compose host, so the k3s Ingress serves a valid + chain from the first request. Let cert-manager take over renewal once DNS + points at k3s. This gives no TLS gap, but moves a private key between hosts + and must be done in one automated step that never prints it. +2. Accept a short gap: create the Ingress rule and Certificate, switch DNS, and + let HTTP-01 succeed once records propagate. Simpler, but users who land on + k3s before issuance completes see a browser TLS warning. + +TTL is also the rollback clock. At 1799 s a DNS revert takes up to ~30 minutes +to reach cached clients, in both directions. Lower it to 60 s and wait out the +old TTL **before** the cutover window, otherwise the rollback path is far +slower than the failure it is meant to cover. + +Rollback itself remains sound and was re-confirmed: Compose production is +untouched and still serving, so reverting the A record is the whole procedure. +Do not stop that instance until the acceptance window closes, and never +terminate it — its EBS root volume is `DeleteOnTermination=true`. + +## Operational note — practice SSH access + +The practice security group scopes ports 22 and 6443 to single IPs, and this +workstation has multiple egress paths: `api.ipify.org`, `ifconfig.me` and +`icanhazip.com` each reported a *different* address (`202.60.105.126`, +`101.99.23.84`, `103.238.70.200`). SSH actually egresses via **101.99.23.84**. +Claude added port-22 rules for `202.60.105.126/32` (unused, safe to delete) and +`101.99.23.84/32` on `sg-018fc3cde8282f26d`, both described as removable. Port +6443 was left closed; everything was done over SSH plus `k3s kubectl`. + +## Cutover preparation completed (Claude, 2026-08-17 afternoon) + +Owner authorised acting on Namecheap directly. Two prerequisites are now done +and verified, with the DNS switch itself still pending approval. + +**TTL lowered.** The `realvuxbaro.me` A record moved from `Automatic` (1799 s) +to 1 minute. Only the TTL cell changed; type, host and value (`52.0.158.61`) +are untouched, and no other record was edited. Confirmed both in the Namecheap +panel and from a public resolver, which now returns TTL 60. Resolvers that had +already cached the record keep the old value until the original 1799 s expires, +so allow roughly 30 minutes from the change before relying on fast rollback. + +**Route staged ahead of DNS.** The chart gained `ingress.extraHosts`, and the +practice release now serves `realvuxbaro.me` alongside +`readytochat.realvuxbaro.me`. Verified on the live cluster: the Ingress lists +both hosts, the `tls` block still lists only `readytochat`, no new ACME order +or challenge was created, and the only two Orders present remain the +pre-existing valid ones. + +The path was then exercised end to end *before* any DNS change, by resolving +`realvuxbaro.me` to the k3s node explicitly: + +- `GET https://realvuxbaro.me/` → HTTP 200, 17,412 bytes. +- `POST https://realvuxbaro.me/api/chat` → HTTP 200 in 8.02 s, `answerable`, + one citation, correct paracetamol dosing text. + +So the cluster already answers correctly for the production hostname; only the +A record and the certificate remain. + +**Remaining TLS step.** `realvuxbaro.me` is intentionally absent from `tls`, +because HTTP-01 cannot validate while the name still resolves to Compose. +Immediately after the A record flips, add it to `tls` with its own +`secretName`; cert-manager then issues on the first attempt. Expect a browser +TLS warning between the DNS flip and issuance — acceptable here only because +the deployment has no real users yet. + +## CUTOVER DONE — realvuxbaro.me runs on k3s/ArgoCD (2026-08-17) + +The owner edited the A record themselves; Claude's browser input was blocked by +the harness permission classifier at that step, which is the right guard for a +live DNS change. Everything before and after it was automated and verified. + +Sequence as executed: + +1. TTL lowered to 60 s and allowed to take effect (verified at `1.1.1.1` and + `8.8.8.8`). +2. `realvuxbaro.me` A record: `52.0.158.61` → `44.206.194.195`. Both public + resolvers picked it up within the minute. +3. `realvuxbaro.me` added to the Ingress `tls` block with its own + `realvuxbaro-tls` secret (`master` at `0d8e366`). ArgoCD converged + Synced/Healthy. +4. cert-manager issued on the **first** attempt, ~40 s, one pending Challenge + then Ready — the payoff for keeping the host out of `tls` beforehand. + +Verified after cutover, all with full TLS verification (no `-k`): + +| Check | Result | +| --- | --- | +| `https://realvuxbaro.me/` | HTTP 200, `ssl_verify=0`, from `44.206.194.195` | +| Certificate | `CN=realvuxbaro.me`, Let's Encrypt, valid to 15 Nov 2026 | +| Live chat, 3 clinical questions | all `answerable` with a citation, 7.4–13.5 s | +| `https://realvuxbaro.me/grafana/login` | HTTP 200 | +| `https://readytochat.realvuxbaro.me/` | HTTP 200 (rehearsal name still served) | +| `https://argocd.realvuxbaro.me/` | HTTP 200 | +| Browser render | full UI, Gateway Online, query history populated | + +**Rollback is live and proven, not assumed.** The Compose instance +`i-039fc8f6102467a54` is still `running` and still serves the apex correctly: +addressing `realvuxbaro.me` at `52.0.158.61` returns HTTP 200 with a valid +`CN=realvuxbaro.me` certificate (Let's Encrypt, valid to 8 Nov 2026) and a +byte-identical 17,412-byte page. Reverting the A record is therefore a complete +rollback, bounded by the 60 s TTL. Do not stop that instance until the +acceptance window closes, and never terminate it — its root EBS volume is +`DeleteOnTermination=true`. + +Pre-existing defect surfaced, **not** caused by the cutover: `www.realvuxbaro.me` +does not serve TLS. It still points at the Compose host, and Caddy there has no +certificate for that name — the SNI handshake fails with an internal error, and +the certificate it does hold is apex-only. Decide whether to point `www` at k3s +and add it to `tls`, or drop the record. + ## Migration risks currently open -1. PostgreSQL/Qdrant snapshot, restore, rollback, ingress/TLS/DNS, secrets, - resource limits, and failure recovery still need explicit rehearsal gates. +1. `www.realvuxbaro.me` is broken (see above) and always was. +2. Secrets, resource limits and failure recovery still need explicit rehearsal + gates. +3. Answer-content parity between environments is still unverified — decisions + and latency match, answer length does not always. +4. The cluster still serves production from values files named `values-practice*`. + Rename once the acceptance window closes and the Compose host is retired; + `values-prod.yaml` is now unused by anything live. 3. Cluster-level inspection is currently blocked from this machine: the practice security group scopes port 6443 to the operator's own outbound IP, which has changed again, so `kubectl` hangs. ArgoCD's API and the public @@ -323,13 +496,17 @@ and character-identical content on both environments (11.36 s practice, ## Next safe execution order -1. Create the production ArgoCD Application against `values-prod.yaml`, and the - `medical-chatbot-prod` Secret it expects, without pointing DNS at it yet. -2. Restore the security-group rule for the operator's current IP, then verify - the running Pod environment, image digest, Qdrant identity, health, - readiness and traces through a cluster-level inspection path. -3. Run the golden set against both environments and compare answer content, not - just decision and latency. -4. Rehearse state restore and DNS rollback. Do not repoint `realvuxbaro.me` - until the gates and rollback path pass. +1. Lower the `realvuxbaro.me` A-record TTL to 60 s at Namecheap and wait out + the old 1799 s TTL. Nothing else in the cutover should start before this, + because it is what makes the rollback fast. +2. Pin production's Qdrant image to an explicit version instead of `latest`. +3. Create the `medical-chatbot-prod` Secret and the production ArgoCD + Application against `values-prod.yaml`, with an immutable SHA image tag. + Do not point DNS at it yet. +4. Decide the TLS approach (pre-seed Caddy's certificate, or accept a short + issuance gap) and put the `realvuxbaro.me` Ingress rule in place. +5. Run the golden set against both environments and compare answer content, not + just decision and latency — the one parity gap still unmeasured. +6. Cut DNS over, keep Compose running through the acceptance window, and only + then stop — never terminate — the Compose instance. diff --git a/infra/helm/medical-chatbot/templates/ingress.yaml b/infra/helm/medical-chatbot/templates/ingress.yaml index ad741ea..526ddd6 100644 --- a/infra/helm/medical-chatbot/templates/ingress.yaml +++ b/infra/helm/medical-chatbot/templates/ingress.yaml @@ -1,4 +1,16 @@ {{- if .Values.ingress.enabled }} +{{- /* + The cutover serves the new production hostname alongside the rehearsal one + from the same release, so `host` alone is not enough. `extraHosts` adds more + names against the identical path set; with none set the render is unchanged. + + Note that cert-manager's ingress-shim derives Certificates from the `tls` + block, not from these rules. A host listed here but absent from `tls` is + routed over HTTP without triggering an ACME order — which is what lets a new + hostname be wired up *before* its DNS moves, without burning Let's Encrypt + failed-validation attempts on a challenge that cannot succeed yet. +*/}} +{{- $hosts := concat (list .Values.ingress.host) (default (list) .Values.ingress.extraHosts) }} apiVersion: networking.k8s.io/v1 kind: Ingress metadata: @@ -14,27 +26,29 @@ spec: {{- toYaml . | nindent 4 }} {{- end }} rules: - - host: {{ .Values.ingress.host | quote }} + {{- range $host := $hosts }} + - host: {{ $host | quote }} http: paths: - path: /v1/rag pathType: Prefix backend: service: - name: {{ include "medical-chatbot.fullname" . }}-ai-service + name: {{ include "medical-chatbot.fullname" $ }}-ai-service port: { name: http } - {{- if .Values.observability.enabled }} + {{- if $.Values.observability.enabled }} - path: /grafana pathType: Prefix backend: service: - name: {{ include "medical-chatbot.fullname" . }}-grafana + name: {{ include "medical-chatbot.fullname" $ }}-grafana port: { name: http } {{- end }} - path: / pathType: Prefix backend: service: - name: {{ include "medical-chatbot.fullname" . }}-web + name: {{ include "medical-chatbot.fullname" $ }}-web port: { name: http } + {{- end }} {{- end }} diff --git a/infra/helm/medical-chatbot/templates/observability-workloads.yaml b/infra/helm/medical-chatbot/templates/observability-workloads.yaml index 3f8f27b..b84bcd7 100644 --- a/infra/helm/medical-chatbot/templates/observability-workloads.yaml +++ b/infra/helm/medical-chatbot/templates/observability-workloads.yaml @@ -236,13 +236,21 @@ spec: secretKeyRef: name: {{ include "medical-chatbot.secretName" . }} key: grafana-admin-password + # Anonymous access is a viewing convenience, never an admin + # bypass: this Grafana is reachable from the public internet + # through the same ingress as the app, so an anonymous Admin role + # hands strangers the datasource and dashboard APIs. - name: GF_AUTH_ANONYMOUS_ENABLED - value: {{ .Values.observability.grafana.anonymousAdmin | quote }} + value: {{ .Values.observability.grafana.anonymous.enabled | quote }} - name: GF_AUTH_ANONYMOUS_ORG_ROLE - value: Admin + value: {{ .Values.observability.grafana.anonymous.role | quote }} + # Never disabled. Hiding the login form alongside anonymous Admin + # left no way to authenticate as a real user at all. - name: GF_AUTH_DISABLE_LOGIN_FORM - value: {{ .Values.observability.grafana.anonymousAdmin | quote }} + value: "false" {{- if .Values.ingress.enabled }} + # Must be the hostname users actually arrive on, or Grafana's own + # redirects and asset links point at the wrong site. - name: GF_SERVER_ROOT_URL value: "https://{{ .Values.ingress.host }}/grafana/" - name: GF_SERVER_SERVE_FROM_SUB_PATH diff --git a/infra/helm/medical-chatbot/values-practice.yaml b/infra/helm/medical-chatbot/values-practice.yaml index 3b9b702..0645864 100644 --- a/infra/helm/medical-chatbot/values-practice.yaml +++ b/infra/helm/medical-chatbot/values-practice.yaml @@ -55,18 +55,37 @@ aiService: observability: grafana: - # Practice-only convenience: the rehearsal cluster is a throwaway box and - # Grafana sits behind the same ingress host. Never set this on production. - anonymousAdmin: true + # Dashboards stay open so a demo needs no credentials, but read-only: this + # Grafana answers on the public production hostname, and it previously ran + # anonymous *Admin* with the login form disabled — which handed the + # datasource and dashboard APIs to anyone who found the URL, and left no + # way to sign in as a real admin either. + anonymous: + enabled: true + role: Viewer ingress: enabled: true className: traefik - host: readytochat.realvuxbaro.me + # Primary, not merely first in a list: `ingress.host` is what Grafana's + # GF_SERVER_ROOT_URL is built from, so it must be the name users arrive on. + # `realvuxbaro.me` moved here from the Compose host on 2026-08-17 — routed + # first and certificated only once its A record pointed at this cluster, + # because cert-manager solves HTTP-01 and the challenge could not reach here + # before that. + host: realvuxbaro.me + # The rehearsal name is kept so existing links and bookmarks still resolve. + extraHosts: + - readytochat.realvuxbaro.me annotations: cert-manager.io/cluster-issuer: letsencrypt-prod traefik.ingress.kubernetes.io/router.entrypoints: websecure + # Separate secrets per hostname: a renewal failure on one name must not take + # the other offline. tls: - hosts: - readytochat.realvuxbaro.me secretName: readytochat-tls + - hosts: + - realvuxbaro.me + secretName: realvuxbaro-tls diff --git a/infra/helm/medical-chatbot/values.yaml b/infra/helm/medical-chatbot/values.yaml index 7d895e6..9d82714 100644 --- a/infra/helm/medical-chatbot/values.yaml +++ b/infra/helm/medical-chatbot/values.yaml @@ -73,6 +73,11 @@ ingress: className: nginx annotations: {} host: duocthu.local + # Additional hostnames served by the same rules as `host`. Used during a + # cutover, when one release must answer to both the old and the new name. + # A host here is routed but gets no certificate unless it also appears in + # `tls` below — see the note in templates/ingress.yaml. + extraHosts: [] tls: [] postgres: @@ -121,7 +126,12 @@ observability: port: 3000 nodePort: null storage: 2Gi - anonymousAdmin: false + # Anonymous access, when enabled, must stay read-only. `role: Admin` on an + # internet-reachable Grafana exposes the datasource and dashboard APIs to + # anyone; the login form is always available for real admin access. + anonymous: + enabled: false + role: Viewer resources: requests: { cpu: 50m, memory: 128Mi } limits: { cpu: 500m, memory: 512Mi }