158 lines
5.8 KiB
Python
158 lines
5.8 KiB
Python
"""Repair the corrupted inline `spec.source.helm.values` on medical-chatbot-app.
|
|
|
|
## What broke
|
|
|
|
Yesterday's manual ArgoCD UI edit saved the Application with a FOLDED block
|
|
scalar (`values: >`) rather than a literal one (`values: |`). A folded scalar
|
|
joins consecutive same-indent lines into one, so these four lines:
|
|
|
|
# Only the image blocks stay here: .github/scripts/sync_practice_argocd.py
|
|
# regex-rewrites these tags on every push. Every other setting is tracked in
|
|
# infra/helm/medical-chatbot/values-practice.yaml.
|
|
aiService:
|
|
|
|
collapsed into a SINGLE line -- putting `aiService:` inside the comment. The
|
|
stored string now starts:
|
|
|
|
0: '# Only the image blocks stay here: ... values-practice.yaml. aiService:'
|
|
1: ' image:'
|
|
...
|
|
5: 'web:'
|
|
|
|
which is not parseable YAML: line 1 is indented 2, so it would open the root
|
|
mapping at indent 2, and `web:` at indent 0 then sits *outside* it.
|
|
|
|
That one broken line explains every symptom together:
|
|
- the GHCR image overrides never apply, so Deployments fall back to the
|
|
chart default `duocthu-*:local`, which exists in no registry
|
|
(ErrImagePull -> stuck ReplicaSets -> App health Degraded);
|
|
- `authService`/`apiGateway` `enabled: true` never applies either, so those
|
|
Deployments were never created at all despite the UI showing them set;
|
|
- the App still reads "Synced" because ArgoCD's last SUCCESSFUL render was
|
|
16 hours ago -- it has been serving a stale comparison ever since.
|
|
|
|
The old pods keep serving traffic (Kubernetes will not retire them until a
|
|
replacement is Ready), which is why the site stayed up throughout.
|
|
|
|
## The repair
|
|
|
|
Rebuild the values with `aiService:` on its own line and drop the comment
|
|
block entirely. Dropping it is deliberate, not laziness:
|
|
|
|
- a base-indent comment directly above a base-indent key is exactly what
|
|
the folded scalar destroys, so re-adding it re-arms the same trap for the
|
|
next person who edits this in the UI;
|
|
- it is stale anyway -- it points at `values-practice.yaml`, renamed to
|
|
`values-production.yaml` on 2026-08-17.
|
|
|
|
The same explanation now lives in infra/argocd/applications/medical-chatbot-app.yaml,
|
|
which is version-controlled and cannot be mangled by a UI text box.
|
|
|
|
Every other line is preserved byte-for-byte, secrets included -- they are read
|
|
from the live object and written straight back, never logged.
|
|
|
|
Refuses to write unless the live values match the exact corruption described
|
|
above, so a rerun (or a differently-broken Application) is a no-op rather than
|
|
a second guess at what the content should be.
|
|
|
|
Required env: ARGOCD_PRACTICE_URL, ARGOCD_PRACTICE_PASSWORD.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import sys
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
|
|
APP_NAME = "medical-chatbot-app"
|
|
MANGLED_PREFIX = "# Only the image blocks stay here:"
|
|
MANGLED_SUFFIX = "aiService:"
|
|
POLL_SECONDS = 20
|
|
POLL_ROUNDS = 9 # ~3 minutes
|
|
|
|
|
|
def call(base: str, method: str, path: str, token: str | None = None, body=None):
|
|
req = urllib.request.Request(
|
|
f"{base}{path}",
|
|
data=json.dumps(body).encode() if body is not None else None,
|
|
method=method,
|
|
headers={"Content-Type": "application/json"},
|
|
)
|
|
if token:
|
|
req.add_header("Authorization", f"Bearer {token}")
|
|
with urllib.request.urlopen(req, timeout=30) as resp:
|
|
raw = resp.read()
|
|
return json.loads(raw) if raw else {}
|
|
|
|
|
|
def report(app: dict) -> None:
|
|
status = app.get("status", {})
|
|
print(f" sync={status.get('sync', {}).get('status')} health={status.get('health', {}).get('status')}")
|
|
for r in status.get("resources", []):
|
|
health = (r.get("health") or {}).get("status")
|
|
if r.get("kind") == "Deployment":
|
|
print(f" {r.get('kind'):<11} {r.get('name'):<48} health={health}")
|
|
|
|
|
|
def main() -> int:
|
|
base = os.environ["ARGOCD_PRACTICE_URL"].rstrip("/")
|
|
password = os.environ["ARGOCD_PRACTICE_PASSWORD"]
|
|
|
|
session = call(base, "POST", "/api/v1/session", body={"username": "admin", "password": password})
|
|
token = session["token"]
|
|
|
|
app = call(base, "GET", f"/api/v1/applications/{APP_NAME}", token=token)
|
|
values = app["spec"]["source"]["helm"].get("values", "")
|
|
lines = values.splitlines()
|
|
|
|
if not lines:
|
|
print("Inline values are empty -- nothing to repair, and nothing safe to guess.", file=sys.stderr)
|
|
return 1
|
|
|
|
first = lines[0]
|
|
if not (first.startswith(MANGLED_PREFIX) and first.rstrip().endswith(MANGLED_SUFFIX)):
|
|
print(
|
|
"Line 0 is not the known folded-comment corruption; refusing to rewrite.\n"
|
|
f" line 0 = {first!r}",
|
|
file=sys.stderr,
|
|
)
|
|
return 1
|
|
|
|
# Drop the mangled comment line, keep `aiService:` that it swallowed, and
|
|
# leave every remaining line untouched.
|
|
repaired = "\n".join(["aiService:"] + lines[1:]) + "\n"
|
|
|
|
print(f"Repairing line 0: {len(values)} chars -> {len(repaired)} chars")
|
|
print("Structural keys after repair:")
|
|
for line in repaired.splitlines():
|
|
if line and not line[0].isspace():
|
|
print(f" {line.split(':')[0]}:")
|
|
|
|
app["spec"]["source"]["helm"]["values"] = repaired
|
|
call(base, "PUT", f"/api/v1/applications/{APP_NAME}", token=token, body=app)
|
|
print("PUT accepted.")
|
|
|
|
try:
|
|
call(base, "POST", f"/api/v1/applications/{APP_NAME}/sync", token=token, body={})
|
|
print("Sync triggered.")
|
|
except urllib.error.HTTPError as exc:
|
|
if exc.code == 400:
|
|
print("Explicit sync raced with autosync (expected under selfHeal) -- continuing.")
|
|
else:
|
|
raise
|
|
|
|
for i in range(POLL_ROUNDS):
|
|
time.sleep(POLL_SECONDS)
|
|
app = call(base, "GET", f"/api/v1/applications/{APP_NAME}", token=token)
|
|
print(f"--- poll {i + 1}/{POLL_ROUNDS} (+{(i + 1) * POLL_SECONDS}s) ---")
|
|
report(app)
|
|
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|