Add an explicit ArgoCD sync and health-poll script

2026-08-19 09:21:35 +07:00
parent b68005be1c
commit f50ccfc5f7
12 changed files with 298 additions and 54 deletions
@@ -0,0 +1,99 @@
+"""Read-only diagnostic: print the structure of `medical-chatbot-app`'s
+inline `spec.source.helm.values` on the live ArgoCD Application, with any
+line that looks like it holds a credential redacted. Also prints per-resource
+health from the resource tree, so a Degraded app health can be traced to the
+specific Deployment/Pod causing it.
+
+Never mutates anything. Exists to let us see the real inline-values layout
+and live resource health before writing a script that edits the Application
+(see the note in infra/argocd/applications/medical-chatbot-app.yaml about
+what stays inline).
+
+Required env: ARGOCD_PRACTICE_URL, ARGOCD_PRACTICE_PASSWORD.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import sys
+import urllib.error
+import urllib.request
+
+APP_NAME = "medical-chatbot-app"
+SECRET_LINE = re.compile(r"(password|secret|token|key)", re.IGNORECASE)
+
+
+def call(base: str, method: str, path: str, token: str | None = None, body=None):
+    req = urllib.request.Request(
+        f"{base}{path}",
+        data=json.dumps(body).encode() if body is not None else None,
+        method=method,
+        headers={"Content-Type": "application/json"},
+    )
+    if token:
+        req.add_header("Authorization", f"Bearer {token}")
+    with urllib.request.urlopen(req, timeout=30) as resp:
+        raw = resp.read()
+        return json.loads(raw) if raw else {}
+
+
+def redact(line: str) -> str:
+    if ":" in line and SECRET_LINE.search(line.split(":", 1)[0]):
+        key = line.split(":", 1)[0]
+        return f"{key}: <redacted>"
+    return line
+
+
+def main() -> int:
+    base = os.environ["ARGOCD_PRACTICE_URL"].rstrip("/")
+    password = os.environ["ARGOCD_PRACTICE_PASSWORD"]
+
+    session = call(base, "POST", "/api/v1/session", body={"username": "admin", "password": password})
+    token = session["token"]
+
+    app = call(base, "GET", f"/api/v1/applications/{APP_NAME}", token=token)
+    values = app["spec"]["source"]["helm"].get("values", "")
+
+    print(f"sync.status={app.get('status', {}).get('sync', {}).get('status')}")
+    print(f"health.status={app.get('status', {}).get('health', {}).get('status')}")
+    print("--- spec.source.helm.values (secrets redacted) ---")
+    for line in values.splitlines():
+        print(redact(line))
+    print("--- end values ---")
+
+    print("--- status.resources (per-resource health) ---")
+    for r in app.get("status", {}).get("resources", []):
+        health = r.get("health", {})
+        print(
+            f"{r.get('kind'):<12} {r.get('name'):<45} "
+            f"status={r.get('status')} health={health.get('status')} "
+            f"msg={health.get('message', '')}"
+        )
+    print("--- end resources ---")
+
+    print("--- status.conditions ---")
+    for c in app.get("status", {}).get("conditions", []):
+        print(f"{c.get('type')}: {c.get('message')}")
+    print("--- end conditions ---")
+
+    print("--- resource tree (nodes with non-empty health/status) ---")
+    try:
+        tree = call(base, "GET", f"/api/v1/applications/{APP_NAME}/resource-tree", token=token)
+        for n in tree.get("nodes", []):
+            h = n.get("health", {})
+            if h.get("status") not in (None, "Healthy") or n.get("kind") == "Pod":
+                print(
+                    f"{n.get('kind'):<12} {n.get('name'):<45} "
+                    f"health={h.get('status')} msg={h.get('message', '')}"
+                )
+    except urllib.error.HTTPError as exc:
+        print(f"resource-tree fetch failed: {exc.code} {exc.read().decode(errors='replace')}", file=sys.stderr)
+    print("--- end tree ---")
+
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
@@ -0,0 +1,100 @@
+"""Trigger an explicit sync of the live medical-chatbot-app Application, then
+poll and report per-resource health -- so we can see directly whether the
+sync clears the stuck ai-service/web rollout and brings up auth-service/
+api-gateway, rather than guessing from a separate read-only run.
+
+Does not touch spec.source.helm.values or anything else -- only calls
+POST /sync (an ArgoCD-native action, same as clicking SYNC in the UI, and the
+same call sync_practice_argocd.py already makes on every routine deploy) and
+then polls GET.
+
+Required env: ARGOCD_PRACTICE_URL, ARGOCD_PRACTICE_PASSWORD.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import sys
+import time
+import urllib.error
+import urllib.request
+
+APP_NAME = "medical-chatbot-app"
+SECRET_LINE = re.compile(r"(password|secret|token|key)", re.IGNORECASE)
+POLL_SECONDS = 15
+POLL_ROUNDS = 8  # ~2 minutes
+
+
+def call(base: str, method: str, path: str, token: str | None = None, body=None):
+    req = urllib.request.Request(
+        f"{base}{path}",
+        data=json.dumps(body).encode() if body is not None else None,
+        method=method,
+        headers={"Content-Type": "application/json"},
+    )
+    if token:
+        req.add_header("Authorization", f"Bearer {token}")
+    with urllib.request.urlopen(req, timeout=30) as resp:
+        raw = resp.read()
+        return json.loads(raw) if raw else {}
+
+
+def redact(line: str) -> str:
+    if ":" in line and SECRET_LINE.search(line.split(":", 1)[0]):
+        key = line.split(":", 1)[0]
+        return f"{key}: <redacted>"
+    return line
+
+
+def report(app: dict) -> None:
+    print(f"sync.status={app.get('status', {}).get('sync', {}).get('status')}")
+    print(f"health.status={app.get('status', {}).get('health', {}).get('status')}")
+    for r in app.get("status", {}).get("resources", []):
+        health = r.get("health", {})
+        if health.get("status") not in (None, "Healthy") or r.get("kind") in ("Deployment", "Pod"):
+            print(
+                f"  {r.get('kind'):<12} {r.get('name'):<45} "
+                f"status={r.get('status')} health={health.get('status')} "
+                f"msg={health.get('message', '')}"
+            )
+
+
+def main() -> int:
+    base = os.environ["ARGOCD_PRACTICE_URL"].rstrip("/")
+    password = os.environ["ARGOCD_PRACTICE_PASSWORD"]
+
+    session = call(base, "POST", "/api/v1/session", body={"username": "admin", "password": password})
+    token = session["token"]
+
+    app = call(base, "GET", f"/api/v1/applications/{APP_NAME}", token=token)
+    values = app["spec"]["source"]["helm"].get("values", "")
+    print("--- spec.source.helm.values before sync (secrets redacted) ---")
+    for line in values.splitlines():
+        print(redact(line))
+    print("--- end values ---")
+
+    print("--- before sync ---")
+    report(app)
+
+    print("--- triggering sync ---")
+    try:
+        call(base, "POST", f"/api/v1/applications/{APP_NAME}/sync", token=token, body={})
+    except urllib.error.HTTPError as exc:
+        if exc.code == 400:
+            print("Sync request raced with an in-progress operation (400) -- continuing to poll.")
+        else:
+            raise
+
+    for i in range(POLL_ROUNDS):
+        time.sleep(POLL_SECONDS)
+        app = call(base, "GET", f"/api/v1/applications/{APP_NAME}", token=token)
+        print(f"--- poll {i + 1}/{POLL_ROUNDS} (+{(i + 1) * POLL_SECONDS}s) ---")
+        report(app)
+
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())