VuQuangBao/duocthu
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add an explicit ArgoCD sync and health-poll script

Browse Source
This commit is contained in:
BaoVu2k4
2026-08-19 09:21:35 +07:00
parent b68005be1c
commit f50ccfc5f7
12 changed files with 298 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/scripts/inspect_argocd_app.py
+99
View File
@@ -0,0 +1,99 @@
"""Read-only diagnostic: print the structure of `medical-chatbot-app`'s
inline `spec.source.helm.values` on the live ArgoCD Application, with any
line that looks like it holds a credential redacted. Also prints per-resource
health from the resource tree, so a Degraded app health can be traced to the
specific Deployment/Pod causing it.
Never mutates anything. Exists to let us see the real inline-values layout
and live resource health before writing a script that edits the Application
(see the note in infra/argocd/applications/medical-chatbot-app.yaml about
what stays inline).
Required env: ARGOCD_PRACTICE_URL, ARGOCD_PRACTICE_PASSWORD.
"""
from __future__ import annotations
import json
import os
import re
import sys
import urllib.error
import urllib.request
APP_NAME = "medical-chatbot-app"
SECRET_LINE = re.compile(r"(password|secret|token|key)", re.IGNORECASE)
def call(base: str, method: str, path: str, token: str | None = None, body=None):
req = urllib.request.Request(
f"{base}{path}",
data=json.dumps(body).encode() if body is not None else None,
method=method,
headers={"Content-Type": "application/json"},
)
if token:
req.add_header("Authorization", f"Bearer {token}")
with urllib.request.urlopen(req, timeout=30) as resp:
raw = resp.read()
return json.loads(raw) if raw else {}
def redact(line: str) -> str:
if ":" in line and SECRET_LINE.search(line.split(":", 1)[0]):
key = line.split(":", 1)[0]
return f"{key}: <redacted>"
return line
def main() -> int:
base = os.environ["ARGOCD_PRACTICE_URL"].rstrip("/")
password = os.environ["ARGOCD_PRACTICE_PASSWORD"]
session = call(base, "POST", "/api/v1/session", body={"username": "admin", "password": password})
token = session["token"]
app = call(base, "GET", f"/api/v1/applications/{APP_NAME}", token=token)
values = app["spec"]["source"]["helm"].get("values", "")
print(f"sync.status={app.get('status', {}).get('sync', {}).get('status')}")
print(f"health.status={app.get('status', {}).get('health', {}).get('status')}")
print("--- spec.source.helm.values (secrets redacted) ---")
for line in values.splitlines():
print(redact(line))
print("--- end values ---")
print("--- status.resources (per-resource health) ---")
for r in app.get("status", {}).get("resources", []):
health = r.get("health", {})
print(
f"{r.get('kind'):<12} {r.get('name'):<45} "
f"status={r.get('status')} health={health.get('status')} "
f"msg={health.get('message', '')}"
)
print("--- end resources ---")
print("--- status.conditions ---")
for c in app.get("status", {}).get("conditions", []):
print(f"{c.get('type')}: {c.get('message')}")
print("--- end conditions ---")
print("--- resource tree (nodes with non-empty health/status) ---")
try:
tree = call(base, "GET", f"/api/v1/applications/{APP_NAME}/resource-tree", token=token)
for n in tree.get("nodes", []):
h = n.get("health", {})
if h.get("status") not in (None, "Healthy") or n.get("kind") == "Pod":
print(
f"{n.get('kind'):<12} {n.get('name'):<45} "
f"health={h.get('status')} msg={h.get('message', '')}"
)
except urllib.error.HTTPError as exc:
print(f"resource-tree fetch failed: {exc.code} {exc.read().decode(errors='replace')}", file=sys.stderr)
print("--- end tree ---")
return 0
if __name__ == "__main__":
raise SystemExit(main())
.github/scripts/sync_and_report_argocd_app.py
+100
View File
@@ -0,0 +1,100 @@
"""Trigger an explicit sync of the live medical-chatbot-app Application, then
poll and report per-resource health -- so we can see directly whether the
sync clears the stuck ai-service/web rollout and brings up auth-service/
api-gateway, rather than guessing from a separate read-only run.
Does not touch spec.source.helm.values or anything else -- only calls
POST /sync (an ArgoCD-native action, same as clicking SYNC in the UI, and the
same call sync_practice_argocd.py already makes on every routine deploy) and
then polls GET.
Required env: ARGOCD_PRACTICE_URL, ARGOCD_PRACTICE_PASSWORD.
"""
from __future__ import annotations
import json
import os
import re
import sys
import time
import urllib.error
import urllib.request
APP_NAME = "medical-chatbot-app"
SECRET_LINE = re.compile(r"(password|secret|token|key)", re.IGNORECASE)
POLL_SECONDS = 15
POLL_ROUNDS = 8 # ~2 minutes
def call(base: str, method: str, path: str, token: str | None = None, body=None):
req = urllib.request.Request(
f"{base}{path}",
data=json.dumps(body).encode() if body is not None else None,
method=method,
headers={"Content-Type": "application/json"},
)
if token:
req.add_header("Authorization", f"Bearer {token}")
with urllib.request.urlopen(req, timeout=30) as resp:
raw = resp.read()
return json.loads(raw) if raw else {}
def redact(line: str) -> str:
if ":" in line and SECRET_LINE.search(line.split(":", 1)[0]):
key = line.split(":", 1)[0]
return f"{key}: <redacted>"
return line
def report(app: dict) -> None:
print(f"sync.status={app.get('status', {}).get('sync', {}).get('status')}")
print(f"health.status={app.get('status', {}).get('health', {}).get('status')}")
for r in app.get("status", {}).get("resources", []):
health = r.get("health", {})
if health.get("status") not in (None, "Healthy") or r.get("kind") in ("Deployment", "Pod"):
print(
f" {r.get('kind'):<12} {r.get('name'):<45} "
f"status={r.get('status')} health={health.get('status')} "
f"msg={health.get('message', '')}"
)
def main() -> int:
base = os.environ["ARGOCD_PRACTICE_URL"].rstrip("/")
password = os.environ["ARGOCD_PRACTICE_PASSWORD"]
session = call(base, "POST", "/api/v1/session", body={"username": "admin", "password": password})
token = session["token"]
app = call(base, "GET", f"/api/v1/applications/{APP_NAME}", token=token)
values = app["spec"]["source"]["helm"].get("values", "")
print("--- spec.source.helm.values before sync (secrets redacted) ---")
for line in values.splitlines():
print(redact(line))
print("--- end values ---")
print("--- before sync ---")
report(app)
print("--- triggering sync ---")
try:
call(base, "POST", f"/api/v1/applications/{APP_NAME}/sync", token=token, body={})
except urllib.error.HTTPError as exc:
if exc.code == 400:
print("Sync request raced with an in-progress operation (400) -- continuing to poll.")
else:
raise
for i in range(POLL_ROUNDS):
time.sleep(POLL_SECONDS)
app = call(base, "GET", f"/api/v1/applications/{APP_NAME}", token=token)
print(f"--- poll {i + 1}/{POLL_ROUNDS} (+{(i + 1) * POLL_SECONDS}s) ---")
report(app)
return 0
if __name__ == "__main__":
raise SystemExit(main())
.github/workflows/helm-chart.yml
+14 -2
View File
@@ -27,12 +27,21 @@ jobs:
run: |
helm template default infra/helm/medical-chatbot > /tmp/default.yaml
# values-production.yaml turns authService/apiGateway on, and their
# jwt-secret Secret key is `required` with no default -- real values
# only ever exist inline on the live Application, never in Git (see
# the comment on secret.jwtSecret in values-production.yaml). This
# placeholder exists purely so these renders reach the assertion
# they're actually testing (the image-tag guard) instead of failing
# on an unrelated missing secret; it is never applied to a cluster.
CI_JWT_SECRET=ci-render-only-not-a-real-secret
# The live releases carry no image tag in Git -- it is supplied per
# deploy as a commit SHA through the ArgoCD Application. Rendering
# with an empty tag must FAIL rather than fall back to the chart's
# `local` development tag, so assert the failure directly; otherwise
# the guard could rot into a silent default unnoticed.
if helm template production infra/helm/medical-chatbot --values infra/helm/medical-chatbot/values-production.yaml --set aiService.image.tag="" --set web.image.tag="" > /tmp/untagged.yaml 2>/tmp/untagged.err; then
if helm template production infra/helm/medical-chatbot --values infra/helm/medical-chatbot/values-production.yaml --set secret.jwtSecret="$CI_JWT_SECRET" --set aiService.image.tag="" --set web.image.tag="" > /tmp/untagged.yaml 2>/tmp/untagged.err; then
echo "::error::render succeeded with no image tag; the immutable-tag guard is gone"
exit 1
fi
@@ -40,7 +49,7 @@ jobs:
# ...and with a tag it must resolve the GHCR package, not the local
# development image name.
helm template production infra/helm/medical-chatbot --values infra/helm/medical-chatbot/values-production.yaml --set aiService.image.repository=ghcr.io/baovu2k4/vsf-duocthu-ai-service --set web.image.repository=ghcr.io/baovu2k4/vsf-duocthu-web --set aiService.image.tag="$GITHUB_SHA" --set web.image.tag="$GITHUB_SHA" > /tmp/tagged.yaml
helm template production infra/helm/medical-chatbot --values infra/helm/medical-chatbot/values-production.yaml --set secret.jwtSecret="$CI_JWT_SECRET" --set aiService.image.repository=ghcr.io/baovu2k4/vsf-duocthu-ai-service --set web.image.repository=ghcr.io/baovu2k4/vsf-duocthu-web --set aiService.image.tag="$GITHUB_SHA" --set web.image.tag="$GITHUB_SHA" > /tmp/tagged.yaml
grep -q "image: \"ghcr.io/baovu2k4/vsf-duocthu-ai-service:$GITHUB_SHA\"" /tmp/tagged.yaml
grep -q "image: \"ghcr.io/baovu2k4/vsf-duocthu-web:$GITHUB_SHA\"" /tmp/tagged.yaml
@@ -48,8 +57,11 @@ jobs:
# contract is asserted here rather than trusted by review.
- name: Render the live production manifests
run: |
# See the same placeholder note in the previous step -- real secret
# material never enters Git and this is a render-only dry run.
helm template medical-chatbot-app infra/helm/medical-chatbot \
--values infra/helm/medical-chatbot/values-production.yaml \
--set secret.jwtSecret=ci-render-only-not-a-real-secret \
> /tmp/prod-app.yaml
helm template medical-chatbot-data infra/helm/medical-chatbot \
--values infra/helm/medical-chatbot/values-production-data.yaml \
.github/workflows/inspect-argocd-app.yml
+22
View File
@@ -0,0 +1,22 @@
name: Inspect ArgoCD Application (read-only)
# One-off diagnostic to see the live medical-chatbot-app Application's inline
# helm values before writing a script that edits them (adding secret.jwtSecret
# alongside the existing secret.grafanaAdminPassword). Read-only: only calls
# GET on the ArgoCD API, never PUT or sync.
on:
workflow_dispatch: {}
jobs:
inspect:
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- uses: actions/checkout@v4
- name: Inspect live Application
env:
ARGOCD_PRACTICE_URL: ${{ secrets.ARGOCD_PRACTICE_URL }}
ARGOCD_PRACTICE_PASSWORD: ${{ secrets.ARGOCD_PRACTICE_PASSWORD }}
run: python3 .github/scripts/inspect_argocd_app.py
.github/workflows/sync-argocd-app.yml
+25
View File
@@ -0,0 +1,25 @@
name: Sync ArgoCD Application and report health
# Yesterday's session left the live medical-chatbot-app Application with
# secret.jwtSecret + authService/apiGateway enabled set inline (via manual
# ArgoCD UI edits) but no explicit sync afterward -- health is Synced/
# Degraded with a stuck ai-service/web rollout using the chart's default
# (nonexistent) image. This forces one explicit sync (same action as the
# UI's SYNC button) and polls health afterward so we see the real result
# instead of guessing.
on:
workflow_dispatch: {}
jobs:
sync:
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- uses: actions/checkout@v4
- name: Sync and report
env:
ARGOCD_PRACTICE_URL: ${{ secrets.ARGOCD_PRACTICE_URL }}
ARGOCD_PRACTICE_PASSWORD: ${{ secrets.ARGOCD_PRACTICE_PASSWORD }}
run: python3 .github/scripts/sync_and_report_argocd_app.py