Enable auth-service/api-gateway on production, build their images in CI
This commit is contained in:
@@ -1,9 +1,12 @@
|
||||
"""Point the k3s practice cluster's ArgoCD Application at a freshly-built
|
||||
image tag, then trigger an immediate sync.
|
||||
"""Point the k3s ArgoCD Application at an image tag, then trigger an
|
||||
immediate sync.
|
||||
|
||||
Only touches `medical-chatbot-app` on the practice cluster
|
||||
(argocd.realvuxbaro.me). Never touches production — the EC2 Compose
|
||||
deployment isn't ArgoCD-managed at all.
|
||||
`medical-chatbot-app` (argocd.realvuxbaro.me) is the production Application —
|
||||
it serves both realvuxbaro.me and readytochat.realvuxbaro.me — since the
|
||||
2026-08-17 cutover. This script does not distinguish "forward" from
|
||||
"rollback": it points the Application at whatever IMAGE_TAG it is given and
|
||||
syncs, so `rollback-k3s.yml` reuses it unchanged with an older tag rather than
|
||||
duplicating this logic.
|
||||
|
||||
Required env: ARGOCD_PRACTICE_URL, ARGOCD_PRACTICE_PASSWORD, IMAGE_TAG.
|
||||
"""
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
"""Block until the k3s ArgoCD Application is running EXPECT_TAG and is
|
||||
Synced + Healthy, or time out.
|
||||
|
||||
Companion to `sync_practice_argocd.py`, which fires a sync and returns
|
||||
immediately (it must, since ArgoCD's own selfHeal can race an explicit sync
|
||||
call — see its comment). A rollback caller needs the opposite guarantee: do
|
||||
not report success until the rollout has actually landed. Kept as a separate
|
||||
script rather than merged into that one, since the normal forward-deploy path
|
||||
in `build-practice-images.yml` confirms liveness a different way (an HTTP
|
||||
smoke check against readytochat) and doesn't need this blocking behaviour.
|
||||
|
||||
Required env: ARGOCD_PRACTICE_URL, ARGOCD_PRACTICE_PASSWORD, EXPECT_TAG.
|
||||
Optional env: APP_NAME (default medical-chatbot-app), TIMEOUT_SECONDS
|
||||
(default 300), POLL_INTERVAL_SECONDS (default 10).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
IMAGES = ("vsf-duocthu-ai-service", "vsf-duocthu-web")
|
||||
|
||||
|
||||
def call(base: str, method: str, path: str, token: str | None = None, body=None):
|
||||
req = urllib.request.Request(
|
||||
f"{base}{path}",
|
||||
data=json.dumps(body).encode() if body is not None else None,
|
||||
method=method,
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
if token:
|
||||
req.add_header("Authorization", f"Bearer {token}")
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
raw = resp.read()
|
||||
return json.loads(raw) if raw else {}
|
||||
except urllib.error.HTTPError as exc:
|
||||
print(f"{method} {path} -> {exc.code}: {exc.read().decode(errors='replace')}", file=sys.stderr)
|
||||
raise
|
||||
|
||||
|
||||
def deployed_tag(values: str, image: str) -> str | None:
|
||||
# Same shape sync_practice_argocd.py writes: a `repository:` line
|
||||
# immediately followed by its `tag:` line. Reading it back with the
|
||||
# mirror-image regex, rather than a looser substring check, means this
|
||||
# can't be fooled by the tag also appearing in a comment or another
|
||||
# image's block.
|
||||
match = re.search(
|
||||
rf"repository:\s*ghcr\.io/baovu2k4/{re.escape(image)}\s*\n\s*tag:\s*(\S+)",
|
||||
values,
|
||||
)
|
||||
return match.group(1) if match else None
|
||||
|
||||
|
||||
def main() -> int:
|
||||
base = os.environ["ARGOCD_PRACTICE_URL"].rstrip("/")
|
||||
password = os.environ["ARGOCD_PRACTICE_PASSWORD"]
|
||||
expect_tag = os.environ["EXPECT_TAG"]
|
||||
app_name = os.environ.get("APP_NAME", "medical-chatbot-app")
|
||||
timeout_seconds = int(os.environ.get("TIMEOUT_SECONDS", "300"))
|
||||
poll_interval = int(os.environ.get("POLL_INTERVAL_SECONDS", "10"))
|
||||
|
||||
session = call(base, "POST", "/api/v1/session", body={"username": "admin", "password": password})
|
||||
token = session["token"]
|
||||
|
||||
deadline = time.monotonic() + timeout_seconds
|
||||
last_state = "no poll yet"
|
||||
while time.monotonic() < deadline:
|
||||
app = call(base, "GET", f"/api/v1/applications/{app_name}", token=token)
|
||||
values = app["spec"]["source"]["helm"]["values"]
|
||||
tags = {image: deployed_tag(values, image) for image in IMAGES}
|
||||
sync_status = app.get("status", {}).get("sync", {}).get("status")
|
||||
health_status = app.get("status", {}).get("health", {}).get("status")
|
||||
last_state = f"tags={tags} sync={sync_status} health={health_status}"
|
||||
|
||||
if (
|
||||
all(tag == expect_tag for tag in tags.values())
|
||||
and sync_status == "Synced"
|
||||
and health_status == "Healthy"
|
||||
):
|
||||
print(f"{app_name} is on {expect_tag}, Synced, Healthy: {last_state}")
|
||||
return 0
|
||||
|
||||
print(f"waiting: {last_state}")
|
||||
time.sleep(poll_interval)
|
||||
|
||||
print(
|
||||
f"Timed out after {timeout_seconds}s waiting for {app_name} to reach "
|
||||
f"{expect_tag}/Synced/Healthy. Last observed: {last_state}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -18,6 +18,8 @@ on:
|
||||
paths:
|
||||
- apps/ai-service/**
|
||||
- apps/web/**
|
||||
- apps/auth-service/**
|
||||
- apps/api-gateway/**
|
||||
- packages/**
|
||||
- ingestion/data/verified/drug_entities.json
|
||||
- .github/workflows/build-practice-images.yml
|
||||
@@ -66,6 +68,33 @@ jobs:
|
||||
cache-from: type=gha,scope=practice-web
|
||||
cache-to: type=gha,mode=max,scope=practice-web
|
||||
|
||||
- name: Build and push auth-service
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
file: apps/auth-service/Dockerfile
|
||||
push: true
|
||||
tags: ghcr.io/baovu2k4/vsf-duocthu-auth-service:${{ github.sha }}
|
||||
cache-from: type=gha,scope=practice-auth-service
|
||||
cache-to: type=gha,mode=max,scope=practice-auth-service
|
||||
|
||||
- name: Build and push api-gateway
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
file: apps/api-gateway/Dockerfile
|
||||
push: true
|
||||
tags: ghcr.io/baovu2k4/vsf-duocthu-api-gateway:${{ github.sha }}
|
||||
cache-from: type=gha,scope=practice-api-gateway
|
||||
cache-to: type=gha,mode=max,scope=practice-api-gateway
|
||||
|
||||
# auth-service/api-gateway are NOT in sync_practice_argocd.py's IMAGES
|
||||
# tuple yet — that script fails the whole run if a tag it expects to
|
||||
# rewrite isn't already present inline on the Application, so extending
|
||||
# it must wait until someone has added `authService.image` /
|
||||
# `apiGateway.image` blocks to the live Application by hand (see
|
||||
# infra/helm/medical-chatbot/values-production.yaml). Until then, these
|
||||
# two images are pushed here but must be repointed manually.
|
||||
- name: Point the practice ArgoCD Application at the new images
|
||||
env:
|
||||
ARGOCD_PRACTICE_URL: ${{ secrets.ARGOCD_PRACTICE_URL }}
|
||||
|
||||
@@ -102,3 +102,28 @@ jobs:
|
||||
|
||||
- name: Build
|
||||
run: pnpm --filter @duoc-thu/web build
|
||||
|
||||
auth-and-gateway:
|
||||
name: auth-service + api-gateway — lint + build + test
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: "20"
|
||||
|
||||
- name: Enable pnpm
|
||||
run: corepack enable
|
||||
|
||||
- name: Install
|
||||
run: pnpm install --frozen-lockfile
|
||||
|
||||
- name: Lint
|
||||
run: pnpm --filter @duoc-thu/auth-service --filter @duoc-thu/api-gateway lint
|
||||
|
||||
- name: Build
|
||||
run: pnpm --filter @duoc-thu/auth-service --filter @duoc-thu/api-gateway build
|
||||
|
||||
- name: Test
|
||||
run: pnpm --filter @duoc-thu/auth-service --filter @duoc-thu/api-gateway test
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
name: Rollback k3s production
|
||||
|
||||
# One-command escape hatch for medical-chatbot-app (ArgoCD, k3s) — the
|
||||
# Application build-practice-images.yml normally advances, and the same one
|
||||
# serving realvuxbaro.me since the 2026-08-17 cutover. This workflow does not
|
||||
# build anything: it only repoints the Application at an OLDER image tag that
|
||||
# a previous build-practice-images.yml run already pushed to GHCR, using the
|
||||
# exact same sync_practice_argocd.py logic that workflow uses to move
|
||||
# forward — a rollback is just that script pointed backward.
|
||||
#
|
||||
# Does not touch the Compose EC2 (stopped 2026-08-18, no CI/CD path left —
|
||||
# see docs/operations.md).
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
target_sha:
|
||||
description: >-
|
||||
Commit SHA to roll back to. Must have a successful "Build and sync
|
||||
k3s images" run (check: gh run list --workflow=build-practice-images.yml).
|
||||
required: true
|
||||
|
||||
concurrency:
|
||||
# Same group as build-practice-images.yml: a rollback and a forward deploy
|
||||
# must never race to repoint the same Application at the same time.
|
||||
group: practice-images
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
rollback:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
packages: read
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: docker/setup-buildx-action@v3
|
||||
|
||||
- uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
# Fails fast and clearly on the most likely operator mistake: a typo'd
|
||||
# or never-built SHA, rather than that surfacing later as a confusing
|
||||
# ArgoCD/pod-level image pull failure.
|
||||
- name: Confirm images exist for target_sha
|
||||
run: |
|
||||
set -eu
|
||||
docker buildx imagetools inspect "ghcr.io/baovu2k4/vsf-duocthu-ai-service:${{ inputs.target_sha }}" > /dev/null
|
||||
docker buildx imagetools inspect "ghcr.io/baovu2k4/vsf-duocthu-web:${{ inputs.target_sha }}" > /dev/null
|
||||
|
||||
- name: Point medical-chatbot-app at target_sha
|
||||
env:
|
||||
ARGOCD_PRACTICE_URL: ${{ secrets.ARGOCD_PRACTICE_URL }}
|
||||
ARGOCD_PRACTICE_PASSWORD: ${{ secrets.ARGOCD_PRACTICE_PASSWORD }}
|
||||
IMAGE_TAG: ${{ inputs.target_sha }}
|
||||
run: python3 .github/scripts/sync_practice_argocd.py
|
||||
|
||||
# Blocks until the rollout has actually landed, not just until the sync
|
||||
# call returned — sync_practice_argocd.py deliberately doesn't wait
|
||||
# (see its own comment on the selfHeal race), so a rollback specifically
|
||||
# needs this extra confirmation before it can claim success.
|
||||
- name: Wait for the Application to be Synced, Healthy, and on target_sha
|
||||
env:
|
||||
ARGOCD_PRACTICE_URL: ${{ secrets.ARGOCD_PRACTICE_URL }}
|
||||
ARGOCD_PRACTICE_PASSWORD: ${{ secrets.ARGOCD_PRACTICE_PASSWORD }}
|
||||
EXPECT_TAG: ${{ inputs.target_sha }}
|
||||
run: python3 .github/scripts/wait_for_argocd_sync.py
|
||||
|
||||
- name: Confirm realvuxbaro.me is serving the rolled-back build
|
||||
run: |
|
||||
for attempt in $(seq 1 18); do
|
||||
code=$(curl -s -o /dev/null -w '%{http_code}' \
|
||||
"https://realvuxbaro.me/api/history?conversation_id=rollback-smoke-${{ inputs.target_sha }}")
|
||||
if [ "$code" = "200" ]; then
|
||||
echo "realvuxbaro.me is live on ${{ inputs.target_sha }}"
|
||||
exit 0
|
||||
fi
|
||||
sleep 10
|
||||
done
|
||||
echo "realvuxbaro.me did not respond healthy within 3 minutes after rollback"
|
||||
exit 1
|
||||
Reference in New Issue
Block a user