Add read-only production runtime audit
This commit is contained in:
@@ -0,0 +1,122 @@
|
||||
name: Audit production runtime (read-only)
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: audit-production-runtime
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
audit:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Inspect production over SSH
|
||||
uses: appleboy/ssh-action@v1.0.3
|
||||
with:
|
||||
host: ${{ secrets.EC2_HOST }}
|
||||
username: ubuntu
|
||||
key: ${{ secrets.EC2_SSH_KEY }}
|
||||
command_timeout: 10m
|
||||
script: |
|
||||
set -eu
|
||||
cd ~/app
|
||||
|
||||
printf '%s\n' '=== source ==='
|
||||
printf 'git_sha='
|
||||
git rev-parse HEAD
|
||||
printf 'git_branch='
|
||||
git branch --show-current
|
||||
|
||||
cd infra/docker
|
||||
ai_id=$(sudo docker compose -f docker-compose.prod.yml ps -q ai-service)
|
||||
web_id=$(sudo docker compose -f docker-compose.prod.yml ps -q web)
|
||||
postgres_id=$(sudo docker compose -f docker-compose.prod.yml ps -q postgres)
|
||||
qdrant_id=$(sudo docker compose -f docker-compose.prod.yml ps -q qdrant)
|
||||
test -n "$ai_id"
|
||||
test -n "$web_id"
|
||||
test -n "$postgres_id"
|
||||
test -n "$qdrant_id"
|
||||
|
||||
printf '%s\n' '=== containers ==='
|
||||
for entry in "ai-service:$ai_id" "web:$web_id" "postgres:$postgres_id" "qdrant:$qdrant_id"; do
|
||||
service=${entry%%:*}
|
||||
container=${entry#*:}
|
||||
state=$(sudo docker inspect --format '{{.State.Status}}' "$container")
|
||||
image_id=$(sudo docker inspect --format '{{.Image}}' "$container")
|
||||
printf '%s state=%s image_id=%s\n' "$service" "$state" "$image_id"
|
||||
done
|
||||
|
||||
printf '%s\n' '=== ai_runtime_contract ==='
|
||||
sudo docker exec -i "$ai_id" python - <<'PY'
|
||||
import json
|
||||
|
||||
from config import Settings
|
||||
|
||||
settings = Settings()
|
||||
safe_fields = (
|
||||
"app_name",
|
||||
"environment",
|
||||
"qdrant_url",
|
||||
"qdrant_collection",
|
||||
"embedding_provider",
|
||||
"embedding_dimensions",
|
||||
"evidence_minimum_score",
|
||||
"aws_region",
|
||||
"answer_provider",
|
||||
"answer_model_id",
|
||||
"rerank_enabled",
|
||||
"metrics_enabled",
|
||||
"otel_enabled",
|
||||
"otel_service_name",
|
||||
"otel_exporter_otlp_endpoint",
|
||||
"otel_sample_ratio",
|
||||
"entities_path",
|
||||
"max_wall_clock_ms",
|
||||
"max_llm_calls_per_turn",
|
||||
)
|
||||
contract = {name: str(getattr(settings, name)) for name in safe_fields}
|
||||
print(json.dumps(contract, ensure_ascii=True, sort_keys=True))
|
||||
PY
|
||||
|
||||
printf '%s\n' '=== persistent_mounts ==='
|
||||
for entry in "postgres:$postgres_id" "qdrant:$qdrant_id"; do
|
||||
service=${entry%%:*}
|
||||
container=${entry#*:}
|
||||
sudo docker inspect --format \
|
||||
"$service {{range .Mounts}}{{.Type}}:{{.Name}}:{{.Destination}} {{end}}" \
|
||||
"$container"
|
||||
done
|
||||
|
||||
printf '%s\n' '=== datastore_identity ==='
|
||||
sudo docker exec -i "$ai_id" python - <<'PY'
|
||||
import json
|
||||
import urllib.request
|
||||
|
||||
from config import Settings
|
||||
|
||||
settings = Settings()
|
||||
url = settings.qdrant_url.rstrip("/") + "/collections/" + settings.qdrant_collection
|
||||
with urllib.request.urlopen(url, timeout=10) as response:
|
||||
payload = json.load(response)
|
||||
result = payload.get("result", {})
|
||||
config = result.get("config", {}).get("params", {}).get("vectors", {})
|
||||
print(json.dumps({
|
||||
"collection": settings.qdrant_collection,
|
||||
"points_count": result.get("points_count"),
|
||||
"status": result.get("status"),
|
||||
"vector_config": config,
|
||||
}, ensure_ascii=True, sort_keys=True))
|
||||
PY
|
||||
|
||||
printf '%s\n' '=== health ==='
|
||||
sudo docker exec -i "$ai_id" python - <<'PY'
|
||||
import urllib.request
|
||||
|
||||
for path in ("/health", "/ready"):
|
||||
with urllib.request.urlopen("http://127.0.0.1:8000" + path, timeout=10) as response:
|
||||
print(path, response.status)
|
||||
PY
|
||||
@@ -0,0 +1,86 @@
|
||||
name: Build and sync k3s practice images
|
||||
|
||||
# Practice-cluster only (readytochat.realvuxbaro.me, ArgoCD-managed on the
|
||||
# self-hosted k3s box). Does not touch deploy.yml or the production
|
||||
# EC2/Compose stack — production never pulls a GHCR image and isn't
|
||||
# ArgoCD-managed at all, so this workflow has no path to affect it.
|
||||
#
|
||||
# ArgoCD's Applications already autosync (syncPolicy.automated) — the gap
|
||||
# this closes is that the image tag they deploy was a static string
|
||||
# (`:practice`) that nothing ever rebuilt. This tags every build with the
|
||||
# commit SHA and repoints the Application at it.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [master]
|
||||
paths:
|
||||
- apps/ai-service/**
|
||||
- apps/web/**
|
||||
- packages/**
|
||||
- ingestion/data/verified/drug_entities.json
|
||||
- .github/workflows/build-practice-images.yml
|
||||
- .github/scripts/sync_practice_argocd.py
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: practice-images
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
build-and-sync:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Log in to GHCR
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Build and push ai-service
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
file: apps/ai-service/Dockerfile
|
||||
push: true
|
||||
tags: ghcr.io/baovu2k4/vsf-duocthu-ai-service:${{ github.sha }}
|
||||
cache-from: type=gha,scope=practice-ai-service
|
||||
cache-to: type=gha,mode=max,scope=practice-ai-service
|
||||
|
||||
- name: Build and push web
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
file: apps/web/Dockerfile
|
||||
push: true
|
||||
tags: ghcr.io/baovu2k4/vsf-duocthu-web:${{ github.sha }}
|
||||
cache-from: type=gha,scope=practice-web
|
||||
cache-to: type=gha,mode=max,scope=practice-web
|
||||
|
||||
- name: Point the practice ArgoCD Application at the new images
|
||||
env:
|
||||
ARGOCD_PRACTICE_URL: ${{ secrets.ARGOCD_PRACTICE_URL }}
|
||||
ARGOCD_PRACTICE_PASSWORD: ${{ secrets.ARGOCD_PRACTICE_PASSWORD }}
|
||||
IMAGE_TAG: ${{ github.sha }}
|
||||
run: python3 .github/scripts/sync_practice_argocd.py
|
||||
|
||||
- name: Confirm readytochat is serving the new build
|
||||
run: |
|
||||
for attempt in $(seq 1 18); do
|
||||
code=$(curl -s -o /dev/null -w '%{http_code}' \
|
||||
"https://readytochat.realvuxbaro.me/api/history?conversation_id=ci-smoke-${{ github.sha }}")
|
||||
if [ "$code" = "200" ]; then
|
||||
echo "readytochat.realvuxbaro.me is live on ${{ github.sha }}"
|
||||
exit 0
|
||||
fi
|
||||
sleep 10
|
||||
done
|
||||
echo "readytochat.realvuxbaro.me did not pick up ${{ github.sha }} within 3 minutes"
|
||||
exit 1
|
||||
Reference in New Issue
Block a user