Stop the slugifier from deleting the letter D-stroke

2026-08-19 13:07:13 +07:00
parent 24c55d1627
commit 4490a1abf0
14 changed files with 739 additions and 198 deletions
@@ -0,0 +1,144 @@
+"""Run every eval dataset in `evals/` against a deployment, in one command.
+
+Before this, only `production_manual_60.jsonl` had a working runner. The other
+two datasets sat unreferenced by any code: one's intended consumer was a
+metrics dataclass with no entry point, the other's (`rag/run_eval.py`) had
+been unable to load the corpus since the ingestion artifact format changed.
+So two thirds of the checked-in eval coverage was never actually checked, and
+nothing in the repo said so.
+
+Reuses `run_manual_battery.py`'s checker rather than reimplementing it -- a
+second copy of the invariants would drift from the first, and the point of
+this script is to widen coverage, not to fork the definition of "correct".
+`evals/adapters.py` holds the per-dataset schema mapping and the reasoning
+behind each choice.
+
+Usage:
+    python3 scripts/run_all_evals.py --base-url https://realvuxbaro.me \\
+        --output-dir /tmp/evals
+"""
+
+from __future__ import annotations
+
+import argparse
+import importlib.util
+import json
+import sys
+import time
+from pathlib import Path
+from typing import Any
+
+SERVICE_ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(SERVICE_ROOT))
+
+from evals.adapters import SUITES  # noqa: E402
+
+
+def _load_battery_module():
+    """Import the sibling script by path -- `scripts/` is not a package."""
+    path = Path(__file__).resolve().parent / "run_manual_battery.py"
+    spec = importlib.util.spec_from_file_location("run_manual_battery", path)
+    module = importlib.util.module_from_spec(spec)
+    assert spec.loader is not None
+    spec.loader.exec_module(module)
+    return module
+
+
+def run_suite(
+    battery: Any,
+    name: str,
+    cases: list[dict[str, Any]],
+    endpoint: str,
+    timeout: float,
+    run_id: str,
+    output: Path,
+) -> tuple[int, int]:
+    passed = 0
+    with output.open("w", encoding="utf-8") as handle:
+        for index, case in enumerate(cases, start=1):
+            # Split out rather than nested: reusing the outer quote inside an
+            # f-string is a 3.12+ syntax, and ruff targets 3.11 here.
+            default_conversation = f"{name}-{case['id']}"
+            base = case.get("conversation_id") or default_conversation
+            conversation_id = f"{base}-{run_id}"
+            payload = {"content": case["query"], "conversationId": conversation_id}
+            started = time.monotonic()
+            try:
+                raw = battery._post(endpoint, payload, timeout)
+                response = battery._normalise_response(raw)
+                failures = battery._check(case, response)
+                error = None
+            except Exception as exc:  # noqa: BLE001 - recorded, not swallowed
+                response, failures, error = {}, [f"request error: {exc}"], repr(exc)
+            elapsed = round(time.monotonic() - started, 3)
+            ok = not failures
+            passed += int(ok)
+            handle.write(
+                json.dumps(
+                    {
+                        "suite": name,
+                        "case": case,
+                        "passed": ok,
+                        "failures": failures,
+                        "elapsed_seconds": elapsed,
+                        "error": error,
+                        "response": response,
+                    },
+                    ensure_ascii=False,
+                )
+                + "\n"
+            )
+            handle.flush()
+            print(
+                f"  [{index:02d}/{len(cases)}] {case['id']} "
+                f"{'PASS' if ok else 'FAIL'} {elapsed:.1f}s {'; '.join(failures)}",
+                flush=True,
+            )
+    return passed, len(cases)
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--base-url", required=True)
+    parser.add_argument("--output-dir", type=Path, required=True)
+    parser.add_argument("--timeout", type=float, default=120.0)
+    parser.add_argument("--suites", help="comma-separated subset of: " + ", ".join(SUITES))
+    parser.add_argument("--run-id", default=str(int(time.time())))
+    args = parser.parse_args()
+
+    wanted = (
+        [s.strip() for s in args.suites.split(",") if s.strip()]
+        if args.suites
+        else list(SUITES)
+    )
+    unknown = [s for s in wanted if s not in SUITES]
+    if unknown:
+        print(f"unknown suite(s): {unknown}; known: {list(SUITES)}", file=sys.stderr)
+        return 2
+
+    battery = _load_battery_module()
+    endpoint = args.base_url.rstrip("/") + "/api/chat"
+    args.output_dir.mkdir(parents=True, exist_ok=True)
+
+    totals: list[tuple[str, int, int]] = []
+    for name in wanted:
+        cases = SUITES[name]()
+        print(f"\n=== {name} ({len(cases)} cases) ===", flush=True)
+        passed, total = run_suite(
+            battery, name, cases, endpoint, args.timeout, args.run_id,
+            args.output_dir / f"{name}.jsonl",
+        )
+        totals.append((name, passed, total))
+
+    print("\n=== SUMMARY ===")
+    grand_passed = grand_total = 0
+    for name, passed, total in totals:
+        print(f"  {name:<16} {passed}/{total}")
+        grand_passed += passed
+        grand_total += total
+    print(f"  {'TOTAL':<16} {grand_passed}/{grand_total}")
+    return 0 if grand_passed == grand_total else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())