"""Run every eval dataset in `evals/` against a deployment, in one command.

Before this, only `production_manual_60.jsonl` had a working runner. The other
two datasets sat unreferenced by any code: one's intended consumer was a
metrics dataclass with no entry point, the other's (`rag/run_eval.py`) had
been unable to load the corpus since the ingestion artifact format changed.
So two thirds of the checked-in eval coverage was never actually checked, and
nothing in the repo said so.

Reuses `run_manual_battery.py`'s checker rather than reimplementing it -- a
second copy of the invariants would drift from the first, and the point of
this script is to widen coverage, not to fork the definition of "correct".
`evals/adapters.py` holds the per-dataset schema mapping and the reasoning
behind each choice.

Usage:
    python3 scripts/run_all_evals.py --base-url https://realvuxbaro.me \\
        --output-dir /tmp/evals
"""

from __future__ import annotations

import argparse
import importlib.util
import json
import sys
import time
from pathlib import Path
from typing import Any

SERVICE_ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(SERVICE_ROOT))

from evals.adapters import SUITES  # noqa: E402


def _load_battery_module():
    """Import the sibling script by path -- `scripts/` is not a package."""
    path = Path(__file__).resolve().parent / "run_manual_battery.py"
    spec = importlib.util.spec_from_file_location("run_manual_battery", path)
    module = importlib.util.module_from_spec(spec)
    assert spec.loader is not None
    spec.loader.exec_module(module)
    return module


def run_suite(
    battery: Any,
    name: str,
    cases: list[dict[str, Any]],
    endpoint: str,
    timeout: float,
    run_id: str,
    output: Path,
) -> tuple[int, int]:
    passed = 0
    with output.open("w", encoding="utf-8") as handle:
        for index, case in enumerate(cases, start=1):
            # Split out rather than nested: reusing the outer quote inside an
            # f-string is a 3.12+ syntax, and ruff targets 3.11 here.
            default_conversation = f"{name}-{case['id']}"
            base = case.get("conversation_id") or default_conversation
            conversation_id = f"{base}-{run_id}"
            payload = {"content": case["query"], "conversationId": conversation_id}
            started = time.monotonic()
            try:
                raw = battery._post(endpoint, payload, timeout)
                response = battery._normalise_response(raw)
                failures = battery._check(case, response)
                error = None
            except Exception as exc:  # noqa: BLE001 - recorded, not swallowed
                response, failures, error = {}, [f"request error: {exc}"], repr(exc)
            elapsed = round(time.monotonic() - started, 3)
            ok = not failures
            passed += int(ok)
            handle.write(
                json.dumps(
                    {
                        "suite": name,
                        "case": case,
                        "passed": ok,
                        "failures": failures,
                        "elapsed_seconds": elapsed,
                        "error": error,
                        "response": response,
                    },
                    ensure_ascii=False,
                )
                + "\n"
            )
            handle.flush()
            print(
                f"  [{index:02d}/{len(cases)}] {case['id']} "
                f"{'PASS' if ok else 'FAIL'} {elapsed:.1f}s {'; '.join(failures)}",
                flush=True,
            )
    return passed, len(cases)


def main() -> int:
    parser = argparse.ArgumentParser()
    parser.add_argument("--base-url", required=True)
    parser.add_argument("--output-dir", type=Path, required=True)
    parser.add_argument("--timeout", type=float, default=120.0)
    parser.add_argument("--suites", help="comma-separated subset of: " + ", ".join(SUITES))
    parser.add_argument("--run-id", default=str(int(time.time())))
    args = parser.parse_args()

    wanted = (
        [s.strip() for s in args.suites.split(",") if s.strip()]
        if args.suites
        else list(SUITES)
    )
    unknown = [s for s in wanted if s not in SUITES]
    if unknown:
        print(f"unknown suite(s): {unknown}; known: {list(SUITES)}", file=sys.stderr)
        return 2

    battery = _load_battery_module()
    endpoint = args.base_url.rstrip("/") + "/api/chat"
    args.output_dir.mkdir(parents=True, exist_ok=True)

    totals: list[tuple[str, int, int]] = []
    for name in wanted:
        cases = SUITES[name]()
        print(f"\n=== {name} ({len(cases)} cases) ===", flush=True)
        passed, total = run_suite(
            battery, name, cases, endpoint, args.timeout, args.run_id,
            args.output_dir / f"{name}.jsonl",
        )
        totals.append((name, passed, total))

    print("\n=== SUMMARY ===")
    grand_passed = grand_total = 0
    for name, passed, total in totals:
        print(f"  {name:<16} {passed}/{total}")
        grand_passed += passed
        grand_total += total
    print(f"  {'TOTAL':<16} {grand_passed}/{grand_total}")
    return 0 if grand_passed == grand_total else 1


if __name__ == "__main__":
    raise SystemExit(main())