Stop the slugifier from deleting the letter D-stroke
This commit is contained in:
@@ -0,0 +1,183 @@
|
||||
"""Normalise the three eval datasets onto the battery case schema.
|
||||
|
||||
`scripts/run_manual_battery.py` carries the only invariant checker that is
|
||||
actually exercised against a deployment, but it only understands
|
||||
`production_manual_60.jsonl`'s shape (`id`, `decision`, `must_have_citations`,
|
||||
…). The other two datasets were written for other runners and, as a result,
|
||||
had **no runner at all**:
|
||||
|
||||
- `condition_to_drug_v1.jsonl` uses `case_id` / `expected_intent` /
|
||||
`expected_relation` / `expected_clarification`. Its intended consumer,
|
||||
`rag/condition_evaluation.py`, is a metrics dataclass with no entry point
|
||||
-- only a unit test ever constructed one.
|
||||
- `manual_adversarial_hard10.jsonl` uses `case_id` / `expected_drug_id` /
|
||||
`expected_id`. Its intended consumer, `rag/run_eval.py`, could not run:
|
||||
it loads documents expecting `doc_id` / `kind` / `source_refs` /
|
||||
`parent_id`, while the corpus ingestion actually produces
|
||||
(`ingestion/data/processed/chunks.jsonl`) has `chunk_id` /
|
||||
`source_page_range` and none of those fields. It raised `KeyError:
|
||||
'doc_id'` on the first record, so it had been dead since the artifact
|
||||
format changed.
|
||||
|
||||
Both files therefore sat in the repo describing tests nobody could run. The
|
||||
mappings below are what makes them runnable, written out explicitly rather
|
||||
than buried in a one-off script, because a mapping that silently weakens an
|
||||
assertion is worse than no runner at all.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
EVALS_DIR = Path(__file__).resolve().parent
|
||||
|
||||
|
||||
def _read(path: Path) -> list[dict[str, Any]]:
|
||||
return [
|
||||
json.loads(line)
|
||||
for line in path.read_text(encoding="utf-8").splitlines()
|
||||
if line.strip()
|
||||
]
|
||||
|
||||
|
||||
def load_production_60() -> list[dict[str, Any]]:
|
||||
"""Already in the battery schema -- passed through untouched."""
|
||||
return _read(EVALS_DIR / "production_manual_60.jsonl")
|
||||
|
||||
|
||||
# Three cases in condition_to_drug_v1.jsonl expect behaviour the service
|
||||
# deliberately does not have. Verified against production 2026-08-19: in each
|
||||
# case the service is the safer of the two, so the expectation is corrected
|
||||
# here rather than the service being changed to match it.
|
||||
#
|
||||
# The reasons are pinned, not just the decisions. Asserting only "abstain"
|
||||
# would still pass if the service abstained for some unrelated reason -- and
|
||||
# the whole value of these three is that it declines for the RIGHT reason.
|
||||
_STALE_CONDITION_EXPECTATIONS: dict[str, dict[str, Any]] = {
|
||||
# "Thuốc nào gây tăng huyết áp?" -- a reverse lookup by adverse effect.
|
||||
# The dataset expects an answer; the service replies that reverse lookup
|
||||
# by cause/contraindication is not supported in this version. Answering
|
||||
# would mean reading the indication section to answer a question about
|
||||
# harm, which inverts the meaning of the source.
|
||||
"relation_adverse": {
|
||||
"decision": "abstain",
|
||||
"reason": "unsupported_reverse_relation",
|
||||
"no_citations": True,
|
||||
},
|
||||
# "Thuốc nào chống chỉ định ở bệnh nhân gout?" -- same inversion.
|
||||
"relation_contraindication": {
|
||||
"decision": "abstain",
|
||||
"reason": "unsupported_reverse_relation",
|
||||
"no_citations": True,
|
||||
},
|
||||
# "Liều amoxicillin?" -- no population given. The dataset expects a dose;
|
||||
# the service asks whether the patient is an adult or a child. Handing
|
||||
# over a dose here is the defect, not the clarification.
|
||||
"regression_dose": {
|
||||
"decision": "clarify",
|
||||
"reason": "missing_population",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def load_condition_20() -> list[dict[str, Any]]:
|
||||
"""`condition_to_drug_v1.jsonl` -> battery cases.
|
||||
|
||||
`expected_clarification` maps onto the decision directly: an ambiguous
|
||||
condition ("viêm gan", "ung thư") must be narrowed rather than answered
|
||||
with a drug list.
|
||||
|
||||
`condition_mode: general` is applied ONLY to the `general_*` cases. That
|
||||
flag asserts every citation comes from `chi_dinh`, which is right for a
|
||||
reverse indication lookup but wrong for the rest: `relation_adverse` and
|
||||
`relation_contraindication` are *supposed* to cite other sections, and the
|
||||
`patient_*` / `regression_*` cases legitimately cite dose, interaction or
|
||||
ADR. Applying it everywhere would manufacture failures that say nothing
|
||||
about correctness.
|
||||
"""
|
||||
cases = []
|
||||
for raw in _read(EVALS_DIR / "condition_to_drug_v1.jsonl"):
|
||||
case: dict[str, Any] = {
|
||||
"id": raw["case_id"],
|
||||
"query": raw["query"],
|
||||
"category": "condition_to_drug",
|
||||
}
|
||||
if raw.get("expected_clarification"):
|
||||
case["decision"] = "clarify"
|
||||
elif raw["case_id"] in _STALE_CONDITION_EXPECTATIONS:
|
||||
case.update(_STALE_CONDITION_EXPECTATIONS[raw["case_id"]])
|
||||
else:
|
||||
case["decision"] = "answerable"
|
||||
case["must_have_citations"] = True
|
||||
if raw["case_id"].startswith("general_") and raw.get("expected_relation") == "indication":
|
||||
case["condition_mode"] = "general"
|
||||
cases.append(case)
|
||||
return cases
|
||||
|
||||
|
||||
# Cases whose entire purpose is that the service REFUSES. Scoring them as
|
||||
# "should answer" inverts the test: passing would mean the guard is gone.
|
||||
_ADVERSARIAL_MUST_REFUSE: dict[str, dict[str, Any]] = {
|
||||
# "Liều famciclovir điều trị cho mèo là bao nhiêu?" -- the formulary covers
|
||||
# human medicine only. `expected_drug_id: famciclovir` in the source file
|
||||
# records which monograph the question brushes against, not a licence to
|
||||
# quote a dose from it.
|
||||
"unsupported-veterinary": {
|
||||
"decision": "abstain",
|
||||
"reason": "out_of_scope_non_human",
|
||||
"no_citations": True,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def load_adversarial_10() -> list[dict[str, Any]]:
|
||||
"""`manual_adversarial_hard10.jsonl` -> battery cases.
|
||||
|
||||
`expected_id` (a chunk id such as `p132_t0`) is deliberately dropped: it
|
||||
identifies a retrieval hit inside the index, which an HTTP response does
|
||||
not expose. `expected_drug_id` survives as `expected_any_drug_ids`, so the
|
||||
assertion that remains -- the answer is grounded in the right monograph --
|
||||
is one this transport can actually verify.
|
||||
|
||||
`decision_any` admits `clarify` because several of these are adversarial
|
||||
precisely by being under-specified (a paediatric dose with no age, an
|
||||
unsupported veterinary use); refusing to answer is a correct outcome
|
||||
there, not a failure.
|
||||
"""
|
||||
cases = []
|
||||
for raw in _read(EVALS_DIR / "manual_adversarial_hard10.jsonl"):
|
||||
case: dict[str, Any] = {
|
||||
"id": raw["case_id"],
|
||||
"query": raw["query"],
|
||||
"category": "adversarial_hard",
|
||||
# `verify_pdf` belongs here as a first-class success. Most of these
|
||||
# ask for a number that lives in a table or formula, and the
|
||||
# service is required to show the source crop and refuse to read
|
||||
# the figure out of it. Verified 2026-08-19: it returns
|
||||
# `visual_verification_required` while citing the correct
|
||||
# monograph, which is the designed outcome, not a miss.
|
||||
"decision_any": ["answerable", "clarify", "verify_pdf"],
|
||||
"must_have_citations_if_answerable": True,
|
||||
}
|
||||
if raw.get("expected_drug_id"):
|
||||
case["expected_any_drug_ids"] = [raw["expected_drug_id"]]
|
||||
cases.append(case)
|
||||
if raw["case_id"] in _ADVERSARIAL_MUST_REFUSE:
|
||||
case.update(_ADVERSARIAL_MUST_REFUSE[raw["case_id"]])
|
||||
# `decision_any` must go, not just be shadowed: the checker reads
|
||||
# `case.get("decision_any", [case.get("decision")])`, so leaving it
|
||||
# in place would keep admitting the answers this case exists to
|
||||
# forbid while the stricter `decision` sat there doing nothing.
|
||||
case.pop("decision_any", None)
|
||||
case.pop("expected_any_drug_ids", None)
|
||||
case.pop("must_have_citations_if_answerable", None)
|
||||
return cases
|
||||
|
||||
|
||||
SUITES = {
|
||||
"production60": load_production_60,
|
||||
"condition20": load_condition_20,
|
||||
"adversarial10": load_adversarial_10,
|
||||
}
|
||||
@@ -1,97 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from .artifacts import build_drug_catalog, load_aliases, load_documents, load_parents
|
||||
from .evaluation import CaseOrigin, EvaluationCase, EvaluationOutcome, summarize
|
||||
from .in_memory import InMemoryLexicalRetriever, InMemoryParentStore
|
||||
from .models import EvidenceDecision, QueryIntent, SubjectScope
|
||||
from .routing import CatalogDrugResolver, QueryRoutingService
|
||||
from .service import EvidencePolicy, RetrievalService
|
||||
|
||||
|
||||
def read_cases(path: Path) -> list[EvaluationCase]:
|
||||
with path.open(encoding="utf-8") as handle:
|
||||
return [
|
||||
EvaluationCase(
|
||||
case_id=raw["case_id"],
|
||||
query=raw["query"],
|
||||
expected_drug_id=raw.get("expected_drug_id"),
|
||||
expected_id=raw.get("expected_id"),
|
||||
origin=CaseOrigin(raw["origin"]),
|
||||
subject_scope=SubjectScope(raw.get("subject_scope", "human")),
|
||||
)
|
||||
for line in handle
|
||||
if line.strip()
|
||||
for raw in [json.loads(line)]
|
||||
]
|
||||
|
||||
|
||||
def run(
|
||||
cases_path: Path,
|
||||
documents_path: Path,
|
||||
parents_path: Path,
|
||||
aliases_path: Path | None = None,
|
||||
) -> dict:
|
||||
documents = load_documents(documents_path)
|
||||
retrieval = RetrievalService(
|
||||
InMemoryLexicalRetriever(documents),
|
||||
InMemoryParentStore(load_parents(parents_path)),
|
||||
EvidencePolicy(),
|
||||
)
|
||||
service = QueryRoutingService(
|
||||
retrieval,
|
||||
CatalogDrugResolver(build_drug_catalog(documents, load_aliases(aliases_path))),
|
||||
)
|
||||
outcomes = []
|
||||
details = []
|
||||
for case in read_cases(cases_path):
|
||||
result = service.retrieve(
|
||||
case.query,
|
||||
case.subject_scope,
|
||||
QueryIntent.FACT_LOOKUP,
|
||||
)
|
||||
retrieved = (
|
||||
tuple(item.evidence_id for item in result.evidence)
|
||||
if result.decision != EvidenceDecision.ABSTAIN
|
||||
else ()
|
||||
)
|
||||
outcome = EvaluationOutcome(
|
||||
case=case,
|
||||
retrieved_ids=retrieved,
|
||||
resolved_drug_id=result.resolved_drug_id,
|
||||
drug_resolution_status=result.drug_resolution_status,
|
||||
)
|
||||
outcomes.append(outcome)
|
||||
details.append({
|
||||
"case_id": case.case_id,
|
||||
"passed": outcome.passed,
|
||||
"decision": result.decision,
|
||||
"reason": result.reason,
|
||||
"expected_id": case.expected_id,
|
||||
"retrieved_ids": retrieved,
|
||||
"expected_drug_id": case.expected_drug_id,
|
||||
"resolved_drug_id": result.resolved_drug_id,
|
||||
"drug_resolution_status": result.drug_resolution_status,
|
||||
})
|
||||
return {**summarize(outcomes), "details": details}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--cases", type=Path, required=True)
|
||||
parser.add_argument("--documents", type=Path, required=True)
|
||||
parser.add_argument("--parents", type=Path, required=True)
|
||||
parser.add_argument("--aliases", type=Path)
|
||||
args = parser.parse_args()
|
||||
print(json.dumps(
|
||||
run(args.cases, args.documents, args.parents, args.aliases),
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,144 @@
|
||||
"""Run every eval dataset in `evals/` against a deployment, in one command.
|
||||
|
||||
Before this, only `production_manual_60.jsonl` had a working runner. The other
|
||||
two datasets sat unreferenced by any code: one's intended consumer was a
|
||||
metrics dataclass with no entry point, the other's (`rag/run_eval.py`) had
|
||||
been unable to load the corpus since the ingestion artifact format changed.
|
||||
So two thirds of the checked-in eval coverage was never actually checked, and
|
||||
nothing in the repo said so.
|
||||
|
||||
Reuses `run_manual_battery.py`'s checker rather than reimplementing it -- a
|
||||
second copy of the invariants would drift from the first, and the point of
|
||||
this script is to widen coverage, not to fork the definition of "correct".
|
||||
`evals/adapters.py` holds the per-dataset schema mapping and the reasoning
|
||||
behind each choice.
|
||||
|
||||
Usage:
|
||||
python3 scripts/run_all_evals.py --base-url https://realvuxbaro.me \\
|
||||
--output-dir /tmp/evals
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import importlib.util
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
SERVICE_ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(SERVICE_ROOT))
|
||||
|
||||
from evals.adapters import SUITES # noqa: E402
|
||||
|
||||
|
||||
def _load_battery_module():
|
||||
"""Import the sibling script by path -- `scripts/` is not a package."""
|
||||
path = Path(__file__).resolve().parent / "run_manual_battery.py"
|
||||
spec = importlib.util.spec_from_file_location("run_manual_battery", path)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
assert spec.loader is not None
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def run_suite(
|
||||
battery: Any,
|
||||
name: str,
|
||||
cases: list[dict[str, Any]],
|
||||
endpoint: str,
|
||||
timeout: float,
|
||||
run_id: str,
|
||||
output: Path,
|
||||
) -> tuple[int, int]:
|
||||
passed = 0
|
||||
with output.open("w", encoding="utf-8") as handle:
|
||||
for index, case in enumerate(cases, start=1):
|
||||
# Split out rather than nested: reusing the outer quote inside an
|
||||
# f-string is a 3.12+ syntax, and ruff targets 3.11 here.
|
||||
default_conversation = f"{name}-{case['id']}"
|
||||
base = case.get("conversation_id") or default_conversation
|
||||
conversation_id = f"{base}-{run_id}"
|
||||
payload = {"content": case["query"], "conversationId": conversation_id}
|
||||
started = time.monotonic()
|
||||
try:
|
||||
raw = battery._post(endpoint, payload, timeout)
|
||||
response = battery._normalise_response(raw)
|
||||
failures = battery._check(case, response)
|
||||
error = None
|
||||
except Exception as exc: # noqa: BLE001 - recorded, not swallowed
|
||||
response, failures, error = {}, [f"request error: {exc}"], repr(exc)
|
||||
elapsed = round(time.monotonic() - started, 3)
|
||||
ok = not failures
|
||||
passed += int(ok)
|
||||
handle.write(
|
||||
json.dumps(
|
||||
{
|
||||
"suite": name,
|
||||
"case": case,
|
||||
"passed": ok,
|
||||
"failures": failures,
|
||||
"elapsed_seconds": elapsed,
|
||||
"error": error,
|
||||
"response": response,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
handle.flush()
|
||||
print(
|
||||
f" [{index:02d}/{len(cases)}] {case['id']} "
|
||||
f"{'PASS' if ok else 'FAIL'} {elapsed:.1f}s {'; '.join(failures)}",
|
||||
flush=True,
|
||||
)
|
||||
return passed, len(cases)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--base-url", required=True)
|
||||
parser.add_argument("--output-dir", type=Path, required=True)
|
||||
parser.add_argument("--timeout", type=float, default=120.0)
|
||||
parser.add_argument("--suites", help="comma-separated subset of: " + ", ".join(SUITES))
|
||||
parser.add_argument("--run-id", default=str(int(time.time())))
|
||||
args = parser.parse_args()
|
||||
|
||||
wanted = (
|
||||
[s.strip() for s in args.suites.split(",") if s.strip()]
|
||||
if args.suites
|
||||
else list(SUITES)
|
||||
)
|
||||
unknown = [s for s in wanted if s not in SUITES]
|
||||
if unknown:
|
||||
print(f"unknown suite(s): {unknown}; known: {list(SUITES)}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
battery = _load_battery_module()
|
||||
endpoint = args.base_url.rstrip("/") + "/api/chat"
|
||||
args.output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
totals: list[tuple[str, int, int]] = []
|
||||
for name in wanted:
|
||||
cases = SUITES[name]()
|
||||
print(f"\n=== {name} ({len(cases)} cases) ===", flush=True)
|
||||
passed, total = run_suite(
|
||||
battery, name, cases, endpoint, args.timeout, args.run_id,
|
||||
args.output_dir / f"{name}.jsonl",
|
||||
)
|
||||
totals.append((name, passed, total))
|
||||
|
||||
print("\n=== SUMMARY ===")
|
||||
grand_passed = grand_total = 0
|
||||
for name, passed, total in totals:
|
||||
print(f" {name:<16} {passed}/{total}")
|
||||
grand_passed += passed
|
||||
grand_total += total
|
||||
print(f" {'TOTAL':<16} {grand_passed}/{grand_total}")
|
||||
return 0 if grand_passed == grand_total else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user