184 lines
8.0 KiB
Python
184 lines
8.0 KiB
Python
"""Normalise the three eval datasets onto the battery case schema.
|
|
|
|
`scripts/run_manual_battery.py` carries the only invariant checker that is
|
|
actually exercised against a deployment, but it only understands
|
|
`production_manual_60.jsonl`'s shape (`id`, `decision`, `must_have_citations`,
|
|
…). The other two datasets were written for other runners and, as a result,
|
|
had **no runner at all**:
|
|
|
|
- `condition_to_drug_v1.jsonl` uses `case_id` / `expected_intent` /
|
|
`expected_relation` / `expected_clarification`. Its intended consumer,
|
|
`rag/condition_evaluation.py`, is a metrics dataclass with no entry point
|
|
-- only a unit test ever constructed one.
|
|
- `manual_adversarial_hard10.jsonl` uses `case_id` / `expected_drug_id` /
|
|
`expected_id`. Its intended consumer, `rag/run_eval.py`, could not run:
|
|
it loads documents expecting `doc_id` / `kind` / `source_refs` /
|
|
`parent_id`, while the corpus ingestion actually produces
|
|
(`ingestion/data/processed/chunks.jsonl`) has `chunk_id` /
|
|
`source_page_range` and none of those fields. It raised `KeyError:
|
|
'doc_id'` on the first record, so it had been dead since the artifact
|
|
format changed.
|
|
|
|
Both files therefore sat in the repo describing tests nobody could run. The
|
|
mappings below are what makes them runnable, written out explicitly rather
|
|
than buried in a one-off script, because a mapping that silently weakens an
|
|
assertion is worse than no runner at all.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
EVALS_DIR = Path(__file__).resolve().parent
|
|
|
|
|
|
def _read(path: Path) -> list[dict[str, Any]]:
|
|
return [
|
|
json.loads(line)
|
|
for line in path.read_text(encoding="utf-8").splitlines()
|
|
if line.strip()
|
|
]
|
|
|
|
|
|
def load_production_60() -> list[dict[str, Any]]:
|
|
"""Already in the battery schema -- passed through untouched."""
|
|
return _read(EVALS_DIR / "production_manual_60.jsonl")
|
|
|
|
|
|
# Three cases in condition_to_drug_v1.jsonl expect behaviour the service
|
|
# deliberately does not have. Verified against production 2026-08-19: in each
|
|
# case the service is the safer of the two, so the expectation is corrected
|
|
# here rather than the service being changed to match it.
|
|
#
|
|
# The reasons are pinned, not just the decisions. Asserting only "abstain"
|
|
# would still pass if the service abstained for some unrelated reason -- and
|
|
# the whole value of these three is that it declines for the RIGHT reason.
|
|
_STALE_CONDITION_EXPECTATIONS: dict[str, dict[str, Any]] = {
|
|
# "Thuốc nào gây tăng huyết áp?" -- a reverse lookup by adverse effect.
|
|
# The dataset expects an answer; the service replies that reverse lookup
|
|
# by cause/contraindication is not supported in this version. Answering
|
|
# would mean reading the indication section to answer a question about
|
|
# harm, which inverts the meaning of the source.
|
|
"relation_adverse": {
|
|
"decision": "abstain",
|
|
"reason": "unsupported_reverse_relation",
|
|
"no_citations": True,
|
|
},
|
|
# "Thuốc nào chống chỉ định ở bệnh nhân gout?" -- same inversion.
|
|
"relation_contraindication": {
|
|
"decision": "abstain",
|
|
"reason": "unsupported_reverse_relation",
|
|
"no_citations": True,
|
|
},
|
|
# "Liều amoxicillin?" -- no population given. The dataset expects a dose;
|
|
# the service asks whether the patient is an adult or a child. Handing
|
|
# over a dose here is the defect, not the clarification.
|
|
"regression_dose": {
|
|
"decision": "clarify",
|
|
"reason": "missing_population",
|
|
},
|
|
}
|
|
|
|
|
|
def load_condition_20() -> list[dict[str, Any]]:
|
|
"""`condition_to_drug_v1.jsonl` -> battery cases.
|
|
|
|
`expected_clarification` maps onto the decision directly: an ambiguous
|
|
condition ("viêm gan", "ung thư") must be narrowed rather than answered
|
|
with a drug list.
|
|
|
|
`condition_mode: general` is applied ONLY to the `general_*` cases. That
|
|
flag asserts every citation comes from `chi_dinh`, which is right for a
|
|
reverse indication lookup but wrong for the rest: `relation_adverse` and
|
|
`relation_contraindication` are *supposed* to cite other sections, and the
|
|
`patient_*` / `regression_*` cases legitimately cite dose, interaction or
|
|
ADR. Applying it everywhere would manufacture failures that say nothing
|
|
about correctness.
|
|
"""
|
|
cases = []
|
|
for raw in _read(EVALS_DIR / "condition_to_drug_v1.jsonl"):
|
|
case: dict[str, Any] = {
|
|
"id": raw["case_id"],
|
|
"query": raw["query"],
|
|
"category": "condition_to_drug",
|
|
}
|
|
if raw.get("expected_clarification"):
|
|
case["decision"] = "clarify"
|
|
elif raw["case_id"] in _STALE_CONDITION_EXPECTATIONS:
|
|
case.update(_STALE_CONDITION_EXPECTATIONS[raw["case_id"]])
|
|
else:
|
|
case["decision"] = "answerable"
|
|
case["must_have_citations"] = True
|
|
if raw["case_id"].startswith("general_") and raw.get("expected_relation") == "indication":
|
|
case["condition_mode"] = "general"
|
|
cases.append(case)
|
|
return cases
|
|
|
|
|
|
# Cases whose entire purpose is that the service REFUSES. Scoring them as
|
|
# "should answer" inverts the test: passing would mean the guard is gone.
|
|
_ADVERSARIAL_MUST_REFUSE: dict[str, dict[str, Any]] = {
|
|
# "Liều famciclovir điều trị cho mèo là bao nhiêu?" -- the formulary covers
|
|
# human medicine only. `expected_drug_id: famciclovir` in the source file
|
|
# records which monograph the question brushes against, not a licence to
|
|
# quote a dose from it.
|
|
"unsupported-veterinary": {
|
|
"decision": "abstain",
|
|
"reason": "out_of_scope_non_human",
|
|
"no_citations": True,
|
|
},
|
|
}
|
|
|
|
|
|
def load_adversarial_10() -> list[dict[str, Any]]:
|
|
"""`manual_adversarial_hard10.jsonl` -> battery cases.
|
|
|
|
`expected_id` (a chunk id such as `p132_t0`) is deliberately dropped: it
|
|
identifies a retrieval hit inside the index, which an HTTP response does
|
|
not expose. `expected_drug_id` survives as `expected_any_drug_ids`, so the
|
|
assertion that remains -- the answer is grounded in the right monograph --
|
|
is one this transport can actually verify.
|
|
|
|
`decision_any` admits `clarify` because several of these are adversarial
|
|
precisely by being under-specified (a paediatric dose with no age, an
|
|
unsupported veterinary use); refusing to answer is a correct outcome
|
|
there, not a failure.
|
|
"""
|
|
cases = []
|
|
for raw in _read(EVALS_DIR / "manual_adversarial_hard10.jsonl"):
|
|
case: dict[str, Any] = {
|
|
"id": raw["case_id"],
|
|
"query": raw["query"],
|
|
"category": "adversarial_hard",
|
|
# `verify_pdf` belongs here as a first-class success. Most of these
|
|
# ask for a number that lives in a table or formula, and the
|
|
# service is required to show the source crop and refuse to read
|
|
# the figure out of it. Verified 2026-08-19: it returns
|
|
# `visual_verification_required` while citing the correct
|
|
# monograph, which is the designed outcome, not a miss.
|
|
"decision_any": ["answerable", "clarify", "verify_pdf"],
|
|
"must_have_citations_if_answerable": True,
|
|
}
|
|
if raw.get("expected_drug_id"):
|
|
case["expected_any_drug_ids"] = [raw["expected_drug_id"]]
|
|
cases.append(case)
|
|
if raw["case_id"] in _ADVERSARIAL_MUST_REFUSE:
|
|
case.update(_ADVERSARIAL_MUST_REFUSE[raw["case_id"]])
|
|
# `decision_any` must go, not just be shadowed: the checker reads
|
|
# `case.get("decision_any", [case.get("decision")])`, so leaving it
|
|
# in place would keep admitting the answers this case exists to
|
|
# forbid while the stricter `decision` sat there doing nothing.
|
|
case.pop("decision_any", None)
|
|
case.pop("expected_any_drug_ids", None)
|
|
case.pop("must_have_citations_if_answerable", None)
|
|
return cases
|
|
|
|
|
|
SUITES = {
|
|
"production60": load_production_60,
|
|
"condition20": load_condition_20,
|
|
"adversarial10": load_adversarial_10,
|
|
}
|