Stop the slugifier from deleting the letter D-stroke
This commit is contained in:
@@ -0,0 +1,183 @@
|
||||
"""Normalise the three eval datasets onto the battery case schema.
|
||||
|
||||
`scripts/run_manual_battery.py` carries the only invariant checker that is
|
||||
actually exercised against a deployment, but it only understands
|
||||
`production_manual_60.jsonl`'s shape (`id`, `decision`, `must_have_citations`,
|
||||
…). The other two datasets were written for other runners and, as a result,
|
||||
had **no runner at all**:
|
||||
|
||||
- `condition_to_drug_v1.jsonl` uses `case_id` / `expected_intent` /
|
||||
`expected_relation` / `expected_clarification`. Its intended consumer,
|
||||
`rag/condition_evaluation.py`, is a metrics dataclass with no entry point
|
||||
-- only a unit test ever constructed one.
|
||||
- `manual_adversarial_hard10.jsonl` uses `case_id` / `expected_drug_id` /
|
||||
`expected_id`. Its intended consumer, `rag/run_eval.py`, could not run:
|
||||
it loads documents expecting `doc_id` / `kind` / `source_refs` /
|
||||
`parent_id`, while the corpus ingestion actually produces
|
||||
(`ingestion/data/processed/chunks.jsonl`) has `chunk_id` /
|
||||
`source_page_range` and none of those fields. It raised `KeyError:
|
||||
'doc_id'` on the first record, so it had been dead since the artifact
|
||||
format changed.
|
||||
|
||||
Both files therefore sat in the repo describing tests nobody could run. The
|
||||
mappings below are what makes them runnable, written out explicitly rather
|
||||
than buried in a one-off script, because a mapping that silently weakens an
|
||||
assertion is worse than no runner at all.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
EVALS_DIR = Path(__file__).resolve().parent
|
||||
|
||||
|
||||
def _read(path: Path) -> list[dict[str, Any]]:
|
||||
return [
|
||||
json.loads(line)
|
||||
for line in path.read_text(encoding="utf-8").splitlines()
|
||||
if line.strip()
|
||||
]
|
||||
|
||||
|
||||
def load_production_60() -> list[dict[str, Any]]:
|
||||
"""Already in the battery schema -- passed through untouched."""
|
||||
return _read(EVALS_DIR / "production_manual_60.jsonl")
|
||||
|
||||
|
||||
# Three cases in condition_to_drug_v1.jsonl expect behaviour the service
|
||||
# deliberately does not have. Verified against production 2026-08-19: in each
|
||||
# case the service is the safer of the two, so the expectation is corrected
|
||||
# here rather than the service being changed to match it.
|
||||
#
|
||||
# The reasons are pinned, not just the decisions. Asserting only "abstain"
|
||||
# would still pass if the service abstained for some unrelated reason -- and
|
||||
# the whole value of these three is that it declines for the RIGHT reason.
|
||||
_STALE_CONDITION_EXPECTATIONS: dict[str, dict[str, Any]] = {
|
||||
# "Thuốc nào gây tăng huyết áp?" -- a reverse lookup by adverse effect.
|
||||
# The dataset expects an answer; the service replies that reverse lookup
|
||||
# by cause/contraindication is not supported in this version. Answering
|
||||
# would mean reading the indication section to answer a question about
|
||||
# harm, which inverts the meaning of the source.
|
||||
"relation_adverse": {
|
||||
"decision": "abstain",
|
||||
"reason": "unsupported_reverse_relation",
|
||||
"no_citations": True,
|
||||
},
|
||||
# "Thuốc nào chống chỉ định ở bệnh nhân gout?" -- same inversion.
|
||||
"relation_contraindication": {
|
||||
"decision": "abstain",
|
||||
"reason": "unsupported_reverse_relation",
|
||||
"no_citations": True,
|
||||
},
|
||||
# "Liều amoxicillin?" -- no population given. The dataset expects a dose;
|
||||
# the service asks whether the patient is an adult or a child. Handing
|
||||
# over a dose here is the defect, not the clarification.
|
||||
"regression_dose": {
|
||||
"decision": "clarify",
|
||||
"reason": "missing_population",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def load_condition_20() -> list[dict[str, Any]]:
|
||||
"""`condition_to_drug_v1.jsonl` -> battery cases.
|
||||
|
||||
`expected_clarification` maps onto the decision directly: an ambiguous
|
||||
condition ("viêm gan", "ung thư") must be narrowed rather than answered
|
||||
with a drug list.
|
||||
|
||||
`condition_mode: general` is applied ONLY to the `general_*` cases. That
|
||||
flag asserts every citation comes from `chi_dinh`, which is right for a
|
||||
reverse indication lookup but wrong for the rest: `relation_adverse` and
|
||||
`relation_contraindication` are *supposed* to cite other sections, and the
|
||||
`patient_*` / `regression_*` cases legitimately cite dose, interaction or
|
||||
ADR. Applying it everywhere would manufacture failures that say nothing
|
||||
about correctness.
|
||||
"""
|
||||
cases = []
|
||||
for raw in _read(EVALS_DIR / "condition_to_drug_v1.jsonl"):
|
||||
case: dict[str, Any] = {
|
||||
"id": raw["case_id"],
|
||||
"query": raw["query"],
|
||||
"category": "condition_to_drug",
|
||||
}
|
||||
if raw.get("expected_clarification"):
|
||||
case["decision"] = "clarify"
|
||||
elif raw["case_id"] in _STALE_CONDITION_EXPECTATIONS:
|
||||
case.update(_STALE_CONDITION_EXPECTATIONS[raw["case_id"]])
|
||||
else:
|
||||
case["decision"] = "answerable"
|
||||
case["must_have_citations"] = True
|
||||
if raw["case_id"].startswith("general_") and raw.get("expected_relation") == "indication":
|
||||
case["condition_mode"] = "general"
|
||||
cases.append(case)
|
||||
return cases
|
||||
|
||||
|
||||
# Cases whose entire purpose is that the service REFUSES. Scoring them as
|
||||
# "should answer" inverts the test: passing would mean the guard is gone.
|
||||
_ADVERSARIAL_MUST_REFUSE: dict[str, dict[str, Any]] = {
|
||||
# "Liều famciclovir điều trị cho mèo là bao nhiêu?" -- the formulary covers
|
||||
# human medicine only. `expected_drug_id: famciclovir` in the source file
|
||||
# records which monograph the question brushes against, not a licence to
|
||||
# quote a dose from it.
|
||||
"unsupported-veterinary": {
|
||||
"decision": "abstain",
|
||||
"reason": "out_of_scope_non_human",
|
||||
"no_citations": True,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def load_adversarial_10() -> list[dict[str, Any]]:
|
||||
"""`manual_adversarial_hard10.jsonl` -> battery cases.
|
||||
|
||||
`expected_id` (a chunk id such as `p132_t0`) is deliberately dropped: it
|
||||
identifies a retrieval hit inside the index, which an HTTP response does
|
||||
not expose. `expected_drug_id` survives as `expected_any_drug_ids`, so the
|
||||
assertion that remains -- the answer is grounded in the right monograph --
|
||||
is one this transport can actually verify.
|
||||
|
||||
`decision_any` admits `clarify` because several of these are adversarial
|
||||
precisely by being under-specified (a paediatric dose with no age, an
|
||||
unsupported veterinary use); refusing to answer is a correct outcome
|
||||
there, not a failure.
|
||||
"""
|
||||
cases = []
|
||||
for raw in _read(EVALS_DIR / "manual_adversarial_hard10.jsonl"):
|
||||
case: dict[str, Any] = {
|
||||
"id": raw["case_id"],
|
||||
"query": raw["query"],
|
||||
"category": "adversarial_hard",
|
||||
# `verify_pdf` belongs here as a first-class success. Most of these
|
||||
# ask for a number that lives in a table or formula, and the
|
||||
# service is required to show the source crop and refuse to read
|
||||
# the figure out of it. Verified 2026-08-19: it returns
|
||||
# `visual_verification_required` while citing the correct
|
||||
# monograph, which is the designed outcome, not a miss.
|
||||
"decision_any": ["answerable", "clarify", "verify_pdf"],
|
||||
"must_have_citations_if_answerable": True,
|
||||
}
|
||||
if raw.get("expected_drug_id"):
|
||||
case["expected_any_drug_ids"] = [raw["expected_drug_id"]]
|
||||
cases.append(case)
|
||||
if raw["case_id"] in _ADVERSARIAL_MUST_REFUSE:
|
||||
case.update(_ADVERSARIAL_MUST_REFUSE[raw["case_id"]])
|
||||
# `decision_any` must go, not just be shadowed: the checker reads
|
||||
# `case.get("decision_any", [case.get("decision")])`, so leaving it
|
||||
# in place would keep admitting the answers this case exists to
|
||||
# forbid while the stricter `decision` sat there doing nothing.
|
||||
case.pop("decision_any", None)
|
||||
case.pop("expected_any_drug_ids", None)
|
||||
case.pop("must_have_citations_if_answerable", None)
|
||||
return cases
|
||||
|
||||
|
||||
SUITES = {
|
||||
"production60": load_production_60,
|
||||
"condition20": load_condition_20,
|
||||
"adversarial10": load_adversarial_10,
|
||||
}
|
||||
Reference in New Issue
Block a user