203 lines
8.6 KiB
Python
203 lines
8.6 KiB
Python
"""`rag/understanding.py::LlmQueryUnderstander` — zero coverage before this
|
|
(Codex's 2026-08-06 review, F-03/F-10), despite being the entry point for
|
|
every live turn once F-03 wired it in.
|
|
|
|
Also covers F-04 (bounded candidates): the resolver decides which drug_ids
|
|
are even plausible for a turn *before* the model runs, and the model's pick
|
|
is validated against that bound, not the full catalog.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
|
|
from rag.understanding import SECTION_KEY_HINTS, SECTION_KEYS, LlmQueryUnderstander
|
|
|
|
CATALOG = {
|
|
"paracetamol_acetaminophen": "paracetamol acetaminophen, PARACETAMOL",
|
|
"metformin": "metformin, METFORMIN",
|
|
}
|
|
|
|
|
|
class _FixedLlm:
|
|
def __init__(self, payload) -> None:
|
|
self._payload = payload
|
|
|
|
def generate(self, system: str, user: str, schema: dict) -> str:
|
|
if isinstance(self._payload, str):
|
|
return self._payload
|
|
return json.dumps(self._payload, ensure_ascii=False)
|
|
|
|
|
|
class _Resolution:
|
|
def __init__(self, status="not_found", drug_id=None, candidate_drug_ids=()) -> None:
|
|
self.status = status
|
|
self.drug_id = drug_id
|
|
self.candidate_drug_ids = candidate_drug_ids
|
|
|
|
|
|
class _FakeResolver:
|
|
"""Deterministic stand-in for `CatalogDrugResolver`: resolves a line to
|
|
a drug_id if one of `known`'s substrings appears in it (case-insensitive),
|
|
with no fuzzy suggestions unless `suggestions` is given."""
|
|
|
|
def __init__(self, known: dict[str, str], suggestions: dict[str, str] | None = None) -> None:
|
|
self._known = known
|
|
self._suggestions = suggestions or {}
|
|
|
|
def resolve(self, query: str) -> _Resolution:
|
|
low = query.lower()
|
|
for needle, drug_id in self._known.items():
|
|
if needle in low:
|
|
return _Resolution(status="resolved", drug_id=drug_id)
|
|
return _Resolution()
|
|
|
|
def suggest(self, query: str, k: int = 3, min_score: float = 0.5):
|
|
low = query.lower()
|
|
return [
|
|
(drug_id, 0.9) for needle, drug_id in self._suggestions.items() if needle in low
|
|
][:k]
|
|
|
|
|
|
RESOLVER = _FakeResolver({
|
|
"metformin": "metformin",
|
|
"paracetamol": "paracetamol_acetaminophen",
|
|
})
|
|
|
|
|
|
def test_drug_id_in_exact_underscore_form_resolves():
|
|
understander = LlmQueryUnderstander(_FixedLlm({
|
|
"turn_type": "drug_attribute", "drugs": ["metformin"],
|
|
"unknown_drugs": [], "attribute": None, "population": None,
|
|
"weight_kg": None, "age_text": None, "indication": None,
|
|
"needs_clarify": False, "clarify_reason": None,
|
|
}), CATALOG, RESOLVER)
|
|
frame = understander.understand("liều metformin")
|
|
assert frame.drugs == ("metformin",)
|
|
assert frame.unknown_drugs == ()
|
|
|
|
|
|
def test_drug_id_echoed_with_spaces_instead_of_underscores_still_resolves():
|
|
"""Reproduces the live 2026-08-06 bug on a genuine multi-turn shape: the
|
|
drug is named in an earlier turn (in history), the current turn is just
|
|
"30 cân", and the model echoed the spaced display name instead of the
|
|
underscored id. The old strict-equality check demoted a correctly
|
|
identified drug to unknown_drugs — producing "Không tìm thấy
|
|
paracetamol trong Dược thư" for a drug that plainly is in it."""
|
|
understander = LlmQueryUnderstander(_FixedLlm({
|
|
"turn_type": "dosing_calc", "drugs": ["paracetamol acetaminophen"],
|
|
"unknown_drugs": [], "attribute": None, "population": "tre_em",
|
|
"weight_kg": 30, "age_text": "7 tuổi", "indication": None,
|
|
"needs_clarify": False, "clarify_reason": None,
|
|
}), CATALOG, RESOLVER)
|
|
frame = understander.understand(
|
|
"30 cân",
|
|
history=("Người dùng: Liều paracetamol cho trẻ em", "Trợ lý: Bé mấy tuổi?"),
|
|
)
|
|
assert frame.drugs == ("paracetamol_acetaminophen",)
|
|
assert frame.unknown_drugs == ()
|
|
|
|
|
|
def test_a_genuinely_invented_name_is_unknown_not_substituted():
|
|
understander = LlmQueryUnderstander(_FixedLlm({
|
|
"turn_type": "drug_attribute", "drugs": [],
|
|
"unknown_drugs": ["aspirinol"], "attribute": None, "population": None,
|
|
"weight_kg": None, "age_text": None, "indication": None,
|
|
"needs_clarify": False, "clarify_reason": None,
|
|
}), CATALOG, RESOLVER)
|
|
frame = understander.understand("liều aspirinol")
|
|
assert frame.drugs == ()
|
|
assert frame.unknown_drugs == ("aspirinol",)
|
|
|
|
|
|
def test_a_name_with_no_deterministic_candidate_is_unknown_even_if_the_model_names_a_real_id():
|
|
"""F-04's actual guarantee: the model naming a *real* catalog id is not
|
|
enough — that id must also be among the turn's deterministic candidates.
|
|
Nothing in "liều aspirinol" fuzzy/exact-matches any real drug (per
|
|
RESOLVER), so even if the model output a real id here, it must be
|
|
rejected: the candidate bound, not just catalog membership, is what's
|
|
trusted."""
|
|
understander = LlmQueryUnderstander(_FixedLlm({
|
|
"turn_type": "drug_attribute", "drugs": ["metformin"],
|
|
"unknown_drugs": [], "attribute": None, "population": None,
|
|
"weight_kg": None, "age_text": None, "indication": None,
|
|
"needs_clarify": False, "clarify_reason": None,
|
|
}), CATALOG, RESOLVER)
|
|
frame = understander.understand("liều aspirinol")
|
|
assert frame.drugs == ()
|
|
assert "metformin" in frame.unknown_drugs
|
|
|
|
|
|
def test_fuzzy_suggestion_bounds_a_typo_into_the_candidate_set():
|
|
resolver = _FakeResolver({}, suggestions={"metfomin": "metformin"})
|
|
understander = LlmQueryUnderstander(_FixedLlm({
|
|
"turn_type": "drug_attribute", "drugs": ["metformin"],
|
|
"unknown_drugs": [], "attribute": None, "population": None,
|
|
"weight_kg": None, "age_text": None, "indication": None,
|
|
"needs_clarify": False, "clarify_reason": None,
|
|
}), CATALOG, resolver)
|
|
frame = understander.understand("liều metfomin")
|
|
assert frame.drugs == ("metformin",)
|
|
|
|
|
|
def test_malformed_json_fails_closed_to_a_clarify():
|
|
understander = LlmQueryUnderstander(_FixedLlm("not json"), CATALOG, RESOLVER)
|
|
frame = understander.understand("gì đó")
|
|
assert frame.turn_type == "out_of_scope"
|
|
assert frame.needs_clarify is True
|
|
|
|
|
|
def test_unrecognised_turn_type_falls_back_based_on_whether_a_drug_resolved():
|
|
understander = LlmQueryUnderstander(_FixedLlm({
|
|
"turn_type": "not_a_real_type", "drugs": ["metformin"],
|
|
"unknown_drugs": [], "attribute": None, "population": None,
|
|
"weight_kg": None, "age_text": None, "indication": None,
|
|
"needs_clarify": False, "clarify_reason": None,
|
|
}), CATALOG, RESOLVER)
|
|
frame = understander.understand("metformin")
|
|
assert frame.turn_type == "drug_attribute"
|
|
|
|
|
|
def test_every_section_key_has_a_hint():
|
|
# A key with no gloss shown to the model is exactly the bug this fixed —
|
|
# never let a new SECTION_KEYS entry silently ship without one.
|
|
assert set(SECTION_KEYS) == set(SECTION_KEY_HINTS)
|
|
|
|
|
|
def test_prompt_disambiguates_than_trong_from_chong_chi_dinh():
|
|
"""Reproduces the live 2026-08-06 golden-eval finding: a bare section
|
|
key list gave the model nothing to tell "thận trọng" (precautions) apart
|
|
from "chống chỉ định" (contraindications) — 9/9 live calls for "X cần
|
|
thận trọng gì?" picked chong_chi_dinh, silently answering from the wrong
|
|
section and dropping safety content the precautions section actually
|
|
has (metformin's lactic-acidosis warning, gentamicin's oto/nephrotoxicity).
|
|
Fixed with an inline gloss; this pins the gloss's presence in the actual
|
|
request sent, not just its existence in the hints dict."""
|
|
captured = {}
|
|
|
|
class _CapturingLlm:
|
|
def generate(self, system, user, schema):
|
|
captured["user"] = user
|
|
return json.dumps({
|
|
"turn_type": "drug_attribute", "drugs": ["metformin"],
|
|
"unknown_drugs": [], "attribute": "than_trong", "population": None,
|
|
"weight_kg": None, "age_text": None, "indication": None,
|
|
"needs_clarify": False, "clarify_reason": None,
|
|
})
|
|
|
|
understander = LlmQueryUnderstander(_CapturingLlm(), CATALOG, RESOLVER)
|
|
understander.understand("metformin cần thận trọng gì?")
|
|
|
|
assert "KHÁC chống chỉ định" in captured["user"]
|
|
assert "nhiễm toan lactic" in captured["user"]
|
|
|
|
|
|
def test_invalid_attribute_is_dropped_not_passed_through():
|
|
understander = LlmQueryUnderstander(_FixedLlm({
|
|
"turn_type": "drug_attribute", "drugs": ["metformin"],
|
|
"unknown_drugs": [], "attribute": "not_a_real_section", "population": None,
|
|
"weight_kg": None, "age_text": None, "indication": None,
|
|
"needs_clarify": False, "clarify_reason": None,
|
|
}), CATALOG, RESOLVER)
|
|
frame = understander.understand("metformin")
|
|
assert frame.attribute is None
|