Checkpoint frontend UI/UX overhaul and ingestion embed benchmark work

2026-08-06 17:21:21 +07:00
parent 1e8cbdb586
commit a4b8e1c4db
78 changed files with 6761 additions and 654 deletions
@@ -0,0 +1,202 @@
+"""`rag/understanding.py::LlmQueryUnderstander` — zero coverage before this
+(Codex's 2026-08-06 review, F-03/F-10), despite being the entry point for
+every live turn once F-03 wired it in.
+
+Also covers F-04 (bounded candidates): the resolver decides which drug_ids
+are even plausible for a turn *before* the model runs, and the model's pick
+is validated against that bound, not the full catalog.
+"""
+from __future__ import annotations
+
+import json
+
+from rag.understanding import SECTION_KEY_HINTS, SECTION_KEYS, LlmQueryUnderstander
+
+CATALOG = {
+    "paracetamol_acetaminophen": "paracetamol acetaminophen, PARACETAMOL",
+    "metformin": "metformin, METFORMIN",
+}
+
+
+class _FixedLlm:
+    def __init__(self, payload) -> None:
+        self._payload = payload
+
+    def generate(self, system: str, user: str, schema: dict) -> str:
+        if isinstance(self._payload, str):
+            return self._payload
+        return json.dumps(self._payload, ensure_ascii=False)
+
+
+class _Resolution:
+    def __init__(self, status="not_found", drug_id=None, candidate_drug_ids=()) -> None:
+        self.status = status
+        self.drug_id = drug_id
+        self.candidate_drug_ids = candidate_drug_ids
+
+
+class _FakeResolver:
+    """Deterministic stand-in for `CatalogDrugResolver`: resolves a line to
+    a drug_id if one of `known`'s substrings appears in it (case-insensitive),
+    with no fuzzy suggestions unless `suggestions` is given."""
+
+    def __init__(self, known: dict[str, str], suggestions: dict[str, str] | None = None) -> None:
+        self._known = known
+        self._suggestions = suggestions or {}
+
+    def resolve(self, query: str) -> _Resolution:
+        low = query.lower()
+        for needle, drug_id in self._known.items():
+            if needle in low:
+                return _Resolution(status="resolved", drug_id=drug_id)
+        return _Resolution()
+
+    def suggest(self, query: str, k: int = 3, min_score: float = 0.5):
+        low = query.lower()
+        return [
+            (drug_id, 0.9) for needle, drug_id in self._suggestions.items() if needle in low
+        ][:k]
+
+
+RESOLVER = _FakeResolver({
+    "metformin": "metformin",
+    "paracetamol": "paracetamol_acetaminophen",
+})
+
+
+def test_drug_id_in_exact_underscore_form_resolves():
+    understander = LlmQueryUnderstander(_FixedLlm({
+        "turn_type": "drug_attribute", "drugs": ["metformin"],
+        "unknown_drugs": [], "attribute": None, "population": None,
+        "weight_kg": None, "age_text": None, "indication": None,
+        "needs_clarify": False, "clarify_reason": None,
+    }), CATALOG, RESOLVER)
+    frame = understander.understand("liều metformin")
+    assert frame.drugs == ("metformin",)
+    assert frame.unknown_drugs == ()
+
+
+def test_drug_id_echoed_with_spaces_instead_of_underscores_still_resolves():
+    """Reproduces the live 2026-08-06 bug on a genuine multi-turn shape: the
+    drug is named in an earlier turn (in history), the current turn is just
+    "30 cân", and the model echoed the spaced display name instead of the
+    underscored id. The old strict-equality check demoted a correctly
+    identified drug to unknown_drugs — producing "Không tìm thấy
+    paracetamol trong Dược thư" for a drug that plainly is in it."""
+    understander = LlmQueryUnderstander(_FixedLlm({
+        "turn_type": "dosing_calc", "drugs": ["paracetamol acetaminophen"],
+        "unknown_drugs": [], "attribute": None, "population": "tre_em",
+        "weight_kg": 30, "age_text": "7 tuổi", "indication": None,
+        "needs_clarify": False, "clarify_reason": None,
+    }), CATALOG, RESOLVER)
+    frame = understander.understand(
+        "30 cân",
+        history=("Người dùng: Liều paracetamol cho trẻ em", "Trợ lý: Bé mấy tuổi?"),
+    )
+    assert frame.drugs == ("paracetamol_acetaminophen",)
+    assert frame.unknown_drugs == ()
+
+
+def test_a_genuinely_invented_name_is_unknown_not_substituted():
+    understander = LlmQueryUnderstander(_FixedLlm({
+        "turn_type": "drug_attribute", "drugs": [],
+        "unknown_drugs": ["aspirinol"], "attribute": None, "population": None,
+        "weight_kg": None, "age_text": None, "indication": None,
+        "needs_clarify": False, "clarify_reason": None,
+    }), CATALOG, RESOLVER)
+    frame = understander.understand("liều aspirinol")
+    assert frame.drugs == ()
+    assert frame.unknown_drugs == ("aspirinol",)
+
+
+def test_a_name_with_no_deterministic_candidate_is_unknown_even_if_the_model_names_a_real_id():
+    """F-04's actual guarantee: the model naming a *real* catalog id is not
+    enough — that id must also be among the turn's deterministic candidates.
+    Nothing in "liều aspirinol" fuzzy/exact-matches any real drug (per
+    RESOLVER), so even if the model output a real id here, it must be
+    rejected: the candidate bound, not just catalog membership, is what's
+    trusted."""
+    understander = LlmQueryUnderstander(_FixedLlm({
+        "turn_type": "drug_attribute", "drugs": ["metformin"],
+        "unknown_drugs": [], "attribute": None, "population": None,
+        "weight_kg": None, "age_text": None, "indication": None,
+        "needs_clarify": False, "clarify_reason": None,
+    }), CATALOG, RESOLVER)
+    frame = understander.understand("liều aspirinol")
+    assert frame.drugs == ()
+    assert "metformin" in frame.unknown_drugs
+
+
+def test_fuzzy_suggestion_bounds_a_typo_into_the_candidate_set():
+    resolver = _FakeResolver({}, suggestions={"metfomin": "metformin"})
+    understander = LlmQueryUnderstander(_FixedLlm({
+        "turn_type": "drug_attribute", "drugs": ["metformin"],
+        "unknown_drugs": [], "attribute": None, "population": None,
+        "weight_kg": None, "age_text": None, "indication": None,
+        "needs_clarify": False, "clarify_reason": None,
+    }), CATALOG, resolver)
+    frame = understander.understand("liều metfomin")
+    assert frame.drugs == ("metformin",)
+
+
+def test_malformed_json_fails_closed_to_a_clarify():
+    understander = LlmQueryUnderstander(_FixedLlm("not json"), CATALOG, RESOLVER)
+    frame = understander.understand("gì đó")
+    assert frame.turn_type == "out_of_scope"
+    assert frame.needs_clarify is True
+
+
+def test_unrecognised_turn_type_falls_back_based_on_whether_a_drug_resolved():
+    understander = LlmQueryUnderstander(_FixedLlm({
+        "turn_type": "not_a_real_type", "drugs": ["metformin"],
+        "unknown_drugs": [], "attribute": None, "population": None,
+        "weight_kg": None, "age_text": None, "indication": None,
+        "needs_clarify": False, "clarify_reason": None,
+    }), CATALOG, RESOLVER)
+    frame = understander.understand("metformin")
+    assert frame.turn_type == "drug_attribute"
+
+
+def test_every_section_key_has_a_hint():
+    # A key with no gloss shown to the model is exactly the bug this fixed —
+    # never let a new SECTION_KEYS entry silently ship without one.
+    assert set(SECTION_KEYS) == set(SECTION_KEY_HINTS)
+
+
+def test_prompt_disambiguates_than_trong_from_chong_chi_dinh():
+    """Reproduces the live 2026-08-06 golden-eval finding: a bare section
+    key list gave the model nothing to tell "thận trọng" (precautions) apart
+    from "chống chỉ định" (contraindications) — 9/9 live calls for "X cần
+    thận trọng gì?" picked chong_chi_dinh, silently answering from the wrong
+    section and dropping safety content the precautions section actually
+    has (metformin's lactic-acidosis warning, gentamicin's oto/nephrotoxicity).
+    Fixed with an inline gloss; this pins the gloss's presence in the actual
+    request sent, not just its existence in the hints dict."""
+    captured = {}
+
+    class _CapturingLlm:
+        def generate(self, system, user, schema):
+            captured["user"] = user
+            return json.dumps({
+                "turn_type": "drug_attribute", "drugs": ["metformin"],
+                "unknown_drugs": [], "attribute": "than_trong", "population": None,
+                "weight_kg": None, "age_text": None, "indication": None,
+                "needs_clarify": False, "clarify_reason": None,
+            })
+
+    understander = LlmQueryUnderstander(_CapturingLlm(), CATALOG, RESOLVER)
+    understander.understand("metformin cần thận trọng gì?")
+
+    assert "KHÁC chống chỉ định" in captured["user"]
+    assert "nhiễm toan lactic" in captured["user"]
+
+
+def test_invalid_attribute_is_dropped_not_passed_through():
+    understander = LlmQueryUnderstander(_FixedLlm({
+        "turn_type": "drug_attribute", "drugs": ["metformin"],
+        "unknown_drugs": [], "attribute": "not_a_real_section", "population": None,
+        "weight_kg": None, "age_text": None, "indication": None,
+        "needs_clarify": False, "clarify_reason": None,
+    }), CATALOG, RESOLVER)
+    frame = understander.understand("metformin")
+    assert frame.attribute is None