Fix ai-service Dockerfile: bake in drug_entities.json, override its path
This commit is contained in:
@@ -8,18 +8,20 @@ Qdrant/Bedrock fixtures).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from rag.agent import RagAgent
|
||||
from rag.agent import MAX_CONSECUTIVE_CLARIFY, RagAgent
|
||||
from rag.answer import GroundedAnswerService
|
||||
from rag.metrics import GENERATION_REJECTED, InMemoryMetrics
|
||||
from rag.models import Evidence, EvidenceDecision, RetrievalResult, SourceRef
|
||||
from rag.understanding import QueryFrame
|
||||
|
||||
SOURCE = SourceRef(physical_page=100, precision="region", printed_page=100)
|
||||
|
||||
|
||||
def _evidence(text: str) -> Evidence:
|
||||
def _evidence(text: str, requires_visual_check: bool = False) -> Evidence:
|
||||
return Evidence(
|
||||
evidence_id="e0", matched_doc_id="e0", kind="prose", text=text, score=1.0,
|
||||
source_refs=(SOURCE,), hydrated_from_parent=False, requires_visual_check=False,
|
||||
source_refs=(SOURCE,), hydrated_from_parent=False,
|
||||
requires_visual_check=requires_visual_check,
|
||||
)
|
||||
|
||||
|
||||
@@ -27,7 +29,7 @@ class _FixedUnderstander:
|
||||
def __init__(self, frame: QueryFrame) -> None:
|
||||
self._frame = frame
|
||||
|
||||
def understand(self, turn, history=()):
|
||||
def understand(self, turn, history=(), budget=None, prior_frame=None):
|
||||
return self._frame
|
||||
|
||||
|
||||
@@ -36,20 +38,50 @@ class _FixedRetrieval:
|
||||
drug_id regardless of section/query, so these tests assert routing, not
|
||||
retrieval (that's `test_retrieval_service.py`'s job)."""
|
||||
|
||||
def __init__(self, results: dict[str, RetrievalResult]) -> None:
|
||||
def __init__(
|
||||
self,
|
||||
results: dict[str, RetrievalResult],
|
||||
indication_results: dict[str, RetrievalResult] | None = None,
|
||||
) -> None:
|
||||
self._results = results
|
||||
self._indication_results = indication_results or {}
|
||||
self.calls: list[tuple[str, str | None, str]] = []
|
||||
|
||||
def retrieve_framed(self, drug_id, section_key, query, is_overview=False):
|
||||
self.calls.append((drug_id, section_key, query))
|
||||
return self._results.get(
|
||||
drug_id, RetrievalResult(EvidenceDecision.ABSTAIN, "not_configured")
|
||||
)
|
||||
|
||||
def retrieve_by_indication(self, indication_text):
|
||||
return self._indication_results.get(
|
||||
indication_text, RetrievalResult(EvidenceDecision.ABSTAIN, "no_indication_match")
|
||||
)
|
||||
|
||||
def _agent(frame: QueryFrame, results: dict[str, RetrievalResult] | None = None) -> RagAgent:
|
||||
def decide(self, evidence):
|
||||
# Mirrors `RetrievalService.decide`'s real policy (not a stub that
|
||||
# always says ANSWERABLE) so `_interaction`'s use of it is actually
|
||||
# under test, not just its own call site.
|
||||
if not evidence:
|
||||
return RetrievalResult(EvidenceDecision.ABSTAIN, "parent_hydration_failed")
|
||||
if any(item.requires_visual_check for item in evidence):
|
||||
return RetrievalResult(EvidenceDecision.VERIFY_PDF, "visual_verification_required", evidence)
|
||||
return RetrievalResult(EvidenceDecision.ANSWERABLE, "grounded_evidence_available", evidence)
|
||||
|
||||
|
||||
def _agent(
|
||||
frame: QueryFrame,
|
||||
results: dict[str, RetrievalResult] | None = None,
|
||||
indication_results: dict[str, RetrievalResult] | None = None,
|
||||
) -> RagAgent:
|
||||
# `routing=None`: `answer_from_result` (the only method this path calls)
|
||||
# never touches it — see `rag/answer.py`.
|
||||
answers = GroundedAnswerService(routing=None)
|
||||
return RagAgent(_FixedUnderstander(frame), _FixedRetrieval(results or {}), answers)
|
||||
return RagAgent(
|
||||
_FixedUnderstander(frame),
|
||||
_FixedRetrieval(results or {}, indication_results),
|
||||
answers,
|
||||
)
|
||||
|
||||
|
||||
def test_smalltalk_does_not_touch_retrieval():
|
||||
@@ -100,6 +132,25 @@ def test_needs_clarify_frame_is_surfaced_directly():
|
||||
reply = agent.handle("liều paracetamol cho trẻ em")
|
||||
assert reply.decision == "clarify"
|
||||
assert reply.clarification == "Bé mấy tuổi, cân nặng bao nhiêu kg?"
|
||||
# No quick_replies on the frame -> none surfaced (a specific weight has
|
||||
# no clean short options; must not be fabricated downstream).
|
||||
assert reply.quick_replies == ()
|
||||
|
||||
|
||||
def test_needs_clarify_frame_carries_quick_replies_through():
|
||||
"""This is the path real traffic actually hits (checked live): the
|
||||
understanding LLM call itself sets needs_clarify/clarify_reason before
|
||||
retrieval ever runs, short-circuiting `_route` — quick_replies must
|
||||
survive that same short-circuit, not just the sufficiency-check path
|
||||
inside `GroundedAnswerService`."""
|
||||
agent = _agent(QueryFrame(
|
||||
turn_type="drug_attribute", drugs=("paracetamol",),
|
||||
needs_clarify=True, clarify_reason="Người lớn hay trẻ em?",
|
||||
quick_replies=("Người lớn", "Trẻ em"),
|
||||
))
|
||||
reply = agent.handle("liều paracetamol")
|
||||
assert reply.decision == "clarify"
|
||||
assert reply.quick_replies == ("Người lớn", "Trẻ em")
|
||||
|
||||
|
||||
def test_single_drug_attribute_retrieves_and_answers():
|
||||
@@ -118,6 +169,58 @@ def test_single_drug_attribute_retrieves_and_answers():
|
||||
assert "500 mg" in reply.answer
|
||||
|
||||
|
||||
def test_context_resolved_across_turns_is_folded_into_the_query():
|
||||
"""The 2026-08-07 P0 (named in the 2026-08-06 audit): population/weight/
|
||||
age/route are extracted by understanding.py but were never passed into
|
||||
retrieval or generation — so a reply like "Uống" three turns into a dose
|
||||
conversation reached `GroundedAnswerService` as literally just "Uống",
|
||||
with no notion that population=adult was already established. Fixed via
|
||||
`_synthesize_query`; this asserts the synthesized text — not the bare
|
||||
turn — is what retrieval and generation actually see."""
|
||||
result = RetrievalResult(
|
||||
EvidenceDecision.ANSWERABLE, "grounded_evidence_available",
|
||||
(_evidence("Liều uống người lớn: 500 mg."),
|
||||
_evidence("Liều tiêm người lớn: 1 g.")),
|
||||
resolved_drug_id="paracetamol_acetaminophen",
|
||||
)
|
||||
retrieval = _FixedRetrieval({"paracetamol_acetaminophen": result})
|
||||
agent = RagAgent(
|
||||
_FixedUnderstander(QueryFrame(
|
||||
turn_type="dosing_calc", drugs=("paracetamol_acetaminophen",),
|
||||
attribute="lieu_luong_va_cach_dung", population="nguoi_lon",
|
||||
route="uong", needs_clarify=False,
|
||||
)),
|
||||
retrieval,
|
||||
GroundedAnswerService(routing=None),
|
||||
)
|
||||
agent.handle("Uống")
|
||||
assert len(retrieval.calls) == 1
|
||||
_, _, query = retrieval.calls[0]
|
||||
assert query.startswith("Uống")
|
||||
assert "người lớn" in query
|
||||
assert "uống" in query.lower()
|
||||
|
||||
|
||||
def test_context_synthesis_is_a_no_op_when_the_frame_has_no_resolved_fields():
|
||||
"""A fresh, fully-specified single-shot question already states its own
|
||||
context — synthesis must not alter it or introduce redundant noise."""
|
||||
result = RetrievalResult(
|
||||
EvidenceDecision.ANSWERABLE, "grounded_evidence_available",
|
||||
(_evidence("x"), _evidence("y")), resolved_drug_id="metformin",
|
||||
)
|
||||
retrieval = _FixedRetrieval({"metformin": result})
|
||||
agent = RagAgent(
|
||||
_FixedUnderstander(QueryFrame(
|
||||
turn_type="drug_attribute", drugs=("metformin",),
|
||||
attribute="chong_chi_dinh",
|
||||
)),
|
||||
retrieval,
|
||||
GroundedAnswerService(routing=None),
|
||||
)
|
||||
agent.handle("Chống chỉ định của metformin là gì?")
|
||||
assert retrieval.calls[0][2] == "Chống chỉ định của metformin là gì?"
|
||||
|
||||
|
||||
def test_interaction_combines_both_drugs_evidence():
|
||||
warfarin = RetrievalResult(
|
||||
EvidenceDecision.ANSWERABLE, "grounded_evidence_available",
|
||||
@@ -137,6 +240,34 @@ def test_interaction_combines_both_drugs_evidence():
|
||||
assert "chảy máu" in reply.answer
|
||||
|
||||
|
||||
def test_interaction_with_one_drug_quarantined_never_answers_confidently():
|
||||
"""The P0 the 2026-08-06 audit found: `_interaction` used to keep only
|
||||
`ANSWERABLE` parts, so a quarantined drug's evidence (and the "table
|
||||
exists, verify PDF" notice the quarantine contract requires) was
|
||||
silently dropped — a confident interaction answer could omit a real
|
||||
unverified contraindication table for one of the two drugs. Fixed via
|
||||
`RetrievalService.decide` applied to the combined pool, the same policy
|
||||
the single-drug path already uses."""
|
||||
warfarin = RetrievalResult(
|
||||
EvidenceDecision.ANSWERABLE, "grounded_evidence_available",
|
||||
(_evidence("Tương tác với aspirin làm tăng nguy cơ chảy máu."),),
|
||||
)
|
||||
aspirin = RetrievalResult(
|
||||
EvidenceDecision.VERIFY_PDF, "visual_verification_required",
|
||||
(_evidence("Bảng tương tác cần đối chiếu PDF.", requires_visual_check=True),),
|
||||
)
|
||||
agent = _agent(
|
||||
QueryFrame(turn_type="interaction", drugs=("warfarin", "aspirin")),
|
||||
{"warfarin": warfarin, "aspirin": aspirin},
|
||||
)
|
||||
reply = agent.handle("warfarin với aspirin có dùng chung được không")
|
||||
# Must NOT be a confident "answerable" that silently omits aspirin's
|
||||
# quarantined table — must ask for PDF verification instead.
|
||||
assert reply.decision == "verify_pdf"
|
||||
# Both drugs' evidence must still be present (as citations), not dropped.
|
||||
assert len(reply.citations) == 2
|
||||
|
||||
|
||||
def test_interaction_with_no_evidence_abstains_and_does_not_imply_safety():
|
||||
agent = _agent(
|
||||
QueryFrame(turn_type="interaction", drugs=("drug_a", "drug_b")), {},
|
||||
@@ -147,18 +278,65 @@ def test_interaction_with_no_evidence_abstains_and_does_not_imply_safety():
|
||||
assert "KHÔNG có nghĩa là an toàn" in reply.answer
|
||||
|
||||
|
||||
def test_symptom_to_drug_without_a_drug_name_asks_honestly_not_wired_yet():
|
||||
agent = _agent(QueryFrame(turn_type="symptom_to_drug", indication="sốt cao"))
|
||||
reply = agent.handle("sốt cao uống thuốc gì")
|
||||
def test_symptom_to_drug_with_no_indication_extracted_asks_for_one():
|
||||
agent = _agent(QueryFrame(turn_type="symptom_to_drug"))
|
||||
reply = agent.handle("có thuốc gì không")
|
||||
assert reply.decision == "clarify"
|
||||
assert reply.reason == "reverse_lookup_not_ready"
|
||||
assert reply.reason == "no_indication"
|
||||
|
||||
|
||||
def test_symptom_to_drug_with_no_match_abstains_not_silently_safe():
|
||||
agent = _agent(
|
||||
QueryFrame(turn_type="symptom_to_drug", indication="bệnh hiếm gặp x"),
|
||||
indication_results={
|
||||
"bệnh hiếm gặp x": RetrievalResult(EvidenceDecision.ABSTAIN, "no_indication_match"),
|
||||
},
|
||||
)
|
||||
reply = agent.handle("thuốc gì trị bệnh hiếm gặp x")
|
||||
assert reply.decision == "abstain"
|
||||
assert "bệnh hiếm gặp x" in reply.answer
|
||||
assert "KHÔNG" in reply.answer
|
||||
|
||||
|
||||
def test_symptom_to_drug_returns_the_matched_drugs_not_frame_drugs():
|
||||
"""`frame.drugs` is empty by construction for this turn_type (the router
|
||||
only reaches `_symptom_to_drug` with no named drug) — the reply's
|
||||
`drugs` must come from what retrieval actually found."""
|
||||
result = RetrievalResult(
|
||||
EvidenceDecision.ANSWERABLE, "grounded_evidence_available",
|
||||
(_evidence("Paracetamol chỉ định hạ sốt."),
|
||||
_evidence("Ibuprofen chỉ định hạ sốt, giảm đau.")),
|
||||
)
|
||||
# Overwrite matched_doc_id per evidence to simulate two different drugs
|
||||
# (the shared `_evidence` helper always uses "e0" — construct directly).
|
||||
ev_a = Evidence(
|
||||
evidence_id="paracetamol_acetaminophen__chi_dinh__0",
|
||||
matched_doc_id="paracetamol_acetaminophen__chi_dinh__0",
|
||||
kind="prose", text="Paracetamol chỉ định hạ sốt.", score=1.0,
|
||||
source_refs=(SOURCE,), hydrated_from_parent=False, requires_visual_check=False,
|
||||
)
|
||||
ev_b = Evidence(
|
||||
evidence_id="ibuprofen__chi_dinh__0", matched_doc_id="ibuprofen__chi_dinh__0",
|
||||
kind="prose", text="Ibuprofen chỉ định hạ sốt, giảm đau.", score=1.0,
|
||||
source_refs=(SOURCE,), hydrated_from_parent=False, requires_visual_check=False,
|
||||
)
|
||||
result = RetrievalResult(
|
||||
EvidenceDecision.ANSWERABLE, "grounded_evidence_available", (ev_a, ev_b),
|
||||
)
|
||||
agent = _agent(
|
||||
QueryFrame(turn_type="symptom_to_drug", indication="sốt"),
|
||||
indication_results={"sốt": result},
|
||||
)
|
||||
reply = agent.handle("sốt thì uống thuốc gì")
|
||||
assert reply.decision == "answerable"
|
||||
assert reply.drugs == ("paracetamol_acetaminophen", "ibuprofen")
|
||||
|
||||
|
||||
def test_history_is_passed_to_the_understander_on_the_next_turn():
|
||||
received_history: list[tuple[str, ...]] = []
|
||||
|
||||
class _RecordingUnderstander:
|
||||
def understand(self, turn, history=()):
|
||||
def understand(self, turn, history=(), budget=None, prior_frame=None):
|
||||
received_history.append(tuple(history))
|
||||
return QueryFrame(turn_type="smalltalk")
|
||||
|
||||
@@ -176,7 +354,7 @@ def test_history_is_isolated_per_conversation_id():
|
||||
received_history: list[tuple[str, ...]] = []
|
||||
|
||||
class _RecordingUnderstander:
|
||||
def understand(self, turn, history=()):
|
||||
def understand(self, turn, history=(), budget=None, prior_frame=None):
|
||||
received_history.append(tuple(history))
|
||||
return QueryFrame(turn_type="smalltalk")
|
||||
|
||||
@@ -190,6 +368,26 @@ def test_history_is_isolated_per_conversation_id():
|
||||
assert received_history[1] == ()
|
||||
|
||||
|
||||
# --- F-10: `conversation_id` presence/absence must not change the safety
|
||||
# decision on a fresh (first) turn — only whether the turn is remembered
|
||||
# afterward -------------------------------------------------------------
|
||||
|
||||
|
||||
def test_conversation_id_presence_or_absence_reaches_the_same_decision():
|
||||
"""A single-turn call (no `conversation_id`) and the first turn of a
|
||||
fresh multi-turn conversation must resolve identically — both see empty
|
||||
history, so nothing about `conversation_id` itself may become a second,
|
||||
undocumented safety signal."""
|
||||
agent = _agent(QueryFrame(
|
||||
turn_type="drug_attribute", unknown_drugs=("aspirinol",),
|
||||
))
|
||||
without_id = agent.handle("liều aspirinol")
|
||||
with_id = agent.handle("liều aspirinol", conversation_id="fresh-conv")
|
||||
|
||||
assert without_id.decision == with_id.decision == "abstain"
|
||||
assert without_id.reason == with_id.reason == "drug_not_in_formulary"
|
||||
|
||||
|
||||
def test_autocomplete_delegates_to_the_configured_source():
|
||||
class _Source:
|
||||
def complete(self, prefix, k):
|
||||
@@ -210,3 +408,291 @@ def test_autocomplete_with_no_source_configured_returns_empty():
|
||||
_FixedRetrieval({}), answers,
|
||||
)
|
||||
assert agent.complete("met") == []
|
||||
|
||||
|
||||
# --- F-08: a per-turn budget actually stops real provider calls once spent,
|
||||
# it isn't just bookkeeping ---------------------------------------------
|
||||
|
||||
|
||||
def test_a_budget_exhausted_during_understand_blocks_every_later_call():
|
||||
"""`max_llm_calls_per_turn=1` means the (simulated) `understand()` call
|
||||
spends the entire turn's budget — sufficiency and generate must never
|
||||
reach the generator at all, not just receive an error from it. Proves
|
||||
the budget is threaded end to end through `RagAgent`, not only present
|
||||
at the one call site each unit test exercises in isolation."""
|
||||
class _BudgetSpendingUnderstander:
|
||||
"""Stands in for the real `LlmQueryUnderstander`, which calls
|
||||
`budget.require()` once before its own LLM call — simulated here so
|
||||
this test doesn't need a live-shaped LLM fake for the understand
|
||||
step, only for the reply's calls to fail budget."""
|
||||
|
||||
def understand(self, turn, history=(), budget=None, prior_frame=None):
|
||||
if budget is not None:
|
||||
budget.require()
|
||||
return QueryFrame(
|
||||
turn_type="drug_attribute", drugs=("metformin",),
|
||||
attribute="chi_dinh",
|
||||
)
|
||||
|
||||
result = RetrievalResult(
|
||||
EvidenceDecision.ANSWERABLE, "grounded_evidence_available",
|
||||
(_evidence("đoạn 0"), _evidence("đoạn 1")),
|
||||
)
|
||||
|
||||
generator_calls: list[dict] = []
|
||||
|
||||
class _CountingGenerator:
|
||||
def generate(self, system, user, schema):
|
||||
generator_calls.append(schema)
|
||||
return '{"answer": "unused", "evidence_sufficient": true, "clarifying_question": null}'
|
||||
|
||||
metrics = InMemoryMetrics()
|
||||
answers = GroundedAnswerService(
|
||||
routing=None, generator=_CountingGenerator(), metrics=metrics
|
||||
)
|
||||
agent = RagAgent(
|
||||
_BudgetSpendingUnderstander(), _FixedRetrieval({"metformin": result}), answers,
|
||||
max_llm_calls_per_turn=1,
|
||||
)
|
||||
|
||||
reply = agent.handle("liều metformin")
|
||||
|
||||
# The real point of F-08: no more actual provider calls happen once the
|
||||
# budget is spent — not "the generator returned an error", the generator
|
||||
# is never invoked at all.
|
||||
assert generator_calls == []
|
||||
assert reply.decision == "abstain"
|
||||
assert reply.answer is None
|
||||
assert metrics.total(GENERATION_REJECTED, reason="request_budget_exhausted") >= 1
|
||||
|
||||
|
||||
def test_a_generous_budget_does_not_change_normal_behaviour():
|
||||
"""Control for the test above: with the default (generous) budget, the
|
||||
same setup answers normally — proves the previous test's tiny budget is
|
||||
what caused the block, not some other change to the fixtures."""
|
||||
class _BudgetSpendingUnderstander:
|
||||
def understand(self, turn, history=(), budget=None, prior_frame=None):
|
||||
if budget is not None:
|
||||
budget.require()
|
||||
return QueryFrame(
|
||||
turn_type="drug_attribute", drugs=("metformin",),
|
||||
attribute="chi_dinh",
|
||||
)
|
||||
|
||||
result = RetrievalResult(
|
||||
EvidenceDecision.ANSWERABLE, "grounded_evidence_available",
|
||||
(_evidence("Liều 500 mg mỗi ngày."), _evidence("đoạn 1")),
|
||||
)
|
||||
|
||||
class _Generator:
|
||||
def generate(self, system, user, schema):
|
||||
if "sufficient" in schema.get("properties", {}):
|
||||
return '{"sufficient": true, "clarifying_question": null, "quick_replies": []}'
|
||||
if "entailed" in schema.get("properties", {}):
|
||||
return '{"entailed": true, "unsupported": []}'
|
||||
return '{"answer": "Liều 500 mg [1].", "evidence_sufficient": true, "clarifying_question": null}'
|
||||
|
||||
answers = GroundedAnswerService(routing=None, generator=_Generator())
|
||||
agent = RagAgent(
|
||||
_BudgetSpendingUnderstander(), _FixedRetrieval({"metformin": result}), answers,
|
||||
)
|
||||
|
||||
reply = agent.handle("liều metformin")
|
||||
|
||||
assert reply.decision == "answerable"
|
||||
assert reply.answer == "Liều 500 mg [1]."
|
||||
|
||||
|
||||
# --- durable conversation history (ADR 0008's named gap): an optional
|
||||
# `ConversationStore` replaces the in-process dict when configured --------
|
||||
|
||||
|
||||
class _FakeStore:
|
||||
def __init__(self) -> None:
|
||||
self.lines: dict[str, list[str]] = {}
|
||||
self.fail_reads = False
|
||||
self.fail_writes = False
|
||||
|
||||
def recent(self, conversation_id, limit):
|
||||
if self.fail_reads:
|
||||
raise ConnectionError("store unreachable")
|
||||
return self.lines.get(conversation_id, [])[-limit:]
|
||||
|
||||
def append(self, conversation_id, line):
|
||||
if self.fail_writes:
|
||||
raise ConnectionError("store unreachable")
|
||||
self.lines.setdefault(conversation_id, []).append(line)
|
||||
|
||||
|
||||
def test_history_round_trips_through_a_configured_store():
|
||||
received_history: list[tuple[str, ...]] = []
|
||||
|
||||
class _RecordingUnderstander:
|
||||
def understand(self, turn, history=(), budget=None, prior_frame=None):
|
||||
received_history.append(tuple(history))
|
||||
return QueryFrame(turn_type="smalltalk")
|
||||
|
||||
store = _FakeStore()
|
||||
answers = GroundedAnswerService(routing=None)
|
||||
agent = RagAgent(_RecordingUnderstander(), _FixedRetrieval({}), answers, store=store)
|
||||
|
||||
agent.handle("chào bạn", conversation_id="c1")
|
||||
agent.handle("còn liều thì sao?", conversation_id="c1")
|
||||
|
||||
assert received_history[0] == ()
|
||||
assert any("chào bạn" in line for line in received_history[1])
|
||||
assert store.lines["c1"] # actually persisted, not just read back in-process
|
||||
|
||||
|
||||
def test_store_read_failure_fails_open_to_fresh_history_not_a_crash():
|
||||
store = _FakeStore()
|
||||
store.lines["c1"] = ["Người dùng: câu cũ", "Trợ lý: trả lời cũ"]
|
||||
store.fail_reads = True
|
||||
|
||||
answers = GroundedAnswerService(routing=None)
|
||||
agent = RagAgent(
|
||||
_FixedUnderstander(QueryFrame(turn_type="smalltalk")),
|
||||
_FixedRetrieval({}), answers, store=store,
|
||||
)
|
||||
|
||||
# Must not raise -- degrades to no history for this turn.
|
||||
reply = agent.handle("chào bạn", conversation_id="c1")
|
||||
assert reply.decision == "answerable"
|
||||
|
||||
|
||||
def test_store_write_failure_fails_open_the_response_still_returns():
|
||||
store = _FakeStore()
|
||||
store.fail_writes = True
|
||||
|
||||
answers = GroundedAnswerService(routing=None)
|
||||
agent = RagAgent(
|
||||
_FixedUnderstander(QueryFrame(turn_type="smalltalk")),
|
||||
_FixedRetrieval({}), answers, store=store,
|
||||
)
|
||||
|
||||
# Must not raise, even though persisting this turn silently fails.
|
||||
reply = agent.handle("chào bạn", conversation_id="c1")
|
||||
assert reply.decision == "answerable"
|
||||
assert reply.answer is not None
|
||||
|
||||
|
||||
# --- F-11: the clarify-loop circuit breaker. Found live 2026-08-07
|
||||
# (50-question hand-typed browser audit): the understanding LLM can keep
|
||||
# deciding needs_clarify=true forever with no natural exit — reproduced 3
|
||||
# times independently, one case never converged after 5 real answered turns.
|
||||
# `understanding.py`'s prior-frame merge fixes most of the underlying cause,
|
||||
# but this is the code-level bound that guarantees a user is never stuck. --
|
||||
|
||||
|
||||
class _AlwaysClarifyUnderstander:
|
||||
"""Simulates a model stuck re-asking regardless of what the user
|
||||
answers — exactly the observed live failure, isolated from any real
|
||||
LLM's actual (variable) behaviour so this test is deterministic."""
|
||||
|
||||
def understand(self, turn, history=(), budget=None, prior_frame=None):
|
||||
return QueryFrame(
|
||||
turn_type="dosing_calc", drugs=("paracetamol_acetaminophen",),
|
||||
needs_clarify=True, clarify_reason="Bé nặng bao nhiêu kg?",
|
||||
)
|
||||
|
||||
|
||||
def test_clarify_loop_is_hard_stopped_after_max_consecutive_turns():
|
||||
answers = GroundedAnswerService(routing=None)
|
||||
agent = RagAgent(_AlwaysClarifyUnderstander(), _FixedRetrieval({}), answers)
|
||||
|
||||
replies = [
|
||||
agent.handle(f"turn {i}", conversation_id="stuck")
|
||||
for i in range(MAX_CONSECUTIVE_CLARIFY)
|
||||
]
|
||||
|
||||
# Every turn up to the last stays a genuine clarify...
|
||||
for reply in replies[:-1]:
|
||||
assert reply.decision == "clarify"
|
||||
# ...the Nth forces a hard stop instead of asking again.
|
||||
assert replies[-1].decision == "abstain"
|
||||
assert replies[-1].reason == "clarify_loop_exhausted"
|
||||
assert replies[-1].answer is not None
|
||||
|
||||
|
||||
def test_clarify_streak_resets_after_the_hard_stop_so_a_new_attempt_can_proceed():
|
||||
"""Confirms the breaker is a bounded pause, not a permanent lockout of
|
||||
the conversation id — the very next turn gets a fresh streak."""
|
||||
answers = GroundedAnswerService(routing=None)
|
||||
agent = RagAgent(_AlwaysClarifyUnderstander(), _FixedRetrieval({}), answers)
|
||||
|
||||
for i in range(MAX_CONSECUTIVE_CLARIFY):
|
||||
agent.handle(f"turn {i}", conversation_id="stuck")
|
||||
reply = agent.handle("one more try", conversation_id="stuck")
|
||||
|
||||
assert reply.decision == "clarify"
|
||||
|
||||
|
||||
def test_a_resolved_turn_resets_the_clarify_streak():
|
||||
"""An answerable turn in between two clarify runs must not let their
|
||||
streaks combine — only genuinely consecutive clarifies count."""
|
||||
result = RetrievalResult(
|
||||
EvidenceDecision.ANSWERABLE, "grounded_evidence_available",
|
||||
(_evidence("Liều 500 mg."),),
|
||||
)
|
||||
clarify_frame = QueryFrame(
|
||||
turn_type="dosing_calc", drugs=("paracetamol_acetaminophen",),
|
||||
needs_clarify=True, clarify_reason="Bé nặng bao nhiêu kg?",
|
||||
)
|
||||
resolved_frame = QueryFrame(
|
||||
turn_type="drug_attribute", drugs=("metformin",),
|
||||
attribute="lieu_luong_va_cach_dung",
|
||||
)
|
||||
# (MAX-1) clarifies, one resolved turn, then (MAX-1) clarifies again —
|
||||
# scripted explicitly so the test asserts the streak reset, not an
|
||||
# incidental side effect of some other call-counting scheme.
|
||||
script = (
|
||||
[clarify_frame] * (MAX_CONSECUTIVE_CLARIFY - 1)
|
||||
+ [resolved_frame]
|
||||
+ [clarify_frame] * (MAX_CONSECUTIVE_CLARIFY - 1)
|
||||
)
|
||||
|
||||
class _ScriptedUnderstander:
|
||||
def __init__(self, frames):
|
||||
self._frames = iter(frames)
|
||||
|
||||
def understand(self, turn, history=(), budget=None, prior_frame=None):
|
||||
return next(self._frames)
|
||||
|
||||
answers = GroundedAnswerService(routing=None)
|
||||
agent = RagAgent(
|
||||
_ScriptedUnderstander(script), _FixedRetrieval({"metformin": result}), answers,
|
||||
)
|
||||
|
||||
for i in range(MAX_CONSECUTIVE_CLARIFY - 1):
|
||||
reply = agent.handle(f"clarify {i}", conversation_id="c1")
|
||||
assert reply.decision == "clarify"
|
||||
resolved = agent.handle("answerable turn", conversation_id="c1")
|
||||
assert resolved.decision == "answerable"
|
||||
|
||||
# Streak was reset by the resolved turn -- this run of clarifies must
|
||||
# not be treated as a continuation of the earlier (pre-reset) run.
|
||||
for i in range(MAX_CONSECUTIVE_CLARIFY - 1):
|
||||
reply = agent.handle(f"clarify again {i}", conversation_id="c1")
|
||||
assert reply.decision == "clarify"
|
||||
|
||||
|
||||
def test_prior_frame_is_threaded_from_the_last_turn_to_the_understander():
|
||||
received: list[QueryFrame | None] = []
|
||||
|
||||
class _RecordingUnderstander:
|
||||
def understand(self, turn, history=(), budget=None, prior_frame=None):
|
||||
received.append(prior_frame)
|
||||
return QueryFrame(
|
||||
turn_type="dosing_calc", drugs=("paracetamol_acetaminophen",),
|
||||
needs_clarify=True, clarify_reason="Bé nặng bao nhiêu kg?",
|
||||
)
|
||||
|
||||
answers = GroundedAnswerService(routing=None)
|
||||
agent = RagAgent(_RecordingUnderstander(), _FixedRetrieval({}), answers)
|
||||
|
||||
agent.handle("liều paracetamol cho bé", conversation_id="c1")
|
||||
agent.handle("20kg", conversation_id="c1")
|
||||
|
||||
assert received[0] is None
|
||||
assert received[1] is not None
|
||||
assert received[1].clarify_reason == "Bé nặng bao nhiêu kg?"
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
"""`rag/budget.py::RequestBudget` — F-08."""
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
from rag.budget import RequestBudget, RequestBudgetExhausted
|
||||
from rag.ports import AnswerGenerationUnavailable
|
||||
|
||||
|
||||
def test_budget_exhausted_is_a_subclass_of_answer_generation_unavailable():
|
||||
"""Deliberate: every existing `except AnswerGenerationUnavailable:`
|
||||
fail-open/fail-closed handler in the codebase must catch this with zero
|
||||
changes, since it predates F-08 and already encodes the right behaviour
|
||||
for "the provider is unavailable to us right now"."""
|
||||
assert issubclass(RequestBudgetExhausted, AnswerGenerationUnavailable)
|
||||
|
||||
|
||||
def test_fresh_budget_has_budget():
|
||||
budget = RequestBudget.start(max_wall_clock_ms=20_000, max_calls=5)
|
||||
assert budget.has_budget() is True
|
||||
|
||||
|
||||
def test_require_spends_one_call():
|
||||
budget = RequestBudget.start(max_wall_clock_ms=20_000, max_calls=2)
|
||||
budget.require()
|
||||
assert budget.calls_remaining == 1
|
||||
budget.require()
|
||||
assert budget.calls_remaining == 0
|
||||
assert budget.has_budget() is False
|
||||
|
||||
|
||||
def test_require_raises_once_calls_are_exhausted():
|
||||
budget = RequestBudget.start(max_wall_clock_ms=20_000, max_calls=1)
|
||||
budget.require()
|
||||
with pytest.raises(RequestBudgetExhausted):
|
||||
budget.require()
|
||||
|
||||
|
||||
def test_require_raises_once_the_deadline_has_passed():
|
||||
budget = RequestBudget.start(max_wall_clock_ms=0, max_calls=100)
|
||||
time.sleep(0.01)
|
||||
assert budget.has_budget() is False
|
||||
with pytest.raises(RequestBudgetExhausted):
|
||||
budget.require()
|
||||
|
||||
|
||||
def test_a_failed_require_does_not_spend_a_call():
|
||||
"""`require()` raises before decrementing when there's no budget left —
|
||||
`calls_remaining` must not go negative, which would otherwise make a
|
||||
budget that's already exhausted look like it has "negative debt" instead
|
||||
of cleanly `0`."""
|
||||
budget = RequestBudget.start(max_wall_clock_ms=20_000, max_calls=1)
|
||||
budget.require()
|
||||
for _ in range(3):
|
||||
with pytest.raises(RequestBudgetExhausted):
|
||||
budget.require()
|
||||
assert budget.calls_remaining == 0
|
||||
@@ -16,6 +16,7 @@ from rag.models import (
|
||||
SourceRef,
|
||||
SubjectScope,
|
||||
)
|
||||
from rag.ports import AnswerGenerationUnavailable
|
||||
from rag.prompt import build_request
|
||||
|
||||
|
||||
@@ -41,22 +42,31 @@ class _Routing:
|
||||
|
||||
|
||||
class _Generator:
|
||||
def __init__(self, payload: dict, entailment_payload: dict | None = None) -> None:
|
||||
def __init__(
|
||||
self, payload: dict, entailment_payload: dict | None = None,
|
||||
sufficiency_payload: dict | None = None,
|
||||
) -> None:
|
||||
self._payload = payload
|
||||
self._entailment_payload = entailment_payload or {
|
||||
"entailed": True,
|
||||
"unsupported": [],
|
||||
}
|
||||
self._sufficiency_payload = sufficiency_payload
|
||||
|
||||
def generate(self, system: str, user: str, schema: dict) -> str: # noqa: ARG002
|
||||
# `_generate` also runs a post-generation entailment check; tell the
|
||||
# two request shapes apart by schema so callers here only need to
|
||||
# fake the main answer, not both.
|
||||
payload = (
|
||||
self._entailment_payload
|
||||
if "entailed" in schema.get("properties", {})
|
||||
else self._payload
|
||||
)
|
||||
# `_generate` also runs a post-generation entailment check, and
|
||||
# `_check_sufficiency` runs its own separate call before that — tell
|
||||
# the three request shapes apart by schema so a test asserting on one
|
||||
# doesn't have to also shape a payload for the others.
|
||||
props = schema.get("properties", {})
|
||||
if "entailed" in props:
|
||||
payload = self._entailment_payload
|
||||
elif "sufficient" in props and self._sufficiency_payload is not None:
|
||||
payload = self._sufficiency_payload
|
||||
else:
|
||||
payload = self._payload
|
||||
if isinstance(payload, BaseException):
|
||||
raise payload
|
||||
return json.dumps(payload, ensure_ascii=False)
|
||||
|
||||
|
||||
@@ -122,6 +132,42 @@ def test_underspecified_dose_asks_instead_of_dumping():
|
||||
assert "tuổi" in g.clarification
|
||||
assert g.answer == g.clarification
|
||||
assert g.generated is False
|
||||
assert g.quick_replies == ()
|
||||
|
||||
|
||||
def test_underspecified_dose_carries_quick_replies_when_the_model_offers_them():
|
||||
result = _answerable(_evidence(0, 100), _evidence(1, 200))
|
||||
gen = _Generator(
|
||||
{"sufficient": False,
|
||||
"clarifying_question": "Người lớn hay trẻ em?",
|
||||
"quick_replies": ["Người lớn", "Trẻ em"]}
|
||||
)
|
||||
service = GroundedAnswerService(_Routing(result), gen)
|
||||
|
||||
g = service.answer("paracetamol", SubjectScope.HUMAN, QueryIntent.FACT_LOOKUP)
|
||||
|
||||
assert g.quick_replies == ("Người lớn", "Trẻ em")
|
||||
|
||||
|
||||
def test_sufficiency_check_outage_fails_open_to_generation_not_abstain():
|
||||
"""F-10: `_check_sufficiency` documents (and this pins) a deliberate
|
||||
fail-OPEN on provider outage — unlike every other failure mode in this
|
||||
service, a sufficiency-check outage does not abstain, it just skips the
|
||||
clarify heuristic and lets grounding/entailment (tested elsewhere) be
|
||||
the real safety net on whatever gets generated next."""
|
||||
result = _answerable(_evidence(0, 100), _evidence(1, 200))
|
||||
gen = _Generator(
|
||||
{"answer": "Đoạn bằng chứng 0 [1].", "evidence_sufficient": True,
|
||||
"clarifying_question": None},
|
||||
sufficiency_payload=AnswerGenerationUnavailable("Bedrock unreachable"),
|
||||
)
|
||||
service = GroundedAnswerService(_Routing(result), gen)
|
||||
|
||||
g = service.answer("liều người lớn", SubjectScope.HUMAN, QueryIntent.FACT_LOOKUP)
|
||||
|
||||
assert g.clarification is None
|
||||
assert g.result.decision == EvidenceDecision.ANSWERABLE
|
||||
assert g.generated is True
|
||||
|
||||
|
||||
def test_sufficient_query_is_not_turned_into_a_clarification():
|
||||
@@ -144,3 +190,52 @@ def test_bare_name_builds_an_intro_prompt():
|
||||
normal = build_request("Liều?", ("đoạn A",), intro=False)
|
||||
assert "CÂU HỎI:" in normal.user
|
||||
assert "GIỚI THIỆU" not in normal.user
|
||||
|
||||
|
||||
def test_list_mode_prompt_instructs_enumerating_every_drug_not_ranking():
|
||||
"""2026-08-07 finding: without an explicit instruction, the model picked
|
||||
one drug out of 8 real symptom_to_drug matches and silently dropped the
|
||||
rest — verified live. `list_mode` closes that."""
|
||||
request = build_request("thuốc gì trị sốt", ("chỉ định A", "chỉ định B"), list_mode=True)
|
||||
assert "LIỆT KÊ TẤT CẢ" in request.user
|
||||
assert "không xếp hạng" in request.user or "KHÔNG" in request.user
|
||||
assert "CÂU HỎI:" in request.user
|
||||
|
||||
|
||||
def test_list_mode_skips_the_sufficiency_clarify():
|
||||
"""If the sufficiency call were NOT actually skipped, it would read
|
||||
`sufficiency_payload` (`sufficient=False`) and clarify. `list_mode=True`
|
||||
must never call it at all, so only the real answer payload is ever read."""
|
||||
result = _answerable(_evidence(0, 100), _evidence(1, 200))
|
||||
gen = _Generator(
|
||||
{"answer": "Đoạn bằng chứng 0 [1]. Đoạn bằng chứng 1 [2].",
|
||||
"evidence_sufficient": True, "clarifying_question": None},
|
||||
sufficiency_payload={
|
||||
"sufficient": False, "clarifying_question": "Loại nào?", "quick_replies": [],
|
||||
},
|
||||
)
|
||||
service = GroundedAnswerService(_Routing(result), gen)
|
||||
|
||||
g = service.answer_from_result("thuốc gì trị sốt", result, list_mode=True)
|
||||
|
||||
assert g.clarification is None
|
||||
assert g.answer is not None
|
||||
assert g.generated is True
|
||||
|
||||
|
||||
def test_without_list_mode_the_same_evidence_does_ask_for_clarification():
|
||||
"""Control for the test above: the same sufficiency payload, without
|
||||
`list_mode`, must actually clarify — proving the previous test's "not
|
||||
skipped" branch is reachable and would have failed loudly."""
|
||||
result = _answerable(_evidence(0, 100), _evidence(1, 200))
|
||||
gen = _Generator(
|
||||
{"answer": "unused", "evidence_sufficient": True, "clarifying_question": None},
|
||||
sufficiency_payload={
|
||||
"sufficient": False, "clarifying_question": "Loại nào?", "quick_replies": [],
|
||||
},
|
||||
)
|
||||
service = GroundedAnswerService(_Routing(result), gen)
|
||||
|
||||
g = service.answer_from_result("thuốc gì trị sốt", result, list_mode=False)
|
||||
|
||||
assert g.clarification == "Loại nào?"
|
||||
|
||||
@@ -1,189 +0,0 @@
|
||||
"""Follow-ups must inherit context, and must never inherit it silently.
|
||||
|
||||
The cases here are the ones the owner named on 2026-08-05: "còn trẻ em thì
|
||||
sao?", "giải thích kỹ hơn", and not making the user repeat themselves. The
|
||||
adversarial cases are the ones that make inheritance dangerous in a formulary
|
||||
— a stale drug, and an explicit mention being overridden by context.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from rag.conversation import (
|
||||
FOCUS_TTL_TURNS,
|
||||
ConversationState,
|
||||
Focus,
|
||||
Turn,
|
||||
detect_population,
|
||||
detect_verbosity,
|
||||
looks_like_followup,
|
||||
resolve_against,
|
||||
update_focus,
|
||||
)
|
||||
|
||||
|
||||
def _state(turn_count: int = 1, **focus_fields) -> ConversationState:
|
||||
focus = Focus()
|
||||
for name, value in focus_fields.items():
|
||||
focus = focus.with_field(name, value, turn_count - 1)
|
||||
return ConversationState("c1", focus=focus, turn_count=turn_count)
|
||||
|
||||
|
||||
# --- the follow-ups the owner asked for --------------------------------------
|
||||
|
||||
|
||||
def test_con_tre_em_thi_sao_inherits_drug_and_section():
|
||||
state = _state(drug_id="metformin", section_key="lieu_luong_va_cach_dung")
|
||||
|
||||
resolved = resolve_against(state, "còn trẻ em thì sao?", None, None)
|
||||
|
||||
assert resolved.drug_id == "metformin"
|
||||
assert resolved.section_key == "lieu_luong_va_cach_dung"
|
||||
assert resolved.population == "tre_em"
|
||||
assert resolved.inherited_drug is True
|
||||
|
||||
|
||||
def test_giai_thich_ky_hon_sets_verbosity_and_keeps_the_topic():
|
||||
state = _state(drug_id="warfarin", section_key="tuong_tac_thuoc")
|
||||
|
||||
resolved = resolve_against(state, "giải thích kỹ hơn", None, None)
|
||||
|
||||
assert resolved.drug_id == "warfarin"
|
||||
assert resolved.verbosity == "detailed"
|
||||
|
||||
|
||||
def test_the_user_is_not_made_to_repeat_the_drug():
|
||||
state = _state(drug_id="metformin")
|
||||
|
||||
resolved = resolve_against(state, "chống chỉ định", None, "chong_chi_dinh")
|
||||
|
||||
assert resolved.drug_id == "metformin"
|
||||
assert resolved.section_key == "chong_chi_dinh"
|
||||
|
||||
|
||||
# --- what makes inheritance safe ---------------------------------------------
|
||||
|
||||
|
||||
def test_an_explicit_drug_always_beats_context():
|
||||
"""Naming a drug must override whatever the conversation was about, or a
|
||||
deliberate topic change silently answers about the previous medicine."""
|
||||
state = _state(drug_id="metformin", section_key="lieu_luong_va_cach_dung")
|
||||
|
||||
resolved = resolve_against(state, "liều dùng warfarin", "warfarin", None)
|
||||
|
||||
assert resolved.drug_id == "warfarin"
|
||||
assert resolved.inherited_drug is False
|
||||
|
||||
|
||||
def test_a_stale_drug_is_dropped_rather_than_inherited():
|
||||
"""Beyond the TTL the drug is not context, it is a hazard."""
|
||||
state = _state(turn_count=FOCUS_TTL_TURNS + 3, drug_id="metformin")
|
||||
# `_state` stamps at turn_count - 1, so age is 1; age it past the TTL.
|
||||
aged = ConversationState(
|
||||
"c1",
|
||||
focus=Focus(drug_id="metformin", set_at_turn={"drug_id": 0}),
|
||||
turn_count=FOCUS_TTL_TURNS + 2,
|
||||
)
|
||||
|
||||
assert state.inherited("drug_id") == "metformin"
|
||||
assert aged.inherited("drug_id") is None
|
||||
|
||||
resolved = resolve_against(aged, "còn trẻ em thì sao?", None, None)
|
||||
assert resolved.drug_id is None
|
||||
|
||||
|
||||
def test_an_inherited_drug_must_be_named_in_the_answer():
|
||||
state = _state(drug_id="metformin")
|
||||
|
||||
inherited = resolve_against(state, "còn trẻ em thì sao?", None, None)
|
||||
explicit = resolve_against(state, "liều warfarin", "warfarin", None)
|
||||
|
||||
assert inherited.needs_carry_over_notice is True
|
||||
assert explicit.needs_carry_over_notice is False
|
||||
|
||||
|
||||
# --- phrase detection ---------------------------------------------------------
|
||||
|
||||
|
||||
def test_longest_population_phrase_wins():
|
||||
"""`phụ nữ cho con bú` must not be read as `phụ nữ`, and `trẻ sơ sinh`
|
||||
must not be read as `trẻ em` — the same rule `sections.py` relies on."""
|
||||
assert detect_population("phụ nữ cho con bú") == "phu_nu_cho_con_bu"
|
||||
assert detect_population("trẻ sơ sinh dùng sao") == "tre_so_sinh"
|
||||
assert detect_population("bà bầu uống được không") == "phu_nu_co_thai"
|
||||
assert detect_population("liều cho người lớn") == "nguoi_lon"
|
||||
|
||||
|
||||
def test_no_population_named_is_none_not_a_guess():
|
||||
assert detect_population("liều dùng paracetamol") is None
|
||||
assert detect_verbosity("liều dùng paracetamol") is None
|
||||
|
||||
|
||||
def test_followup_markers():
|
||||
assert looks_like_followup("còn trẻ em thì sao?") is True
|
||||
assert looks_like_followup("so với metformin thì sao") is True
|
||||
assert looks_like_followup("liều dùng paracetamol") is False
|
||||
|
||||
|
||||
# --- window and focus update --------------------------------------------------
|
||||
|
||||
|
||||
def test_recent_window_evicts_oldest():
|
||||
state = ConversationState("c1")
|
||||
for index in range(8):
|
||||
state = state.append(Turn("user", f"q{index}", "2026-08-05"), window=6)
|
||||
|
||||
assert len(state.recent) == 6
|
||||
assert state.recent[0].text == "q2"
|
||||
assert state.turn_count == 8
|
||||
|
||||
|
||||
def test_evicted_turns_reach_overflow_not_silently_dropped():
|
||||
"""Bug fixed 2026-08-06 (Codex review, F-06): `overflow()` used to check
|
||||
`len(self.recent) > window`, but `append()` already truncates `recent`
|
||||
to `window`, so that comparison could never be true — evicted turns
|
||||
never reached the summariser no matter how long a conversation ran.
|
||||
Exact repro from the review: 8 turns into a window of 6."""
|
||||
state = ConversationState("c1")
|
||||
for index in range(8):
|
||||
state = state.append(Turn("user", f"q{index}", "2026-08-05"), window=6)
|
||||
|
||||
overflow = state.overflow()
|
||||
|
||||
assert [turn.text for turn in overflow] == ["q0", "q1"]
|
||||
|
||||
|
||||
def test_overflow_accumulates_across_the_two_appends_one_turn_makes():
|
||||
"""A live turn typically calls `append()` twice in a row (user, then
|
||||
assistant). Each can evict at most one turn; the second call's overflow
|
||||
must not overwrite, and so lose, the first's."""
|
||||
state = ConversationState("c1")
|
||||
for index in range(6):
|
||||
state = state.append(Turn("user", f"q{index}", "2026-08-05"), window=6)
|
||||
assert state.overflow() == () # window exactly full, nothing evicted yet
|
||||
|
||||
state = state.append(Turn("user", "q6", "2026-08-05"), window=6)
|
||||
state = state.append(Turn("assistant", "a6", "2026-08-05"), window=6)
|
||||
|
||||
assert [turn.text for turn in state.overflow()] == ["q0", "q1"]
|
||||
|
||||
|
||||
def test_overflow_is_empty_again_after_the_caller_clears_it():
|
||||
from dataclasses import replace
|
||||
|
||||
state = ConversationState("c1")
|
||||
for index in range(8):
|
||||
state = state.append(Turn("user", f"q{index}", "2026-08-05"), window=6)
|
||||
assert state.overflow() != ()
|
||||
|
||||
state = replace(state, pending_overflow=())
|
||||
|
||||
assert state.overflow() == ()
|
||||
|
||||
|
||||
def test_focus_update_stamps_the_current_turn():
|
||||
state = _state(turn_count=3)
|
||||
|
||||
resolved = resolve_against(state, "liều dùng metformin", "metformin", "lieu_luong_va_cach_dung")
|
||||
focus = update_focus(state, resolved)
|
||||
|
||||
assert focus.drug_id == "metformin"
|
||||
assert focus.set_at_turn["drug_id"] == 3
|
||||
@@ -1,50 +0,0 @@
|
||||
from rag.conversation import (
|
||||
ConversationState,
|
||||
DeterministicSummariser,
|
||||
InMemoryConversationStore,
|
||||
Turn,
|
||||
)
|
||||
|
||||
|
||||
def test_store_returns_fresh_state_for_unknown_id():
|
||||
store = InMemoryConversationStore()
|
||||
state = store.load("conv-new")
|
||||
assert state.conversation_id == "conv-new"
|
||||
assert state.turn_count == 0
|
||||
assert state.recent == ()
|
||||
|
||||
|
||||
def test_store_round_trips_saved_state():
|
||||
store = InMemoryConversationStore()
|
||||
state = ConversationState("conv-1", summary="s", turn_count=3)
|
||||
store.save(state)
|
||||
assert store.load("conv-1") is state
|
||||
|
||||
|
||||
def test_summariser_records_topic_labels_only():
|
||||
s = DeterministicSummariser()
|
||||
dropped = (
|
||||
Turn("user", "Chống chỉ định của metformin?", "t0",
|
||||
drug_id="metformin", section_key="chong_chi_dinh"),
|
||||
Turn("assistant", "Quá mẫn với metformin, suy thận Clcr < 60...", "t1",
|
||||
drug_id="metformin", section_key="chong_chi_dinh"),
|
||||
)
|
||||
out = s.fold("", dropped)
|
||||
# The label line is present...
|
||||
assert "chong_chi_dinh của metformin" in out
|
||||
# ...and no clinical value leaked from the assistant turn.
|
||||
assert "Clcr" not in out
|
||||
assert "60" not in out
|
||||
|
||||
|
||||
def test_summariser_stays_within_budget_dropping_oldest():
|
||||
s = DeterministicSummariser()
|
||||
dropped = tuple(
|
||||
Turn("user", f"q{i}", f"t{i}", drug_id=f"drug{i}", section_key="lieu_luong")
|
||||
for i in range(400)
|
||||
)
|
||||
out = s.fold("", dropped)
|
||||
assert len(out) <= DeterministicSummariser.MAX_CHARS
|
||||
# Most-recent topic survives, oldest is dropped.
|
||||
assert "drug399" in out
|
||||
assert "drug0 " not in out
|
||||
@@ -1,133 +0,0 @@
|
||||
from rag.answer import GroundedAnswer
|
||||
from rag.conversation import DeterministicSummariser, InMemoryConversationStore
|
||||
from rag.conversational import (
|
||||
SMALLTALK_REPLY,
|
||||
ConversationalLoopService,
|
||||
)
|
||||
from rag.models import (
|
||||
Evidence,
|
||||
EvidenceDecision,
|
||||
QueryIntent,
|
||||
RetrievalResult,
|
||||
SubjectScope,
|
||||
)
|
||||
from rag.reasoning import ClarifyReason
|
||||
from rag.routing import CatalogDrugResolver
|
||||
from rag.sections import SectionResolver
|
||||
|
||||
|
||||
def _grounded(answer, evidence_text):
|
||||
ev = Evidence("e1", "e1", "prose", evidence_text, 1.0, (), False, False)
|
||||
result = RetrievalResult(
|
||||
EvidenceDecision.ANSWERABLE, "grounded_evidence_available",
|
||||
(ev,), "metformin", "resolved",
|
||||
)
|
||||
return GroundedAnswer(result, answer, (), False)
|
||||
|
||||
|
||||
class FakeAnswers:
|
||||
def __init__(self, answer_text, evidence_text):
|
||||
self._a = answer_text
|
||||
self._e = evidence_text
|
||||
self.calls = []
|
||||
|
||||
def answer(self, query, subject_scope, intent, drug_id=None):
|
||||
self.calls.append((query, drug_id))
|
||||
return _grounded(self._a, self._e)
|
||||
|
||||
|
||||
def _service(answers):
|
||||
return ConversationalLoopService(
|
||||
answers=answers,
|
||||
resolver=CatalogDrugResolver({"metformin": {"metformin"}}),
|
||||
section_resolver=SectionResolver(),
|
||||
store=InMemoryConversationStore(),
|
||||
summariser=DeterministicSummariser(),
|
||||
)
|
||||
|
||||
|
||||
def test_smalltalk_answers_socially_without_calling_engine():
|
||||
answers = FakeAnswers("x", "x")
|
||||
svc = _service(answers)
|
||||
out = svc.answer("c1", "chào bạn", SubjectScope.HUMAN, QueryIntent.FACT_LOOKUP)
|
||||
assert out.smalltalk is True
|
||||
assert out.answer == SMALLTALK_REPLY
|
||||
assert answers.calls == [] # a greeting is not a drug lookup
|
||||
|
||||
|
||||
def test_medical_turn_returns_grounded_answer():
|
||||
answers = FakeAnswers("Quá mẫn với metformin.", "Quá mẫn với metformin.")
|
||||
svc = _service(answers)
|
||||
out = svc.answer(
|
||||
"c2", "chống chỉ định metformin", SubjectScope.HUMAN, QueryIntent.FACT_LOOKUP
|
||||
)
|
||||
assert out.smalltalk is False
|
||||
assert out.answer == "Quá mẫn với metformin."
|
||||
assert out.grounded is not None
|
||||
|
||||
|
||||
def test_followup_inherits_drug_and_passes_it_resolved():
|
||||
answers = FakeAnswers(
|
||||
"Ở trẻ em điều chỉnh theo cân nặng.",
|
||||
"Ở trẻ em, liều metformin điều chỉnh theo cân nặng.",
|
||||
)
|
||||
svc = _service(answers)
|
||||
svc.answer("c3", "chống chỉ định metformin", SubjectScope.HUMAN, QueryIntent.FACT_LOOKUP)
|
||||
out = svc.answer("c3", "còn trẻ em thì sao?", SubjectScope.HUMAN, QueryIntent.FACT_LOOKUP)
|
||||
assert out.inherited_drug == "metformin"
|
||||
assert out.answer.startswith("Về Metformin:")
|
||||
# The inherited drug is passed already-resolved (not re-resolved from the
|
||||
# rewritten turn text), and the raw follow-up drives section routing.
|
||||
last_query, last_drug_id = answers.calls[-1]
|
||||
assert last_drug_id == "metformin"
|
||||
assert last_query == "còn trẻ em thì sao?"
|
||||
# State carried the drug forward.
|
||||
assert svc._store.load("c3").focus.drug_id == "metformin"
|
||||
|
||||
|
||||
def test_confirmation_is_not_fuzzy_matched_to_a_drug():
|
||||
"""'đúng' must not be fuzzy-matched to terbinafin/tretinoin (the did-you-mean
|
||||
loop the reviewer hit); it asks which drug instead."""
|
||||
answers = FakeAnswers("x", "x")
|
||||
svc = _service(answers)
|
||||
out = svc.answer("cc", "đúng", SubjectScope.HUMAN, QueryIntent.FACT_LOOKUP)
|
||||
assert out.clarification is not None
|
||||
assert out.clarification.reason == "confirm_without_context"
|
||||
assert answers.calls == []
|
||||
|
||||
|
||||
def test_a_long_sentence_that_names_no_drug_is_not_offered_did_you_mean():
|
||||
"""A full question ('EPO điều trị thiếu máu...') that resolves no drug is
|
||||
answered honestly, not with garbage suggestions from fuzzing the sentence."""
|
||||
answers = FakeAnswers("x", "x")
|
||||
svc = _service(answers) # catalog holds only metformin
|
||||
out = svc.answer(
|
||||
"cl", "EPO điều trị thiếu máu do hóa trị ung thư liều khởi đầu bao nhiêu",
|
||||
SubjectScope.HUMAN, QueryIntent.FACT_LOOKUP,
|
||||
)
|
||||
assert out.clarification is not None
|
||||
assert out.clarification.reason == "drug_not_supported"
|
||||
assert answers.calls == []
|
||||
|
||||
|
||||
def test_no_close_drug_reports_not_supported():
|
||||
answers = FakeAnswers("x", "x")
|
||||
svc = _service(answers) # catalog holds only metformin
|
||||
out = svc.answer("c4", "cái này thế nào?", SubjectScope.HUMAN, QueryIntent.FACT_LOOKUP)
|
||||
assert out.answer is None
|
||||
assert out.clarification is not None
|
||||
# Nothing close to a real drug: honest "not in the formulary", not a guess.
|
||||
assert out.clarification.reason == "drug_not_supported"
|
||||
assert answers.calls == []
|
||||
|
||||
|
||||
def test_typo_offers_did_you_mean_not_silent_resolution():
|
||||
answers = FakeAnswers("x", "x")
|
||||
svc = _service(answers) # catalog holds only metformin
|
||||
out = svc.answer("c5", "metformim", SubjectScope.HUMAN, QueryIntent.FACT_LOOKUP)
|
||||
# A near-miss is asked about, never auto-resolved on a similarity threshold.
|
||||
assert out.answer is None
|
||||
assert out.clarification is not None
|
||||
assert out.clarification.reason == "did_you_mean"
|
||||
assert "Metformin" in out.clarification.options
|
||||
assert answers.calls == []
|
||||
@@ -1,81 +0,0 @@
|
||||
from rag.conversation import DeterministicSummariser, InMemoryConversationStore
|
||||
from rag.conversational import ConversationalRagService, TurnResolution
|
||||
from rag.reasoning import ClarifyReason, MAX_LLM_CALLS, MAX_RETRIEVAL_ROUNDS, TurnBudget
|
||||
|
||||
|
||||
class FakeResolver:
|
||||
"""Maps a turn's text to what it resolves on its own (no context)."""
|
||||
|
||||
def __init__(self, table):
|
||||
self._table = table
|
||||
|
||||
def resolve_turn(self, text):
|
||||
for needle, resolution in self._table:
|
||||
if needle in text:
|
||||
return resolution
|
||||
return TurnResolution(drug_id=None, section_key=None, drug_status="not_found")
|
||||
|
||||
|
||||
def _service(resolver, retrieve, generate):
|
||||
return ConversationalRagService(
|
||||
store=InMemoryConversationStore(),
|
||||
summariser=DeterministicSummariser(),
|
||||
resolver=resolver,
|
||||
retrieve=retrieve,
|
||||
generate=generate,
|
||||
)
|
||||
|
||||
|
||||
def test_followup_inherits_drug_and_answer_names_it():
|
||||
resolver = FakeResolver([
|
||||
("metformin", TurnResolution("metformin", "chong_chi_dinh", "resolved")),
|
||||
# "còn trẻ em" names no drug on its own — must inherit.
|
||||
("trẻ em", TurnResolution(None, None, "not_found")),
|
||||
])
|
||||
# Evidence mentions "trẻ em" so the population assessor is satisfied.
|
||||
retrieve = lambda q: ("Ở trẻ em, liều metformin điều chỉnh theo cân nặng.",)
|
||||
generate = lambda q, ev, st: "liều theo cân nặng"
|
||||
svc = _service(resolver, retrieve, generate)
|
||||
|
||||
first = svc.answer("c1", "Chống chỉ định của metformin?")
|
||||
assert first.inherited_drug is None
|
||||
|
||||
second = svc.answer("c1", "còn trẻ em thì sao?")
|
||||
assert second.inherited_drug == "metformin"
|
||||
assert second.answer.startswith("Về metformin:")
|
||||
|
||||
|
||||
def test_no_drug_and_no_context_asks_without_spending_budget():
|
||||
resolver = FakeResolver([]) # nothing resolves
|
||||
calls = {"retrieve": 0, "generate": 0}
|
||||
|
||||
def retrieve(q):
|
||||
calls["retrieve"] += 1
|
||||
return ("x",)
|
||||
|
||||
def generate(q, ev, st):
|
||||
calls["generate"] += 1
|
||||
return "x"
|
||||
|
||||
svc = _service(resolver, retrieve, generate)
|
||||
budget = TurnBudget()
|
||||
out = svc.answer("c2", "cái này thế nào?", budget=budget)
|
||||
|
||||
assert out.answer is None
|
||||
assert out.clarification is not None
|
||||
assert out.clarification.reason == ClarifyReason.AMBIGUOUS_DRUG
|
||||
# Asking short-circuits before any spend.
|
||||
assert calls == {"retrieve": 0, "generate": 0}
|
||||
assert budget.retrieval_rounds == MAX_RETRIEVAL_ROUNDS
|
||||
assert budget.llm_calls == MAX_LLM_CALLS
|
||||
|
||||
|
||||
def test_state_persists_across_turns():
|
||||
resolver = FakeResolver([
|
||||
("metformin", TurnResolution("metformin", "chi_dinh", "resolved")),
|
||||
])
|
||||
svc = _service(resolver, lambda q: ("Chỉ định của metformin.",), lambda q, ev, st: "ok")
|
||||
svc.answer("c3", "chỉ định metformin?")
|
||||
state = svc._store.load("c3")
|
||||
assert state.turn_count == 2 # user + assistant
|
||||
assert state.focus.drug_id == "metformin"
|
||||
@@ -67,23 +67,28 @@ class _FixedRouting:
|
||||
class _Generator:
|
||||
"""Returns whatever payload the test wants the model to have produced.
|
||||
|
||||
`_generate` now makes up to four calls through this port: the main
|
||||
answer, a sufficiency check (skipped here — one evidence block), and up
|
||||
to two entailment calls (a reject retries once — live probing found the
|
||||
judge noisy on an identical claim/evidence pair). They're told apart by
|
||||
schema, so a test that only cares about the main answer doesn't also
|
||||
have to fake an entailment response by hand; `entailment_payload`
|
||||
overrides it when a test wants the entailment pass to reject. Pass a
|
||||
list of payloads to get a different answer on each successive
|
||||
entailment call (e.g. `[reject, accept]` for the retry-recovers case).
|
||||
`_generate` now makes up to five calls through this port: the main
|
||||
answer (a lone `evidence_sufficient: false` retries once — the same
|
||||
noisy-judge finding as entailment, live-confirmed 2026-08-07), a
|
||||
sufficiency check (skipped here — one evidence block), and up to three
|
||||
entailment calls (widened from two 2026-08-07: live probing found the
|
||||
judge noisy on an identical claim/evidence pair, and a real adversarial
|
||||
sample showed a single retry still discarding correct answers on the
|
||||
unlucky reject-reject draw). They're told apart by schema, so a test
|
||||
that only cares about one call doesn't have to fake the others; `payload`
|
||||
and `entailment_payload` each take either a fixed value or a list for a
|
||||
different answer on each successive call to that schema (e.g.
|
||||
`[reject, reject, accept]` for the third-attempt-recovers case).
|
||||
"""
|
||||
|
||||
def __init__(self, payload, entailment_payload=None) -> None:
|
||||
self._payload = payload
|
||||
payloads = payload
|
||||
self._payloads = list(payloads) if isinstance(payloads, list) else [payloads]
|
||||
self._call = 0
|
||||
default = {"entailed": True, "unsupported": []}
|
||||
payloads = entailment_payload if entailment_payload is not None else default
|
||||
e_payloads = entailment_payload if entailment_payload is not None else default
|
||||
self._entailment_payloads = (
|
||||
list(payloads) if isinstance(payloads, list) else [payloads]
|
||||
list(e_payloads) if isinstance(e_payloads, list) else [e_payloads]
|
||||
)
|
||||
self._entailment_call = 0
|
||||
|
||||
@@ -93,7 +98,9 @@ class _Generator:
|
||||
payload = self._entailment_payloads[index]
|
||||
self._entailment_call += 1
|
||||
else:
|
||||
payload = self._payload
|
||||
index = min(self._call, len(self._payloads) - 1)
|
||||
payload = self._payloads[index]
|
||||
self._call += 1
|
||||
if isinstance(payload, BaseException):
|
||||
raise payload
|
||||
if isinstance(payload, str):
|
||||
@@ -127,7 +134,12 @@ def test_invented_dose_is_refused_and_never_reaches_the_answer():
|
||||
# this is a real LLM chatbot, not the retired offline-extractive build).
|
||||
assert grounded.answer is None
|
||||
assert grounded.result.decision == EvidenceDecision.ABSTAIN
|
||||
assert grounded.result.reason == "generation_unavailable"
|
||||
# The specific check that rejected it, not a generic catch-all — found
|
||||
# live 2026-08-07: every rejection reason used to collapse into
|
||||
# "generation_unavailable" by the time it reached the API response,
|
||||
# making a real provider outage indistinguishable from ordinary
|
||||
# entailment noise without reading server metrics by hand.
|
||||
assert grounded.result.reason == "ungrounded_number"
|
||||
assert metrics.total(GENERATION_REJECTED, reason="ungrounded_number") == 1
|
||||
|
||||
|
||||
@@ -217,17 +229,42 @@ def test_entailment_retries_once_after_a_reject_and_a_later_accept_serves():
|
||||
assert metrics.total(GENERATION_SERVED) == 1
|
||||
|
||||
|
||||
def test_entailment_two_agreeing_rejects_still_discard():
|
||||
def test_entailment_recovers_on_third_attempt_after_two_rejects():
|
||||
"""The improvement 2026-08-07 widened the retry from 2 to 3 attempts
|
||||
after a live 50-question adversarial sample found the 2-attempt policy's
|
||||
own math (~11% false-discard rate on a genuinely valid claim, from the
|
||||
noise probed in the docstring above) matched the observed real
|
||||
abstention rate almost exactly. Two rejects followed by a real accept
|
||||
must now be served, not discarded."""
|
||||
grounded, metrics = _answer(
|
||||
{"answer": "Metformin dùng điều trị đái tháo đường [1].",
|
||||
"evidence_sufficient": True},
|
||||
entailment_payload=[
|
||||
{"entailed": False, "unsupported": [1]},
|
||||
{"entailed": False, "unsupported": [1]},
|
||||
{"entailed": True, "unsupported": []},
|
||||
],
|
||||
)
|
||||
|
||||
assert grounded.generated is True
|
||||
assert metrics.total(GENERATION_SERVED) == 1
|
||||
assert metrics.total(GENERATION_REJECTED) == 0
|
||||
|
||||
|
||||
def test_entailment_three_agreeing_rejects_still_discard():
|
||||
grounded, metrics = _answer(
|
||||
{"answer": "Metformin chữa ung thư [1].", "evidence_sufficient": True},
|
||||
entailment_payload=[
|
||||
{"entailed": False, "unsupported": [1]},
|
||||
{"entailed": False, "unsupported": [1]},
|
||||
{"entailed": False, "unsupported": [1]},
|
||||
],
|
||||
)
|
||||
|
||||
assert grounded.generated is False
|
||||
assert grounded.answer is None
|
||||
# All 3 attempts are noisy-judge calls against the SAME claim/evidence —
|
||||
# a real, reliable rejection must still discard exactly once, not 3x.
|
||||
assert metrics.total(GENERATION_REJECTED, reason="unsupported_claim") == 1
|
||||
|
||||
|
||||
@@ -287,10 +324,46 @@ def test_every_generation_failure_abstains_instead_of_a_raw_source_dump(payload,
|
||||
assert grounded.generated is False
|
||||
assert grounded.answer is None
|
||||
assert grounded.result.decision == EvidenceDecision.ABSTAIN
|
||||
assert grounded.result.reason == "generation_unavailable"
|
||||
# The API/trace-visible reason must match the specific check that
|
||||
# failed, not a generic "generation_unavailable" for every cause —
|
||||
# otherwise a real outage and ordinary model noise are indistinguishable
|
||||
# from the outside (the exact gap a live report 2026-08-07 named).
|
||||
assert grounded.result.reason == reason
|
||||
assert metrics.total(GENERATION_REJECTED, reason=reason) == 1
|
||||
|
||||
|
||||
def test_evidence_insufficient_retries_once_and_recovers():
|
||||
"""Found live 2026-08-07 via a 50-question adversarial sample: a real
|
||||
section that plainly contains the answer (confirmed by re-asking the
|
||||
identical question 3/3 times successfully right after) still drew an
|
||||
`evidence_sufficient: false` self-judgment once — the same noisy-judge
|
||||
pattern already known for entailment, just on a different field of the
|
||||
same call. A lone insufficient verdict must not be final."""
|
||||
grounded, metrics = _answer([
|
||||
{"answer": "...", "evidence_sufficient": False},
|
||||
{"answer": "Metformin dùng điều trị đái tháo đường [1].",
|
||||
"evidence_sufficient": True},
|
||||
])
|
||||
|
||||
assert grounded.generated is True
|
||||
assert grounded.answer == "Metformin dùng điều trị đái tháo đường [1]."
|
||||
assert metrics.total(GENERATION_SERVED) == 1
|
||||
assert metrics.total(GENERATION_REJECTED) == 0
|
||||
|
||||
|
||||
def test_evidence_insufficient_twice_still_abstains():
|
||||
grounded, metrics = _answer([
|
||||
{"answer": "...", "evidence_sufficient": False},
|
||||
{"answer": "...", "evidence_sufficient": False},
|
||||
])
|
||||
|
||||
assert grounded.generated is False
|
||||
assert grounded.answer is None
|
||||
# Both attempts are the same noisy self-judgment on the same evidence —
|
||||
# a real, reliable "insufficient" must still discard exactly once.
|
||||
assert metrics.total(GENERATION_REJECTED, reason="evidence_insufficient") == 1
|
||||
|
||||
|
||||
def test_no_generator_configured_still_answers():
|
||||
service = GroundedAnswerService(_FixedRouting(_result()))
|
||||
|
||||
|
||||
@@ -18,6 +18,9 @@ ROOT = Path(__file__).resolve().parents[3]
|
||||
CHUNKS = ROOT / "ingestion/data/processed/chunks.jsonl"
|
||||
PDF = ROOT / "ingestion/data/raw/duoc-thu-quoc-gia-viet-nam-2018.pdf"
|
||||
MIGRATION = Path(__file__).resolve().parents[1] / "migrations/001_rag_retrieval_trace.sql"
|
||||
CONVERSATION_MIGRATION = (
|
||||
Path(__file__).resolve().parents[1] / "migrations/002_rag_conversation_turn.sql"
|
||||
)
|
||||
|
||||
|
||||
class _PlumbingEmbedder:
|
||||
@@ -131,6 +134,38 @@ def test_real_postgres_migration_insert_and_read_back():
|
||||
assert stored.citations[0]["printed_page_start"] == 101
|
||||
|
||||
|
||||
def test_real_postgres_conversation_store_round_trip():
|
||||
"""F-08's durable conversation history against a real Postgres, not a
|
||||
fake — proves `append`/`recent` actually persist and window correctly,
|
||||
the concrete capability this whole feature exists to add over the
|
||||
in-process dict it replaces."""
|
||||
from adapters.postgres import PostgresConversationStore
|
||||
|
||||
store = PostgresConversationStore(
|
||||
"postgresql://duoc_thu:duoc_thu@localhost:5432/duoc_thu"
|
||||
)
|
||||
store.migrate(CONVERSATION_MIGRATION)
|
||||
conversation_id = f"test-{uuid.uuid4()}"
|
||||
|
||||
store.append(conversation_id, "Người dùng: liều paracetamol")
|
||||
store.append(conversation_id, "Trợ lý: cần biết đối tượng")
|
||||
store.append(conversation_id, "Người dùng: người lớn")
|
||||
|
||||
assert store.recent(conversation_id, limit=10) == [
|
||||
"Người dùng: liều paracetamol",
|
||||
"Trợ lý: cần biết đối tượng",
|
||||
"Người dùng: người lớn",
|
||||
]
|
||||
# Windowing at read time: the oldest line falls outside a limit=2 read.
|
||||
assert store.recent(conversation_id, limit=2) == [
|
||||
"Trợ lý: cần biết đối tượng",
|
||||
"Người dùng: người lớn",
|
||||
]
|
||||
# A conversation_id that was never written to reads back empty, not an
|
||||
# error — the same "no history yet" case a fresh conversation hits live.
|
||||
assert store.recent(f"never-seen-{uuid.uuid4()}", limit=10) == []
|
||||
|
||||
|
||||
class _FakeJsonLlm:
|
||||
"""Deterministic stand-in for the Bedrock Converse generator. Satisfies
|
||||
both `JsonLlm` (query understanding) and `AnswerGenerator` (answer +
|
||||
|
||||
@@ -1,4 +1,36 @@
|
||||
from adapters.qdrant import _source_refs
|
||||
from adapters.qdrant import QdrantRetriever, _source_refs
|
||||
|
||||
|
||||
class _FakePoint:
|
||||
def __init__(self, payload: dict) -> None:
|
||||
self.payload = payload
|
||||
self.score = 1.0
|
||||
|
||||
|
||||
class _FakeScrollClient:
|
||||
"""Mimics qdrant-client's `.scroll()` shape closely enough to exercise
|
||||
`find_by_indication`'s keyword-matching logic directly — a fake filter
|
||||
(not a real one), so it returns every payload handed to it regardless
|
||||
of `scroll_filter`; the payloads given in each test already represent
|
||||
what a real `section_key=chi_dinh, chunk_kind=prose` filter would have
|
||||
returned, which is the part `find_by_indication` cannot get wrong on
|
||||
its own (the filter construction itself is a one-line, inspectable
|
||||
`Filter(must=[...])` — not worth a second fake layer to prove)."""
|
||||
|
||||
def __init__(self, payloads: list[dict]) -> None:
|
||||
self._payloads = payloads
|
||||
|
||||
def scroll(self, collection_name, scroll_filter, limit, offset, with_payload): # noqa: ARG002
|
||||
return [_FakePoint(p) for p in self._payloads], None
|
||||
|
||||
|
||||
def _chi_dinh_payload(drug_id: str, text: str) -> dict:
|
||||
return {
|
||||
"chunk_id": f"{drug_id}__chi_dinh__0", "drug_id": drug_id,
|
||||
"drug_name": drug_id.upper(), "section_key": "chi_dinh",
|
||||
"chunk_kind": "prose", "text": text,
|
||||
"heading_physical_page": 100, "printed_page_range": [101, 101],
|
||||
}
|
||||
|
||||
|
||||
def test_descriptor_source_ref_comes_from_attachment_not_heading_page():
|
||||
@@ -45,3 +77,85 @@ def test_prose_ref_uses_exact_chunk_range_and_keeps_attachment_region():
|
||||
assert refs[1].block_id == "p105_t0"
|
||||
assert refs[1].physical_page == 105
|
||||
assert refs[1].printed_page == 106
|
||||
|
||||
|
||||
def test_find_by_indication_matches_a_drug_that_names_the_symptom():
|
||||
client = _FakeScrollClient([
|
||||
_chi_dinh_payload("paracetamol_acetaminophen", "Điều trị sốt, giảm đau nhẹ và vừa."),
|
||||
_chi_dinh_payload("amoxicilin", "Điều trị nhiễm khuẩn đường hô hấp."),
|
||||
])
|
||||
retriever = QdrantRetriever(client, "duocthu_v1", embedder=None)
|
||||
|
||||
hits = retriever.find_by_indication("sốt", limit=8)
|
||||
|
||||
assert [h.document.drug_id for h in hits] == ["paracetamol_acetaminophen"]
|
||||
|
||||
|
||||
def test_find_by_indication_requires_the_whole_phrase_not_a_scattered_match():
|
||||
""""sốt xuất huyết" (dengue) must not match a chunk that only says "sốt"
|
||||
— the phrase itself has to appear, not just each of its words somewhere."""
|
||||
client = _FakeScrollClient([
|
||||
_chi_dinh_payload("paracetamol_acetaminophen", "Điều trị sốt, giảm đau."),
|
||||
])
|
||||
retriever = QdrantRetriever(client, "duocthu_v1", embedder=None)
|
||||
|
||||
hits = retriever.find_by_indication("sốt xuất huyết", limit=8)
|
||||
|
||||
assert hits == []
|
||||
|
||||
|
||||
def test_find_by_indication_matches_a_multi_word_phrase_contiguously():
|
||||
client = _FakeScrollClient([
|
||||
_chi_dinh_payload("paracetamol_acetaminophen", "Điều trị sốt cao ở người lớn."),
|
||||
])
|
||||
retriever = QdrantRetriever(client, "duocthu_v1", embedder=None)
|
||||
|
||||
hits = retriever.find_by_indication("sốt cao", limit=8)
|
||||
|
||||
assert [h.document.drug_id for h in hits] == ["paracetamol_acetaminophen"]
|
||||
|
||||
|
||||
def test_find_by_indication_rejects_a_scattered_bag_of_common_words():
|
||||
"""Found live 2026-08-07: a token-SUBSET match (every word present
|
||||
*somewhere*, any order) let a long nonsense phrase built from common
|
||||
filler words false-positive against real chi_dinh text — the words are
|
||||
common enough to appear scattered through nearly anything. Phrase
|
||||
matching closes it: none of these words are contiguous in the target
|
||||
text the way they are in the query."""
|
||||
client = _FakeScrollClient([
|
||||
_chi_dinh_payload(
|
||||
"paracetamol_acetaminophen",
|
||||
"Điều trị sốt. Không dùng quá liều khuyến cáo trong sách hướng dẫn.",
|
||||
),
|
||||
])
|
||||
retriever = QdrantRetriever(client, "duocthu_v1", embedder=None)
|
||||
|
||||
hits = retriever.find_by_indication(
|
||||
"bệnh chưa từng ghi nhận trong sách abcxyz123", limit=8
|
||||
)
|
||||
|
||||
assert hits == []
|
||||
|
||||
|
||||
def test_find_by_indication_returns_at_most_one_hit_per_drug():
|
||||
client = _FakeScrollClient([
|
||||
_chi_dinh_payload("paracetamol_acetaminophen", "Điều trị sốt."),
|
||||
{**_chi_dinh_payload("paracetamol_acetaminophen", "Điều trị sốt cao ở trẻ em."),
|
||||
"chunk_id": "paracetamol_acetaminophen__chi_dinh__1"},
|
||||
])
|
||||
retriever = QdrantRetriever(client, "duocthu_v1", embedder=None)
|
||||
|
||||
hits = retriever.find_by_indication("sốt", limit=8)
|
||||
|
||||
assert len(hits) == 1
|
||||
|
||||
|
||||
def test_find_by_indication_respects_the_limit():
|
||||
client = _FakeScrollClient([
|
||||
_chi_dinh_payload(f"drug_{i}", "Điều trị đau.") for i in range(5)
|
||||
])
|
||||
retriever = QdrantRetriever(client, "duocthu_v1", embedder=None)
|
||||
|
||||
hits = retriever.find_by_indication("đau", limit=2)
|
||||
|
||||
assert len(hits) == 2
|
||||
|
||||
@@ -1,244 +0,0 @@
|
||||
"""The loop must improve answers, and must be unable to run away.
|
||||
|
||||
Bounded is the load-bearing property: an unbounded self-improvement loop on a
|
||||
paid provider is a bill and a latency incident, and on a clinical tool it is
|
||||
also an answer nobody is waiting for any more.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from rag.conversation import ConversationState, ResolvedQuestion
|
||||
from rag.metrics import CLARIFY_ASKED, LOOP_REFINED, InMemoryMetrics
|
||||
from rag.reasoning import (
|
||||
ClarifyReason,
|
||||
DeterministicAssessor,
|
||||
LoopTrace,
|
||||
Sufficiency,
|
||||
TurnBudget,
|
||||
run_turn,
|
||||
)
|
||||
|
||||
ADULT = "Người lớn: uống 0,5 - 1 g/lần, cách 4 - 6 giờ; tối đa 4 g/ngày."
|
||||
CHILD = "Trẻ em 6 - 12 tuổi: 240 - 250 mg mỗi lần."
|
||||
|
||||
|
||||
def _q(text: str = "liều dùng paracetamol", population: str | None = None) -> ResolvedQuestion:
|
||||
return ResolvedQuestion(
|
||||
text=text,
|
||||
drug_id="paracetamol",
|
||||
section_key="lieu_luong_va_cach_dung",
|
||||
population=population,
|
||||
verbosity=None,
|
||||
inherited_drug=False,
|
||||
inherited_section=False,
|
||||
)
|
||||
|
||||
|
||||
def _state() -> ConversationState:
|
||||
return ConversationState("c1", turn_count=1)
|
||||
|
||||
|
||||
class _Retriever:
|
||||
"""Returns a different evidence set on each round, recording calls."""
|
||||
|
||||
def __init__(self, *rounds: tuple[str, ...]) -> None:
|
||||
self._rounds = list(rounds)
|
||||
self.queries: list[str] = []
|
||||
|
||||
def __call__(self, resolved: ResolvedQuestion) -> tuple[str, ...]:
|
||||
self.queries.append(resolved.text)
|
||||
if self._rounds:
|
||||
return self._rounds.pop(0)
|
||||
return ()
|
||||
|
||||
|
||||
def _generator(answer: str | None):
|
||||
calls = {"n": 0}
|
||||
|
||||
def generate(resolved, evidence, state):
|
||||
calls["n"] += 1
|
||||
return answer
|
||||
|
||||
generate.calls = calls # type: ignore[attr-defined]
|
||||
return generate
|
||||
|
||||
|
||||
# --- the loop earns its rounds ------------------------------------------------
|
||||
|
||||
|
||||
def test_a_named_gap_buys_exactly_one_more_round():
|
||||
"""Asked for adults, first round returned only paediatric text."""
|
||||
retriever = _Retriever((CHILD,), (ADULT, CHILD))
|
||||
metrics = InMemoryMetrics()
|
||||
|
||||
outcome = run_turn(
|
||||
_state(),
|
||||
_q(population="nguoi_lon"),
|
||||
retriever,
|
||||
_generator("Người lớn: 0,5 - 1 g/lần [1]"),
|
||||
metrics=metrics,
|
||||
)
|
||||
|
||||
assert outcome.retrieval_rounds_used == 2
|
||||
assert outcome.generated is True
|
||||
assert metrics.total(LOOP_REFINED, missing="population:nguoi_lon") == 1
|
||||
assert retriever.queries[1] != retriever.queries[0]
|
||||
|
||||
|
||||
def test_a_satisfied_question_spends_one_round_only():
|
||||
retriever = _Retriever((ADULT,))
|
||||
|
||||
outcome = run_turn(
|
||||
_state(), _q(population="nguoi_lon"), retriever, _generator("ok [1]")
|
||||
)
|
||||
|
||||
assert outcome.retrieval_rounds_used == 1
|
||||
assert outcome.stopped_because == "sufficient"
|
||||
|
||||
|
||||
def test_a_simple_question_does_not_loop():
|
||||
"""No population asked for means nothing to be missing."""
|
||||
retriever = _Retriever((ADULT, CHILD))
|
||||
|
||||
outcome = run_turn(_state(), _q(), retriever, _generator("ok [1]"))
|
||||
|
||||
assert outcome.retrieval_rounds_used == 1
|
||||
|
||||
|
||||
# --- the loop cannot run away -------------------------------------------------
|
||||
|
||||
|
||||
def test_retrieval_rounds_are_hard_capped():
|
||||
"""Evidence never satisfies the assessor; the loop must still stop."""
|
||||
retriever = _Retriever((CHILD,), (CHILD,), (CHILD,), (CHILD,), (CHILD,))
|
||||
|
||||
outcome = run_turn(
|
||||
_state(),
|
||||
_q(population="nguoi_lon"),
|
||||
retriever,
|
||||
_generator("ok [1]"),
|
||||
budget=TurnBudget(retrieval_rounds=2),
|
||||
)
|
||||
|
||||
assert outcome.retrieval_rounds_used == 2
|
||||
assert outcome.stopped_because == "retrieval_budget"
|
||||
assert len(retriever.queries) == 2
|
||||
|
||||
|
||||
def test_repairs_are_hard_capped_and_degrade_to_no_answer():
|
||||
"""`generate` returning None means verification refused it every time."""
|
||||
generate = _generator(None)
|
||||
|
||||
outcome = run_turn(
|
||||
_state(),
|
||||
_q(),
|
||||
_Retriever((ADULT,)),
|
||||
generate,
|
||||
budget=TurnBudget(repairs=1, llm_calls=4),
|
||||
)
|
||||
|
||||
assert outcome.answer is None
|
||||
assert outcome.repairs_used == 1
|
||||
assert generate.calls["n"] == 2 # first attempt + one repair
|
||||
assert outcome.stopped_because == "repair_budget"
|
||||
|
||||
|
||||
def test_llm_call_budget_stops_generation_entirely():
|
||||
generate = _generator(None)
|
||||
|
||||
outcome = run_turn(
|
||||
_state(), _q(), _Retriever((ADULT,)), generate, budget=TurnBudget(llm_calls=0)
|
||||
)
|
||||
|
||||
assert generate.calls["n"] == 0
|
||||
assert outcome.stopped_because == "llm_budget"
|
||||
|
||||
|
||||
def test_a_refinement_that_changes_nothing_stops_the_loop():
|
||||
"""Guards against a loop that keeps re-issuing the same query."""
|
||||
|
||||
class _SameQuery:
|
||||
def assess(self, resolved, evidence):
|
||||
return Sufficiency(False, missing="x", refined_query=resolved.text)
|
||||
|
||||
retriever = _Retriever((CHILD,), (CHILD,))
|
||||
|
||||
outcome = run_turn(
|
||||
_state(), _q(), retriever, _generator("ok [1]"), assessor=_SameQuery()
|
||||
)
|
||||
|
||||
assert outcome.stopped_because == "query_unchanged"
|
||||
assert len(retriever.queries) == 1
|
||||
|
||||
|
||||
def test_an_unnamed_gap_does_not_buy_a_round():
|
||||
""""Feels incomplete" is not a reason to spend the budget."""
|
||||
|
||||
class _Vague:
|
||||
def assess(self, resolved, evidence):
|
||||
return Sufficiency(False)
|
||||
|
||||
retriever = _Retriever((CHILD,), (CHILD,))
|
||||
|
||||
outcome = run_turn(_state(), _q(), retriever, _generator("ok [1]"), assessor=_Vague())
|
||||
|
||||
assert outcome.stopped_because == "no_actionable_gap"
|
||||
assert len(retriever.queries) == 1
|
||||
|
||||
|
||||
# --- clarify beats guessing ---------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"signal",
|
||||
[ClarifyReason.NO_ATTRIBUTE, ClarifyReason.AMBIGUOUS_DRUG, ClarifyReason.MULTI_ATTRIBUTE],
|
||||
)
|
||||
def test_a_clarify_signal_short_circuits_before_any_spend(signal):
|
||||
retriever = _Retriever((ADULT,))
|
||||
generate = _generator("ok [1]")
|
||||
metrics = InMemoryMetrics()
|
||||
budget = TurnBudget()
|
||||
|
||||
outcome = run_turn(
|
||||
_state(), _q(), retriever, generate, clarify_signals=(signal,), budget=budget, metrics=metrics
|
||||
)
|
||||
|
||||
assert outcome.clarification is not None
|
||||
assert outcome.clarification.reason == signal
|
||||
assert outcome.answer is None
|
||||
assert retriever.queries == []
|
||||
assert generate.calls["n"] == 0
|
||||
assert budget.llm_calls == 4 and budget.retrieval_rounds == 2
|
||||
assert metrics.total(CLARIFY_ASKED, reason=signal) == 1
|
||||
|
||||
|
||||
def test_no_evidence_at_all_asks_rather_than_abstaining_silently():
|
||||
outcome = run_turn(_state(), _q(), _Retriever(()), _generator("ok [1]"))
|
||||
|
||||
assert outcome.clarification is not None
|
||||
assert outcome.clarification.reason == ClarifyReason.STILL_INSUFFICIENT
|
||||
assert outcome.stopped_because == "no_evidence"
|
||||
|
||||
|
||||
# --- the deterministic assessor ----------------------------------------------
|
||||
|
||||
|
||||
def test_assessor_only_reports_gaps_it_can_demonstrate():
|
||||
assessor = DeterministicAssessor()
|
||||
|
||||
assert assessor.assess(_q(population="nguoi_lon"), (ADULT,)).sufficient is True
|
||||
assert assessor.assess(_q(population="nguoi_lon"), (CHILD,)).sufficient is False
|
||||
# No population asked for: nothing can be shown missing.
|
||||
assert assessor.assess(_q(), (CHILD,)).sufficient is True
|
||||
|
||||
|
||||
def test_trace_records_the_stages_walked():
|
||||
trace = LoopTrace()
|
||||
|
||||
run_turn(_state(), _q(), _Retriever((ADULT,)), _generator("ok [1]"), trace=trace)
|
||||
|
||||
assert trace.stages[0] == "understand"
|
||||
assert "retrieve" in trace.stages
|
||||
assert "assess" in trace.stages
|
||||
assert trace.stages[-1] == "generate"
|
||||
@@ -339,3 +339,78 @@ def test_recommendation_intent_is_refused_at_policy_boundary():
|
||||
)
|
||||
assert result.decision == EvidenceDecision.ABSTAIN
|
||||
assert result.reason == "recommendation_out_of_scope"
|
||||
|
||||
|
||||
class _IndicationRetriever:
|
||||
"""A fake exposing only `find_by_indication`/`search_indication` (the
|
||||
Qdrant adapter's shape for the reverse-lookup path), so
|
||||
`retrieve_by_indication`'s own orchestration — keyword first, dense
|
||||
fallback only when keyword finds nothing — is what's under test here,
|
||||
not the matching algorithm itself (that's `test_qdrant_adapter.py`'s job)."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
keyword_hits: list[SearchHit] | None = None,
|
||||
dense_hits: list[SearchHit] | None = None,
|
||||
) -> None:
|
||||
self._keyword_hits = keyword_hits or []
|
||||
self._dense_hits = dense_hits or []
|
||||
self.dense_called = False
|
||||
|
||||
def find_by_indication(self, indication_text, limit): # noqa: ARG002
|
||||
return self._keyword_hits
|
||||
|
||||
def search_indication(self, query, limit): # noqa: ARG002
|
||||
self.dense_called = True
|
||||
return self._dense_hits
|
||||
|
||||
|
||||
def _indication_hit(drug_id: str) -> SearchHit:
|
||||
return SearchHit(
|
||||
document=RetrievalDocument(
|
||||
doc_id=f"{drug_id}__chi_dinh__0", drug_id=drug_id, kind="prose",
|
||||
section_key="chi_dinh", text="Điều trị sốt.", source_refs=(SOURCE,),
|
||||
),
|
||||
score=1.0,
|
||||
)
|
||||
|
||||
|
||||
def test_retrieve_by_indication_uses_keyword_hits_without_trying_dense():
|
||||
retriever = _IndicationRetriever(keyword_hits=[_indication_hit("paracetamol_acetaminophen")])
|
||||
service = RetrievalService(retriever, InMemoryParentStore([]))
|
||||
|
||||
result = service.retrieve_by_indication("sốt")
|
||||
|
||||
assert result.decision == EvidenceDecision.ANSWERABLE
|
||||
assert len(result.evidence) == 1
|
||||
assert retriever.dense_called is False
|
||||
|
||||
|
||||
def test_retrieve_by_indication_falls_back_to_dense_only_when_keyword_is_empty():
|
||||
retriever = _IndicationRetriever(dense_hits=[_indication_hit("ibuprofen")])
|
||||
service = RetrievalService(retriever, InMemoryParentStore([]))
|
||||
|
||||
result = service.retrieve_by_indication("thân nhiệt tăng")
|
||||
|
||||
assert result.decision == EvidenceDecision.ANSWERABLE
|
||||
assert retriever.dense_called is True
|
||||
|
||||
|
||||
def test_retrieve_by_indication_with_no_match_anywhere_abstains():
|
||||
retriever = _IndicationRetriever()
|
||||
service = RetrievalService(retriever, InMemoryParentStore([]))
|
||||
|
||||
result = service.retrieve_by_indication("bệnh chưa từng ghi nhận")
|
||||
|
||||
assert result.decision == EvidenceDecision.ABSTAIN
|
||||
assert result.reason == "no_indication_match"
|
||||
|
||||
|
||||
def test_retrieve_by_indication_with_blank_text_abstains_without_calling_retrieval():
|
||||
retriever = _IndicationRetriever(keyword_hits=[_indication_hit("paracetamol_acetaminophen")])
|
||||
service = RetrievalService(retriever, InMemoryParentStore([]))
|
||||
|
||||
result = service.retrieve_by_indication(" ")
|
||||
|
||||
assert result.decision == EvidenceDecision.ABSTAIN
|
||||
assert result.reason == "missing_indication"
|
||||
|
||||
@@ -77,7 +77,8 @@ CONTRA = [
|
||||
]
|
||||
INDICATION = [_doc("i1", "chi_dinh", "Giảm đau, hạ sốt, chống viêm.")]
|
||||
PHARMACOLOGY = [_doc("p1", "duoc_ly_va_co_che_tac_dung", "Ức chế cyclooxygenase.")]
|
||||
ALL_DOCS = CONTRA + INDICATION + PHARMACOLOGY
|
||||
PRECAUTION = [_doc("t1", "than_trong", "Thận trọng với người suy thận.")]
|
||||
ALL_DOCS = CONTRA + INDICATION + PHARMACOLOGY + PRECAUTION
|
||||
|
||||
|
||||
def _service(retriever, resolver: SectionResolver | None) -> RetrievalService:
|
||||
@@ -207,6 +208,22 @@ class TestSectionRouting:
|
||||
assert retriever.section_calls == []
|
||||
assert retriever.search_calls
|
||||
|
||||
def test_than_trong_also_pools_chong_chi_dinh(self) -> None:
|
||||
"""A precaution some drug's own "thận trọng" text never mentions can
|
||||
still be filed under "chống chỉ định" (found live: Aspirin + loét dạ
|
||||
dày). Pooling both keeps that answerable instead of a false "not in
|
||||
this source" clarify/abstain."""
|
||||
retriever = SectionAwareRetriever(ALL_DOCS)
|
||||
result = _service(retriever, SectionResolver()).retrieve(
|
||||
"Thận trọng khi dùng aspirin là gì?", "aspirin"
|
||||
)
|
||||
assert retriever.section_calls == [
|
||||
("aspirin", "than_trong"), ("aspirin", "chong_chi_dinh"),
|
||||
]
|
||||
assert {item.evidence_id for item in result.evidence} == {
|
||||
"t1", "c1", "c2", "c3", "c4", "c5",
|
||||
}
|
||||
|
||||
def test_named_but_empty_section_falls_back(self) -> None:
|
||||
"""A drug with no such section must not abstain — similarity still tries."""
|
||||
retriever = SectionAwareRetriever(INDICATION + PHARMACOLOGY)
|
||||
|
||||
@@ -10,7 +10,13 @@ from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
from rag.understanding import SECTION_KEY_HINTS, SECTION_KEYS, LlmQueryUnderstander
|
||||
from rag.ports import AnswerGenerationUnavailable
|
||||
from rag.understanding import (
|
||||
SECTION_KEY_HINTS,
|
||||
SECTION_KEYS,
|
||||
LlmQueryUnderstander,
|
||||
QueryFrame,
|
||||
)
|
||||
|
||||
CATALOG = {
|
||||
"paracetamol_acetaminophen": "paracetamol acetaminophen, PARACETAMOL",
|
||||
@@ -23,6 +29,8 @@ class _FixedLlm:
|
||||
self._payload = payload
|
||||
|
||||
def generate(self, system: str, user: str, schema: dict) -> str:
|
||||
if isinstance(self._payload, BaseException):
|
||||
raise self._payload
|
||||
if isinstance(self._payload, str):
|
||||
return self._payload
|
||||
return json.dumps(self._payload, ensure_ascii=False)
|
||||
@@ -139,11 +147,108 @@ def test_fuzzy_suggestion_bounds_a_typo_into_the_candidate_set():
|
||||
assert frame.drugs == ("metformin",)
|
||||
|
||||
|
||||
# --- F-10: a small battery of invented near-alias shapes, beyond the single
|
||||
# "aspirinol" case above — each simulates a different way a name could be
|
||||
# crafted to *look* like it should fuzzy-match a real drug ---------------
|
||||
|
||||
|
||||
def test_a_real_drug_name_with_a_brand_like_suffix_is_not_substituted():
|
||||
resolver = _FakeResolver({}) # nothing in this turn resolves or suggests
|
||||
understander = LlmQueryUnderstander(_FixedLlm({
|
||||
"turn_type": "drug_attribute", "drugs": [],
|
||||
"unknown_drugs": ["metforminex"], "attribute": None, "population": None,
|
||||
"weight_kg": None, "age_text": None, "indication": None,
|
||||
"needs_clarify": False, "clarify_reason": None,
|
||||
}), CATALOG, resolver)
|
||||
frame = understander.understand("liều metforminex")
|
||||
assert frame.drugs == ()
|
||||
assert frame.unknown_drugs == ("metforminex",)
|
||||
|
||||
|
||||
def test_a_name_blending_two_real_drugs_is_not_substituted_for_either():
|
||||
resolver = _FakeResolver({})
|
||||
understander = LlmQueryUnderstander(_FixedLlm({
|
||||
"turn_type": "drug_attribute", "drugs": ["metformin", "paracetamol_acetaminophen"],
|
||||
"unknown_drugs": [], "attribute": None, "population": None,
|
||||
"weight_kg": None, "age_text": None, "indication": None,
|
||||
"needs_clarify": False, "clarify_reason": None,
|
||||
}), CATALOG, resolver)
|
||||
frame = understander.understand("liều metformacetamol")
|
||||
# Neither real id has deterministic candidate support for this turn —
|
||||
# F-04's bound must reject both, not accept the ones that happen to be
|
||||
# real catalog members.
|
||||
assert frame.drugs == ()
|
||||
assert "metformin" in frame.unknown_drugs
|
||||
assert "paracetamol_acetaminophen" in frame.unknown_drugs
|
||||
|
||||
|
||||
def test_malformed_json_fails_closed_to_a_clarify():
|
||||
understander = LlmQueryUnderstander(_FixedLlm("not json"), CATALOG, RESOLVER)
|
||||
frame = understander.understand("gì đó")
|
||||
assert frame.turn_type == "out_of_scope"
|
||||
assert frame.needs_clarify is True
|
||||
assert frame.quick_replies == ()
|
||||
|
||||
|
||||
def test_quick_replies_are_parsed_when_the_model_offers_them():
|
||||
understander = LlmQueryUnderstander(_FixedLlm({
|
||||
"turn_type": "dosing_calc", "drugs": ["paracetamol_acetaminophen"],
|
||||
"unknown_drugs": [], "attribute": None, "population": None,
|
||||
"weight_kg": None, "age_text": None, "indication": None,
|
||||
"needs_clarify": True, "clarify_reason": "Người lớn hay trẻ em?",
|
||||
"quick_replies": ["Người lớn", "Trẻ em"],
|
||||
}), CATALOG, RESOLVER)
|
||||
frame = understander.understand("liều paracetamol")
|
||||
assert frame.quick_replies == ("Người lớn", "Trẻ em")
|
||||
|
||||
|
||||
def test_missing_quick_replies_key_defaults_to_empty_not_a_crash():
|
||||
"""The model is asked for `quick_replies` but structured-output providers
|
||||
aren't guaranteed to include every optional key — a clarify without it
|
||||
must still parse, just with no chips."""
|
||||
understander = LlmQueryUnderstander(_FixedLlm({
|
||||
"turn_type": "dosing_calc", "drugs": [],
|
||||
"unknown_drugs": [], "attribute": None, "population": None,
|
||||
"weight_kg": None, "age_text": None, "indication": None,
|
||||
"needs_clarify": True, "clarify_reason": "Cân nặng bao nhiêu kg?",
|
||||
}), CATALOG, RESOLVER)
|
||||
frame = understander.understand("liều cho bé")
|
||||
assert frame.quick_replies == ()
|
||||
|
||||
|
||||
def test_route_is_parsed_when_the_model_resolves_it():
|
||||
"""The 2026-08-07 bug: a bare reply like 'Uống' answering the model's own
|
||||
prior route question had nowhere to be recorded (QueryFrame had no route
|
||||
field), so the model could only repeat its clarify_reason verbatim."""
|
||||
understander = LlmQueryUnderstander(_FixedLlm({
|
||||
"turn_type": "dosing_calc", "drugs": ["paracetamol_acetaminophen"],
|
||||
"unknown_drugs": [], "attribute": "lieu_luong_va_cach_dung",
|
||||
"population": "nguoi_lon", "weight_kg": None, "age_text": None,
|
||||
"indication": None, "route": "uong",
|
||||
"needs_clarify": False, "clarify_reason": None,
|
||||
}), CATALOG, RESOLVER)
|
||||
frame = understander.understand(
|
||||
"Uống",
|
||||
history=(
|
||||
"Người dùng: Liều paracetamol hạ sốt là bao nhiêu?",
|
||||
"Trợ lý: Người lớn hay trẻ em? Uống hay đặt trực tràng?",
|
||||
"Người dùng: Người lớn",
|
||||
"Trợ lý: Uống hay đặt trực tràng?",
|
||||
),
|
||||
)
|
||||
assert frame.route == "uong"
|
||||
assert frame.needs_clarify is False
|
||||
|
||||
|
||||
def test_missing_route_key_defaults_to_none_not_a_crash():
|
||||
understander = LlmQueryUnderstander(_FixedLlm({
|
||||
"turn_type": "dosing_calc", "drugs": [],
|
||||
"unknown_drugs": [], "attribute": None, "population": None,
|
||||
"weight_kg": None, "age_text": None, "indication": None,
|
||||
"needs_clarify": False, "clarify_reason": None,
|
||||
}), CATALOG, RESOLVER)
|
||||
frame = understander.understand("liều metformin")
|
||||
assert frame.route is None
|
||||
|
||||
|
||||
def test_unrecognised_turn_type_falls_back_based_on_whether_a_drug_resolved():
|
||||
@@ -200,3 +305,139 @@ def test_invalid_attribute_is_dropped_not_passed_through():
|
||||
}), CATALOG, RESOLVER)
|
||||
frame = understander.understand("metformin")
|
||||
assert frame.attribute is None
|
||||
|
||||
|
||||
# --- F-10: provider outage during the ONE call site that had no error
|
||||
# handling at all ------------------------------------------------------------
|
||||
|
||||
|
||||
# --- F-11: prior-frame merge — the code-level backstop for the model
|
||||
# dropping an already-established slot mid clarify-chain. Found live
|
||||
# 2026-08-07 (50-question hand-typed browser audit): reproduced 3 times
|
||||
# independently as either a non-terminating re-ask of the same clarify
|
||||
# question, or a stale drug bleeding into an unrelated new topic. --------
|
||||
|
||||
|
||||
def test_prior_frame_known_fields_survive_a_short_reply_the_model_drops():
|
||||
"""The Insulin/Azithromycin shape: the new call's own JSON comes back
|
||||
with the just-answered field null (a real, observed failure — the model
|
||||
is asked to restate it and sometimes doesn't), but since this turn named
|
||||
no drug of its own (a short reply like "20kg" never does), the previously
|
||||
established fields must survive via the merge, not be silently lost."""
|
||||
understander = LlmQueryUnderstander(_FixedLlm({
|
||||
"turn_type": "dosing_calc", "drugs": [],
|
||||
"unknown_drugs": [], "attribute": None, "population": None,
|
||||
"weight_kg": 20, "age_text": None, "indication": None,
|
||||
"needs_clarify": False, "clarify_reason": None,
|
||||
}), CATALOG, RESOLVER)
|
||||
prior = QueryFrame(
|
||||
turn_type="dosing_calc", drugs=("paracetamol_acetaminophen",),
|
||||
attribute="lieu_luong_va_cach_dung", needs_clarify=True,
|
||||
clarify_reason="Bé nặng bao nhiêu kg?",
|
||||
)
|
||||
frame = understander.understand("bé nặng 20 cân", prior_frame=prior)
|
||||
assert frame.drugs == ("paracetamol_acetaminophen",)
|
||||
assert frame.attribute == "lieu_luong_va_cach_dung"
|
||||
assert frame.weight_kg == 20
|
||||
|
||||
|
||||
def test_prior_frame_is_not_merged_when_the_turn_resolves_a_different_drug():
|
||||
"""The headache/OMEPRAZOL bleed this guards against: a turn that itself
|
||||
names a real, different drug is a genuine topic change and must not
|
||||
inherit the old drug's population/weight/route — merging here would
|
||||
reproduce the exact bug being fixed."""
|
||||
understander = LlmQueryUnderstander(_FixedLlm({
|
||||
"turn_type": "drug_attribute", "drugs": ["metformin"],
|
||||
"unknown_drugs": [], "attribute": "chi_dinh", "population": None,
|
||||
"weight_kg": None, "age_text": None, "indication": None,
|
||||
"needs_clarify": False, "clarify_reason": None,
|
||||
}), CATALOG, RESOLVER)
|
||||
prior = QueryFrame(
|
||||
turn_type="drug_attribute", drugs=("paracetamol_acetaminophen",),
|
||||
population="tre_em", weight_kg=20, needs_clarify=True,
|
||||
clarify_reason="Bé nặng bao nhiêu kg?",
|
||||
)
|
||||
frame = understander.understand("chỉ định của metformin là gì", prior_frame=prior)
|
||||
assert frame.drugs == ("metformin",)
|
||||
assert frame.population is None
|
||||
assert frame.weight_kg is None
|
||||
|
||||
|
||||
def test_prior_frame_that_was_already_resolved_is_not_merged():
|
||||
"""A prior turn that already answered (needs_clarify=False) has nothing
|
||||
to continue — merging it into a brand-new turn would leak stale state
|
||||
into an unrelated question that happens to follow it."""
|
||||
understander = LlmQueryUnderstander(_FixedLlm({
|
||||
"turn_type": "smalltalk", "drugs": [],
|
||||
"unknown_drugs": [], "attribute": None, "population": None,
|
||||
"weight_kg": None, "age_text": None, "indication": None,
|
||||
"needs_clarify": False, "clarify_reason": None,
|
||||
}), CATALOG, RESOLVER)
|
||||
prior = QueryFrame(
|
||||
turn_type="drug_attribute", drugs=("metformin",),
|
||||
population="nguoi_lon", needs_clarify=False,
|
||||
)
|
||||
frame = understander.understand("cảm ơn bạn", prior_frame=prior)
|
||||
assert frame.drugs == ()
|
||||
assert frame.population is None
|
||||
|
||||
|
||||
def test_known_facts_block_is_sent_to_the_model_on_a_clarify_continuation():
|
||||
captured = {}
|
||||
|
||||
class _CapturingLlm:
|
||||
def generate(self, system, user, schema):
|
||||
captured["user"] = user
|
||||
return json.dumps({
|
||||
"turn_type": "dosing_calc", "drugs": ["paracetamol_acetaminophen"],
|
||||
"unknown_drugs": [], "attribute": None, "population": None,
|
||||
"weight_kg": 20, "age_text": None, "indication": None,
|
||||
"needs_clarify": False, "clarify_reason": None,
|
||||
})
|
||||
|
||||
understander = LlmQueryUnderstander(_CapturingLlm(), CATALOG, RESOLVER)
|
||||
prior = QueryFrame(
|
||||
turn_type="dosing_calc", drugs=("paracetamol_acetaminophen",),
|
||||
population="tre_em", needs_clarify=True,
|
||||
clarify_reason="Bé nặng bao nhiêu kg?",
|
||||
)
|
||||
understander.understand("20 cân", prior_frame=prior)
|
||||
|
||||
assert "THÔNG TIN ĐÃ XÁC ĐỊNH" in captured["user"]
|
||||
assert "paracetamol_acetaminophen" in captured["user"]
|
||||
|
||||
|
||||
def test_no_known_facts_block_when_there_is_no_prior_clarify():
|
||||
captured = {}
|
||||
|
||||
class _CapturingLlm:
|
||||
def generate(self, system, user, schema):
|
||||
captured["user"] = user
|
||||
return json.dumps({
|
||||
"turn_type": "smalltalk", "drugs": [],
|
||||
"unknown_drugs": [], "attribute": None, "population": None,
|
||||
"weight_kg": None, "age_text": None, "indication": None,
|
||||
"needs_clarify": False, "clarify_reason": None,
|
||||
})
|
||||
|
||||
understander = LlmQueryUnderstander(_CapturingLlm(), CATALOG, RESOLVER)
|
||||
understander.understand("chào bạn")
|
||||
assert "THÔNG TIN ĐÃ XÁC ĐỊNH" not in captured["user"]
|
||||
|
||||
|
||||
def test_provider_outage_fails_closed_to_a_clarify_not_an_unhandled_crash():
|
||||
"""Found live 2026-08-07: unlike every other LLM call site in this
|
||||
product, `understand()` had no try/except around its call at all — a
|
||||
Bedrock outage here propagated straight through `RagAgent.handle()`
|
||||
into an unhandled 500 (`routers/rag.py` only wraps the trace-save call,
|
||||
not `agent.handle()`), instead of the graceful abstain every other
|
||||
failure mode already gets."""
|
||||
understander = LlmQueryUnderstander(
|
||||
_FixedLlm(AnswerGenerationUnavailable("Bedrock unreachable")),
|
||||
CATALOG, RESOLVER,
|
||||
)
|
||||
frame = understander.understand("liều paracetamol cho người lớn")
|
||||
assert frame.needs_clarify is True
|
||||
assert frame.clarify_reason is not None
|
||||
assert frame.drugs == ()
|
||||
assert frame.quick_replies == ()
|
||||
|
||||
Reference in New Issue
Block a user