898 lines
35 KiB
Python
898 lines
35 KiB
Python
"""`rag/agent.py` — the new orchestrator wired live in F-03.
|
|
|
|
Codex's 2026-08-06 review (F-03/F-10) flagged that this module had zero test
|
|
coverage despite being built to replace the live front end. This is the
|
|
first coverage: routing branches with fakes, not an exhaustive golden set
|
|
(that is F-10's job — a production-path regression suite against real
|
|
Qdrant/Bedrock fixtures).
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from rag.agent import MAX_CONSECUTIVE_CLARIFY, RagAgent
|
|
from rag.answer import GroundedAnswerService
|
|
from rag.metrics import GENERATION_REJECTED, InMemoryMetrics
|
|
from rag.models import Evidence, EvidenceDecision, RetrievalResult, SourceRef
|
|
from rag.understanding import QueryFrame
|
|
|
|
SOURCE = SourceRef(physical_page=100, precision="region", printed_page=100)
|
|
|
|
|
|
def _evidence(text: str, requires_visual_check: bool = False) -> Evidence:
|
|
return Evidence(
|
|
evidence_id="e0", matched_doc_id="e0", kind="prose", text=text, score=1.0,
|
|
source_refs=(SOURCE,), hydrated_from_parent=False,
|
|
requires_visual_check=requires_visual_check,
|
|
)
|
|
|
|
|
|
class _FixedUnderstander:
|
|
def __init__(self, frame: QueryFrame) -> None:
|
|
self._frame = frame
|
|
|
|
def understand(self, turn, history=(), budget=None, prior_frame=None):
|
|
return self._frame
|
|
|
|
|
|
class _FixedRetrieval:
|
|
"""Stands in for `RetrievalService.retrieve_framed` — a canned result per
|
|
drug_id regardless of section/query, so these tests assert routing, not
|
|
retrieval (that's `test_retrieval_service.py`'s job)."""
|
|
|
|
def __init__(
|
|
self,
|
|
results: dict[str, RetrievalResult],
|
|
indication_results: dict[str, RetrievalResult] | None = None,
|
|
) -> None:
|
|
self._results = results
|
|
self._indication_results = indication_results or {}
|
|
self.calls: list[tuple[str, str | None, str]] = []
|
|
|
|
def retrieve_framed(self, drug_id, section_key, query, is_overview=False):
|
|
self.calls.append((drug_id, section_key, query))
|
|
return self._results.get(
|
|
drug_id, RetrievalResult(EvidenceDecision.ABSTAIN, "not_configured")
|
|
)
|
|
|
|
def retrieve_by_indication(self, indication_text):
|
|
return self._indication_results.get(
|
|
indication_text, RetrievalResult(EvidenceDecision.ABSTAIN, "no_indication_match")
|
|
)
|
|
|
|
def decide(self, evidence):
|
|
# Mirrors `RetrievalService.decide`'s real policy (not a stub that
|
|
# always says ANSWERABLE) so `_interaction`'s use of it is actually
|
|
# under test, not just its own call site.
|
|
if not evidence:
|
|
return RetrievalResult(EvidenceDecision.ABSTAIN, "parent_hydration_failed")
|
|
if any(item.requires_visual_check for item in evidence):
|
|
return RetrievalResult(EvidenceDecision.VERIFY_PDF, "visual_verification_required", evidence)
|
|
return RetrievalResult(EvidenceDecision.ANSWERABLE, "grounded_evidence_available", evidence)
|
|
|
|
|
|
def _agent(
|
|
frame: QueryFrame,
|
|
results: dict[str, RetrievalResult] | None = None,
|
|
indication_results: dict[str, RetrievalResult] | None = None,
|
|
) -> RagAgent:
|
|
# `routing=None`: `answer_from_result` (the only method this path calls)
|
|
# never touches it — see `rag/answer.py`.
|
|
answers = GroundedAnswerService(routing=None)
|
|
return RagAgent(
|
|
_FixedUnderstander(frame),
|
|
_FixedRetrieval(results or {}, indication_results),
|
|
answers,
|
|
)
|
|
|
|
|
|
def test_smalltalk_does_not_touch_retrieval():
|
|
agent = _agent(QueryFrame(turn_type="smalltalk"))
|
|
reply = agent.handle("chào bạn")
|
|
assert reply.decision == "answerable"
|
|
assert reply.turn_type == "smalltalk"
|
|
assert "Dược thư" in reply.answer
|
|
|
|
|
|
def test_out_of_scope_turn_type_abstains():
|
|
agent = _agent(QueryFrame(turn_type="out_of_scope"))
|
|
reply = agent.handle("cách tiêm truyền tĩnh mạch")
|
|
assert reply.decision == "abstain"
|
|
assert reply.reason == "out_of_scope"
|
|
|
|
|
|
def test_veterinary_phrase_abstains_even_if_the_model_missed_it():
|
|
# The model returned an ordinary-looking frame; the keyword backstop
|
|
# (`rag.policy.looks_non_human`, the same one F-02 wired server-side)
|
|
# still catches it before any retrieval happens.
|
|
agent = _agent(QueryFrame(turn_type="drug_attribute", drugs=("metformin",)))
|
|
reply = agent.handle("liều metformin cho chó bao nhiêu")
|
|
assert reply.decision == "abstain"
|
|
assert reply.reason == "out_of_scope"
|
|
|
|
|
|
def test_unknown_drug_name_is_reported_not_substituted():
|
|
agent = _agent(QueryFrame(turn_type="drug_attribute", unknown_drugs=("aspirinol",)))
|
|
reply = agent.handle("liều aspirinol")
|
|
assert reply.decision == "abstain"
|
|
assert reply.reason == "drug_not_in_formulary"
|
|
assert "aspirinol" in reply.answer
|
|
|
|
|
|
def test_no_drug_named_asks_which_one():
|
|
agent = _agent(QueryFrame(turn_type="drug_attribute"))
|
|
reply = agent.handle("liều dùng bao nhiêu")
|
|
assert reply.decision == "clarify"
|
|
assert reply.reason == "no_drug"
|
|
|
|
|
|
def test_drug_attribute_without_an_attribute_does_not_fall_into_overview_retrieval():
|
|
retrieval = _FixedRetrieval({})
|
|
answers = GroundedAnswerService(routing=None)
|
|
agent = RagAgent(
|
|
_FixedUnderstander(QueryFrame(
|
|
turn_type="drug_attribute", drugs=("paracetamol_acetaminophen",)
|
|
)),
|
|
retrieval,
|
|
answers,
|
|
)
|
|
|
|
reply = agent.handle("paracetamol thì sao?")
|
|
|
|
assert reply.decision == "clarify"
|
|
assert reply.reason == "missing_attribute"
|
|
assert retrieval.calls == []
|
|
|
|
|
|
def test_needs_clarify_frame_is_surfaced_directly():
|
|
agent = _agent(QueryFrame(
|
|
turn_type="dosing_calc", drugs=("paracetamol",),
|
|
needs_clarify=True, clarify_reason="Bé mấy tuổi, cân nặng bao nhiêu kg?",
|
|
))
|
|
reply = agent.handle("liều paracetamol cho trẻ em")
|
|
assert reply.decision == "clarify"
|
|
assert reply.clarification == "Bé mấy tuổi, cân nặng bao nhiêu kg?"
|
|
# No quick_replies on the frame -> none surfaced (a specific weight has
|
|
# no clean short options; must not be fabricated downstream).
|
|
assert reply.quick_replies == ()
|
|
|
|
|
|
def test_understanding_provider_failure_is_an_abstain_not_a_fake_clarification():
|
|
agent = _agent(QueryFrame(
|
|
turn_type="out_of_scope",
|
|
needs_clarify=True,
|
|
clarify_reason="Dịch vụ đang gặp sự cố tạm thời.",
|
|
system_error="understanding_provider_unavailable",
|
|
))
|
|
|
|
reply = agent.handle("liều paracetamol")
|
|
|
|
assert reply.decision == "abstain"
|
|
assert reply.reason == "understanding_provider_unavailable"
|
|
assert reply.answer == "Dịch vụ đang gặp sự cố tạm thời."
|
|
assert reply.clarification is None
|
|
|
|
|
|
def test_needs_clarify_frame_carries_quick_replies_through():
|
|
"""This is the path real traffic actually hits (checked live): the
|
|
understanding LLM call itself sets needs_clarify/clarify_reason before
|
|
retrieval ever runs, short-circuiting `_route` — quick_replies must
|
|
survive that same short-circuit, not just the sufficiency-check path
|
|
inside `GroundedAnswerService`."""
|
|
agent = _agent(QueryFrame(
|
|
turn_type="drug_attribute", drugs=("paracetamol",),
|
|
needs_clarify=True, clarify_reason="Người lớn hay trẻ em?",
|
|
quick_replies=("Người lớn", "Trẻ em"),
|
|
))
|
|
reply = agent.handle("liều paracetamol")
|
|
assert reply.decision == "clarify"
|
|
assert reply.quick_replies == ("Người lớn", "Trẻ em")
|
|
|
|
|
|
def test_dosing_without_route_asks_only_when_evidence_is_ambiguous():
|
|
result = RetrievalResult(
|
|
EvidenceDecision.ANSWERABLE,
|
|
"grounded_evidence_available",
|
|
(
|
|
_evidence("Đường uống, người lớn: 500 mg mỗi lần."),
|
|
_evidence("Đặt trực tràng, người lớn: 500 mg mỗi lần."),
|
|
),
|
|
resolved_drug_id="paracetamol_acetaminophen",
|
|
)
|
|
|
|
class _Generator:
|
|
def generate(self, system, user, schema):
|
|
return (
|
|
'{"claims": [], "evidence_sufficient": false, '
|
|
'"clarifying_question": "Anh/chị muốn dùng đường nào?", '
|
|
'"quick_replies": ["Uống", "Đặt trực tràng"]}'
|
|
)
|
|
|
|
retrieval = _FixedRetrieval({"paracetamol_acetaminophen": result})
|
|
answers = GroundedAnswerService(routing=None, generator=_Generator())
|
|
agent = RagAgent(
|
|
_FixedUnderstander(QueryFrame(
|
|
turn_type="dosing_calc",
|
|
drugs=("paracetamol_acetaminophen",),
|
|
population="nguoi_lon",
|
|
)),
|
|
retrieval,
|
|
answers,
|
|
)
|
|
|
|
reply = agent.handle("Người lớn", conversation_id="dose-route")
|
|
|
|
assert reply.decision == "clarify"
|
|
assert reply.reason == "needs_more_info"
|
|
assert reply.quick_replies == ("Uống", "Đặt trực tràng")
|
|
assert len(retrieval.calls) == 1
|
|
remembered = agent._last_frame["dose-route"]
|
|
assert remembered.needs_clarify is True
|
|
assert remembered.population == "nguoi_lon"
|
|
assert remembered.clarify_reason == reply.clarification
|
|
|
|
|
|
def test_dosing_without_route_answers_directly_when_evidence_has_one_route():
|
|
result = RetrievalResult(
|
|
EvidenceDecision.ANSWERABLE,
|
|
"grounded_evidence_available",
|
|
(_evidence("Đường uống, người lớn: 500 mg mỗi lần."),),
|
|
resolved_drug_id="paracetamol_acetaminophen",
|
|
)
|
|
|
|
class _Generator:
|
|
def generate(self, system, user, schema):
|
|
if "entailed" in schema.get("properties", {}):
|
|
return '{"entailed": true, "unsupported": []}'
|
|
return (
|
|
'{"claims": [{"text": "Đường uống, người lớn: 500 mg mỗi lần.", '
|
|
'"citations": [1]}], "evidence_sufficient": true, '
|
|
'"clarifying_question": null, "quick_replies": []}'
|
|
)
|
|
|
|
retrieval = _FixedRetrieval({"paracetamol_acetaminophen": result})
|
|
answers = GroundedAnswerService(routing=None, generator=_Generator())
|
|
agent = RagAgent(
|
|
_FixedUnderstander(QueryFrame(
|
|
turn_type="dosing_calc",
|
|
drugs=("paracetamol_acetaminophen",),
|
|
population="nguoi_lon",
|
|
)),
|
|
retrieval,
|
|
answers,
|
|
)
|
|
|
|
reply = agent.handle("Liều Paracetamol cho người lớn")
|
|
|
|
assert reply.decision == "answerable"
|
|
assert reply.quick_replies == ()
|
|
assert "500 mg" in reply.answer
|
|
assert len(retrieval.calls) == 1
|
|
|
|
|
|
def test_general_dosage_section_survey_does_not_force_population_chip():
|
|
result = RetrievalResult(
|
|
EvidenceDecision.ANSWERABLE,
|
|
"grounded_evidence_available",
|
|
(_evidence("Người lớn: uống 10 mg. Trẻ em: liều theo cân nặng."),),
|
|
resolved_drug_id="example",
|
|
)
|
|
retrieval = _FixedRetrieval({"example": result})
|
|
agent = RagAgent(
|
|
_FixedUnderstander(QueryFrame(
|
|
turn_type="dosing_calc",
|
|
drugs=("example",),
|
|
needs_clarify=True,
|
|
clarify_reason="Người lớn hay trẻ em?",
|
|
)),
|
|
retrieval,
|
|
GroundedAnswerService(routing=None),
|
|
)
|
|
|
|
reply = agent.handle(
|
|
"Dược thư hướng dẫn dùng Example thế nào: đường dùng và các liều nếu có?"
|
|
)
|
|
|
|
assert reply.decision == "answerable"
|
|
assert reply.quick_replies == ()
|
|
assert "tra cứu tổng quan toàn mục" in retrieval.calls[0][2]
|
|
|
|
|
|
def test_clear_precaution_section_survey_overrides_model_overclarification():
|
|
result = RetrievalResult(
|
|
EvidenceDecision.ANSWERABLE,
|
|
"grounded_evidence_available",
|
|
(_evidence("Theo dõi chức năng thận và điện giải."),),
|
|
resolved_drug_id="example",
|
|
)
|
|
agent = _agent(
|
|
QueryFrame(
|
|
turn_type="drug_attribute",
|
|
drugs=("example",),
|
|
attribute="than_trong",
|
|
needs_clarify=True,
|
|
clarify_reason="Muốn hỏi thận trọng hay chống chỉ định?",
|
|
),
|
|
{"example": result},
|
|
)
|
|
|
|
reply = agent.handle("Những tình huống nào cần thận trọng khi dùng Example?")
|
|
|
|
assert reply.decision == "answerable"
|
|
assert reply.quick_replies == ()
|
|
|
|
|
|
def test_complete_adult_dosing_core_ignores_an_irrelevant_weight_reask():
|
|
result = RetrievalResult(
|
|
EvidenceDecision.ANSWERABLE, "grounded_evidence_available",
|
|
(_evidence("Đường uống, người lớn: 500 mg mỗi lần."),),
|
|
resolved_drug_id="paracetamol_acetaminophen",
|
|
)
|
|
retrieval = _FixedRetrieval({"paracetamol_acetaminophen": result})
|
|
answers = GroundedAnswerService(routing=None)
|
|
agent = RagAgent(
|
|
_FixedUnderstander(QueryFrame(
|
|
turn_type="dosing_calc",
|
|
drugs=("paracetamol_acetaminophen",),
|
|
population="nguoi_lon",
|
|
route="uong",
|
|
needs_clarify=True,
|
|
clarify_reason="Cân nặng của người lớn là bao nhiêu kg?",
|
|
)),
|
|
retrieval,
|
|
answers,
|
|
)
|
|
|
|
reply = agent.handle("Uống")
|
|
|
|
assert reply.decision == "answerable"
|
|
assert len(retrieval.calls) == 1
|
|
assert retrieval.calls[0][1] == "lieu_luong_va_cach_dung"
|
|
|
|
|
|
def test_single_drug_attribute_retrieves_and_answers():
|
|
result = RetrievalResult(
|
|
EvidenceDecision.ANSWERABLE, "grounded_evidence_available",
|
|
(_evidence("Liều 500 mg mỗi ngày."),), resolved_drug_id="metformin",
|
|
)
|
|
agent = _agent(
|
|
QueryFrame(turn_type="drug_attribute", drugs=("metformin",),
|
|
attribute="lieu_luong_va_cach_dung"),
|
|
{"metformin": result},
|
|
)
|
|
reply = agent.handle("liều metformin")
|
|
assert reply.decision == "answerable"
|
|
assert reply.drugs == ("metformin",)
|
|
assert "500 mg" in reply.answer
|
|
|
|
|
|
def test_context_resolved_across_turns_is_folded_into_the_query():
|
|
"""The 2026-08-07 P0 (named in the 2026-08-06 audit): population/weight/
|
|
age/route are extracted by understanding.py but were never passed into
|
|
retrieval or generation — so a reply like "Uống" three turns into a dose
|
|
conversation reached `GroundedAnswerService` as literally just "Uống",
|
|
with no notion that population=adult was already established. Fixed via
|
|
`_synthesize_query`; this asserts the synthesized text — not the bare
|
|
turn — is what retrieval and generation actually see."""
|
|
result = RetrievalResult(
|
|
EvidenceDecision.ANSWERABLE, "grounded_evidence_available",
|
|
(_evidence("Liều uống người lớn: 500 mg."),
|
|
_evidence("Liều tiêm người lớn: 1 g.")),
|
|
resolved_drug_id="paracetamol_acetaminophen",
|
|
)
|
|
retrieval = _FixedRetrieval({"paracetamol_acetaminophen": result})
|
|
agent = RagAgent(
|
|
_FixedUnderstander(QueryFrame(
|
|
turn_type="dosing_calc", drugs=("paracetamol_acetaminophen",),
|
|
attribute="lieu_luong_va_cach_dung", population="nguoi_lon",
|
|
route="uong", needs_clarify=False,
|
|
)),
|
|
retrieval,
|
|
GroundedAnswerService(routing=None),
|
|
)
|
|
agent.handle("Uống")
|
|
assert len(retrieval.calls) == 1
|
|
_, _, query = retrieval.calls[0]
|
|
assert query.startswith("Uống")
|
|
assert "người lớn" in query
|
|
assert "uống" in query.lower()
|
|
|
|
|
|
def test_context_synthesis_is_a_no_op_when_the_frame_has_no_resolved_fields():
|
|
"""A fresh, fully-specified single-shot question already states its own
|
|
context — synthesis must not alter it or introduce redundant noise."""
|
|
result = RetrievalResult(
|
|
EvidenceDecision.ANSWERABLE, "grounded_evidence_available",
|
|
(_evidence("x"), _evidence("y")), resolved_drug_id="metformin",
|
|
)
|
|
retrieval = _FixedRetrieval({"metformin": result})
|
|
agent = RagAgent(
|
|
_FixedUnderstander(QueryFrame(
|
|
turn_type="drug_attribute", drugs=("metformin",),
|
|
attribute="chong_chi_dinh",
|
|
)),
|
|
retrieval,
|
|
GroundedAnswerService(routing=None),
|
|
)
|
|
agent.handle("Chống chỉ định của metformin là gì?")
|
|
assert retrieval.calls[0][2] == "Chống chỉ định của metformin là gì?"
|
|
|
|
|
|
def test_interaction_combines_both_drugs_evidence():
|
|
warfarin = RetrievalResult(
|
|
EvidenceDecision.ANSWERABLE, "grounded_evidence_available",
|
|
(_evidence("Tương tác với aspirin làm tăng nguy cơ chảy máu."),),
|
|
)
|
|
aspirin = RetrievalResult(
|
|
EvidenceDecision.ANSWERABLE, "grounded_evidence_available",
|
|
(_evidence("Tương tác với warfarin làm tăng nguy cơ chảy máu."),),
|
|
)
|
|
agent = _agent(
|
|
QueryFrame(turn_type="interaction", drugs=("warfarin", "aspirin")),
|
|
{"warfarin": warfarin, "aspirin": aspirin},
|
|
)
|
|
reply = agent.handle("warfarin với aspirin có dùng chung được không")
|
|
assert reply.decision == "answerable"
|
|
assert reply.turn_type == "interaction"
|
|
assert "chảy máu" in reply.answer
|
|
|
|
|
|
def test_interaction_with_one_drug_quarantined_never_answers_confidently():
|
|
"""The P0 the 2026-08-06 audit found: `_interaction` used to keep only
|
|
`ANSWERABLE` parts, so a quarantined drug's evidence (and the "table
|
|
exists, verify PDF" notice the quarantine contract requires) was
|
|
silently dropped — a confident interaction answer could omit a real
|
|
unverified contraindication table for one of the two drugs. Fixed via
|
|
`RetrievalService.decide` applied to the combined pool, the same policy
|
|
the single-drug path already uses."""
|
|
warfarin = RetrievalResult(
|
|
EvidenceDecision.ANSWERABLE, "grounded_evidence_available",
|
|
(_evidence("Tương tác với aspirin làm tăng nguy cơ chảy máu."),),
|
|
)
|
|
aspirin = RetrievalResult(
|
|
EvidenceDecision.VERIFY_PDF, "visual_verification_required",
|
|
(_evidence("Bảng tương tác cần đối chiếu PDF.", requires_visual_check=True),),
|
|
)
|
|
agent = _agent(
|
|
QueryFrame(turn_type="interaction", drugs=("warfarin", "aspirin")),
|
|
{"warfarin": warfarin, "aspirin": aspirin},
|
|
)
|
|
reply = agent.handle("warfarin với aspirin có dùng chung được không")
|
|
# Must NOT be a confident "answerable" that silently omits aspirin's
|
|
# quarantined table — must ask for PDF verification instead.
|
|
assert reply.decision == "verify_pdf"
|
|
# Both drugs' evidence must still be present (as citations), not dropped.
|
|
assert len(reply.citations) == 2
|
|
|
|
|
|
def test_interaction_with_no_evidence_abstains_and_does_not_imply_safety():
|
|
agent = _agent(
|
|
QueryFrame(turn_type="interaction", drugs=("drug_a", "drug_b")), {},
|
|
)
|
|
reply = agent.handle("drug_a và drug_b dùng chung được không")
|
|
assert reply.decision == "abstain"
|
|
assert reply.reason == "no_interaction_evidence"
|
|
assert "KHÔNG có nghĩa là an toàn" in reply.answer
|
|
|
|
|
|
def test_symptom_to_drug_with_no_indication_extracted_asks_for_one():
|
|
agent = _agent(QueryFrame(turn_type="symptom_to_drug"))
|
|
reply = agent.handle("có thuốc gì không")
|
|
assert reply.decision == "clarify"
|
|
assert reply.reason == "no_indication"
|
|
|
|
|
|
def test_symptom_to_drug_with_no_match_abstains_not_silently_safe():
|
|
agent = _agent(
|
|
QueryFrame(turn_type="symptom_to_drug", indication="bệnh hiếm gặp x"),
|
|
indication_results={
|
|
"bệnh hiếm gặp x": RetrievalResult(EvidenceDecision.ABSTAIN, "no_indication_match"),
|
|
},
|
|
)
|
|
reply = agent.handle("thuốc gì trị bệnh hiếm gặp x")
|
|
assert reply.decision == "abstain"
|
|
assert "bệnh hiếm gặp x" in reply.answer
|
|
assert "KHÔNG" in reply.answer
|
|
|
|
|
|
def test_symptom_to_drug_returns_the_matched_drugs_not_frame_drugs():
|
|
"""`frame.drugs` is empty by construction for this turn_type (the router
|
|
only reaches `_symptom_to_drug` with no named drug) — the reply's
|
|
`drugs` must come from what retrieval actually found."""
|
|
result = RetrievalResult(
|
|
EvidenceDecision.ANSWERABLE, "grounded_evidence_available",
|
|
(_evidence("Paracetamol chỉ định hạ sốt."),
|
|
_evidence("Ibuprofen chỉ định hạ sốt, giảm đau.")),
|
|
)
|
|
# Overwrite matched_doc_id per evidence to simulate two different drugs
|
|
# (the shared `_evidence` helper always uses "e0" — construct directly).
|
|
ev_a = Evidence(
|
|
evidence_id="paracetamol_acetaminophen__chi_dinh__0",
|
|
matched_doc_id="paracetamol_acetaminophen__chi_dinh__0",
|
|
kind="prose", text="Paracetamol chỉ định hạ sốt.", score=1.0,
|
|
source_refs=(SOURCE,), hydrated_from_parent=False, requires_visual_check=False,
|
|
)
|
|
ev_b = Evidence(
|
|
evidence_id="ibuprofen__chi_dinh__0", matched_doc_id="ibuprofen__chi_dinh__0",
|
|
kind="prose", text="Ibuprofen chỉ định hạ sốt, giảm đau.", score=1.0,
|
|
source_refs=(SOURCE,), hydrated_from_parent=False, requires_visual_check=False,
|
|
)
|
|
result = RetrievalResult(
|
|
EvidenceDecision.ANSWERABLE, "grounded_evidence_available", (ev_a, ev_b),
|
|
)
|
|
agent = _agent(
|
|
QueryFrame(turn_type="symptom_to_drug", indication="sốt"),
|
|
indication_results={"sốt": result},
|
|
)
|
|
reply = agent.handle("sốt thì uống thuốc gì")
|
|
assert reply.decision == "answerable"
|
|
assert reply.drugs == ("paracetamol_acetaminophen", "ibuprofen")
|
|
|
|
|
|
def test_history_is_passed_to_the_understander_on_the_next_turn():
|
|
received_history: list[tuple[str, ...]] = []
|
|
|
|
class _RecordingUnderstander:
|
|
def understand(self, turn, history=(), budget=None, prior_frame=None):
|
|
received_history.append(tuple(history))
|
|
return QueryFrame(turn_type="smalltalk")
|
|
|
|
answers = GroundedAnswerService(routing=None)
|
|
agent = RagAgent(_RecordingUnderstander(), _FixedRetrieval({}), answers)
|
|
|
|
agent.handle("chào bạn", conversation_id="c1")
|
|
agent.handle("còn liều thì sao?", conversation_id="c1")
|
|
|
|
assert received_history[0] == ()
|
|
assert any("chào bạn" in line for line in received_history[1])
|
|
|
|
|
|
def test_history_is_isolated_per_conversation_id():
|
|
received_history: list[tuple[str, ...]] = []
|
|
|
|
class _RecordingUnderstander:
|
|
def understand(self, turn, history=(), budget=None, prior_frame=None):
|
|
received_history.append(tuple(history))
|
|
return QueryFrame(turn_type="smalltalk")
|
|
|
|
answers = GroundedAnswerService(routing=None)
|
|
agent = RagAgent(_RecordingUnderstander(), _FixedRetrieval({}), answers)
|
|
|
|
agent.handle("xin chào", conversation_id="a")
|
|
agent.handle("liều dùng bao nhiêu", conversation_id="b")
|
|
|
|
# The second call, on a different conversation id, must not see "a"'s turn.
|
|
assert received_history[1] == ()
|
|
|
|
|
|
# --- F-10: `conversation_id` presence/absence must not change the safety
|
|
# decision on a fresh (first) turn — only whether the turn is remembered
|
|
# afterward -------------------------------------------------------------
|
|
|
|
|
|
def test_conversation_id_presence_or_absence_reaches_the_same_decision():
|
|
"""A single-turn call (no `conversation_id`) and the first turn of a
|
|
fresh multi-turn conversation must resolve identically — both see empty
|
|
history, so nothing about `conversation_id` itself may become a second,
|
|
undocumented safety signal."""
|
|
agent = _agent(QueryFrame(
|
|
turn_type="drug_attribute", unknown_drugs=("aspirinol",),
|
|
))
|
|
without_id = agent.handle("liều aspirinol")
|
|
with_id = agent.handle("liều aspirinol", conversation_id="fresh-conv")
|
|
|
|
assert without_id.decision == with_id.decision == "abstain"
|
|
assert without_id.reason == with_id.reason == "drug_not_in_formulary"
|
|
|
|
|
|
def test_autocomplete_delegates_to_the_configured_source():
|
|
class _Source:
|
|
def complete(self, prefix, k):
|
|
return ["metformin_id", "metoprolol_id"]
|
|
|
|
answers = GroundedAnswerService(routing=None)
|
|
agent = RagAgent(
|
|
_FixedUnderstander(QueryFrame(turn_type="smalltalk")),
|
|
_FixedRetrieval({}), answers, autocomplete=_Source(),
|
|
)
|
|
assert agent.complete("met") == ["Metformin Id", "Metoprolol Id"]
|
|
|
|
|
|
def test_autocomplete_with_no_source_configured_returns_empty():
|
|
answers = GroundedAnswerService(routing=None)
|
|
agent = RagAgent(
|
|
_FixedUnderstander(QueryFrame(turn_type="smalltalk")),
|
|
_FixedRetrieval({}), answers,
|
|
)
|
|
assert agent.complete("met") == []
|
|
|
|
|
|
# --- F-08: a per-turn budget actually stops real provider calls once spent,
|
|
# it isn't just bookkeeping ---------------------------------------------
|
|
|
|
|
|
def test_a_budget_exhausted_during_understand_blocks_every_later_call():
|
|
"""`max_llm_calls_per_turn=1` means the (simulated) `understand()` call
|
|
spends the entire turn's budget — sufficiency and generate must never
|
|
reach the generator at all, not just receive an error from it. Proves
|
|
the budget is threaded end to end through `RagAgent`, not only present
|
|
at the one call site each unit test exercises in isolation."""
|
|
class _BudgetSpendingUnderstander:
|
|
"""Stands in for the real `LlmQueryUnderstander`, which calls
|
|
`budget.require()` once before its own LLM call — simulated here so
|
|
this test doesn't need a live-shaped LLM fake for the understand
|
|
step, only for the reply's calls to fail budget."""
|
|
|
|
def understand(self, turn, history=(), budget=None, prior_frame=None):
|
|
if budget is not None:
|
|
budget.require()
|
|
return QueryFrame(
|
|
turn_type="drug_attribute", drugs=("metformin",),
|
|
attribute="chi_dinh",
|
|
)
|
|
|
|
result = RetrievalResult(
|
|
EvidenceDecision.ANSWERABLE, "grounded_evidence_available",
|
|
(_evidence("đoạn 0"), _evidence("đoạn 1")),
|
|
)
|
|
|
|
generator_calls: list[dict] = []
|
|
|
|
class _CountingGenerator:
|
|
def generate(self, system, user, schema):
|
|
generator_calls.append(schema)
|
|
return '{"answer": "unused", "evidence_sufficient": true, "clarifying_question": null}'
|
|
|
|
metrics = InMemoryMetrics()
|
|
answers = GroundedAnswerService(
|
|
routing=None, generator=_CountingGenerator(), metrics=metrics
|
|
)
|
|
agent = RagAgent(
|
|
_BudgetSpendingUnderstander(), _FixedRetrieval({"metformin": result}), answers,
|
|
max_llm_calls_per_turn=1,
|
|
)
|
|
|
|
reply = agent.handle("liều metformin")
|
|
|
|
# The real point of F-08: no more actual provider calls happen once the
|
|
# budget is spent — not "the generator returned an error", the generator
|
|
# is never invoked at all.
|
|
assert generator_calls == []
|
|
assert reply.decision == "abstain"
|
|
assert reply.answer is None
|
|
assert metrics.total(GENERATION_REJECTED, reason="request_budget_exhausted") >= 1
|
|
|
|
|
|
def test_a_generous_budget_does_not_change_normal_behaviour():
|
|
"""Control for the test above: with the default (generous) budget, the
|
|
same setup answers normally — proves the previous test's tiny budget is
|
|
what caused the block, not some other change to the fixtures."""
|
|
class _BudgetSpendingUnderstander:
|
|
def understand(self, turn, history=(), budget=None, prior_frame=None):
|
|
if budget is not None:
|
|
budget.require()
|
|
return QueryFrame(
|
|
turn_type="drug_attribute", drugs=("metformin",),
|
|
attribute="chi_dinh",
|
|
)
|
|
|
|
result = RetrievalResult(
|
|
EvidenceDecision.ANSWERABLE, "grounded_evidence_available",
|
|
(_evidence("Liều 500 mg mỗi ngày."), _evidence("đoạn 1")),
|
|
)
|
|
|
|
class _Generator:
|
|
def generate(self, system, user, schema):
|
|
if "sufficient" in schema.get("properties", {}):
|
|
return '{"sufficient": true, "clarifying_question": null, "quick_replies": []}'
|
|
if "entailed" in schema.get("properties", {}):
|
|
return '{"entailed": true, "unsupported": []}'
|
|
return (
|
|
'{"claims": [{"text": "Liều 500 mg", "citations": [1]}], '
|
|
'"evidence_sufficient": true, "clarifying_question": null}'
|
|
)
|
|
|
|
answers = GroundedAnswerService(routing=None, generator=_Generator())
|
|
agent = RagAgent(
|
|
_BudgetSpendingUnderstander(), _FixedRetrieval({"metformin": result}), answers,
|
|
)
|
|
|
|
reply = agent.handle("liều metformin")
|
|
|
|
assert reply.decision == "answerable"
|
|
assert reply.answer == "Liều 500 mg"
|
|
assert reply.blocks[0].claims[0].source_ids
|
|
|
|
|
|
# --- durable conversation history (ADR 0008's named gap): an optional
|
|
# `ConversationStore` replaces the in-process dict when configured --------
|
|
|
|
|
|
class _FakeStore:
|
|
def __init__(self) -> None:
|
|
self.lines: dict[str, list[str]] = {}
|
|
self.fail_reads = False
|
|
self.fail_writes = False
|
|
|
|
def recent(self, conversation_id, limit):
|
|
if self.fail_reads:
|
|
raise ConnectionError("store unreachable")
|
|
return self.lines.get(conversation_id, [])[-limit:]
|
|
|
|
def append(self, conversation_id, line):
|
|
if self.fail_writes:
|
|
raise ConnectionError("store unreachable")
|
|
self.lines.setdefault(conversation_id, []).append(line)
|
|
|
|
|
|
def test_history_round_trips_through_a_configured_store():
|
|
received_history: list[tuple[str, ...]] = []
|
|
|
|
class _RecordingUnderstander:
|
|
def understand(self, turn, history=(), budget=None, prior_frame=None):
|
|
received_history.append(tuple(history))
|
|
return QueryFrame(turn_type="smalltalk")
|
|
|
|
store = _FakeStore()
|
|
answers = GroundedAnswerService(routing=None)
|
|
agent = RagAgent(_RecordingUnderstander(), _FixedRetrieval({}), answers, store=store)
|
|
|
|
agent.handle("chào bạn", conversation_id="c1")
|
|
agent.handle("còn liều thì sao?", conversation_id="c1")
|
|
|
|
assert received_history[0] == ()
|
|
assert any("chào bạn" in line for line in received_history[1])
|
|
assert store.lines["c1"] # actually persisted, not just read back in-process
|
|
|
|
|
|
def test_store_read_failure_fails_open_to_fresh_history_not_a_crash():
|
|
store = _FakeStore()
|
|
store.lines["c1"] = ["Người dùng: câu cũ", "Trợ lý: trả lời cũ"]
|
|
store.fail_reads = True
|
|
|
|
answers = GroundedAnswerService(routing=None)
|
|
agent = RagAgent(
|
|
_FixedUnderstander(QueryFrame(turn_type="smalltalk")),
|
|
_FixedRetrieval({}), answers, store=store,
|
|
)
|
|
|
|
# Must not raise -- degrades to no history for this turn.
|
|
reply = agent.handle("chào bạn", conversation_id="c1")
|
|
assert reply.decision == "answerable"
|
|
|
|
|
|
def test_store_write_failure_fails_open_the_response_still_returns():
|
|
store = _FakeStore()
|
|
store.fail_writes = True
|
|
|
|
answers = GroundedAnswerService(routing=None)
|
|
agent = RagAgent(
|
|
_FixedUnderstander(QueryFrame(turn_type="smalltalk")),
|
|
_FixedRetrieval({}), answers, store=store,
|
|
)
|
|
|
|
# Must not raise, even though persisting this turn silently fails.
|
|
reply = agent.handle("chào bạn", conversation_id="c1")
|
|
assert reply.decision == "answerable"
|
|
assert reply.answer is not None
|
|
|
|
|
|
# --- F-11: the clarify-loop circuit breaker. Found live 2026-08-07
|
|
# (50-question hand-typed browser audit): the understanding LLM can keep
|
|
# deciding needs_clarify=true forever with no natural exit — reproduced 3
|
|
# times independently, one case never converged after 5 real answered turns.
|
|
# `understanding.py`'s prior-frame merge fixes most of the underlying cause,
|
|
# but this is the code-level bound that guarantees a user is never stuck. --
|
|
|
|
|
|
class _AlwaysClarifyUnderstander:
|
|
"""Simulates a model stuck re-asking regardless of what the user
|
|
answers — exactly the observed live failure, isolated from any real
|
|
LLM's actual (variable) behaviour so this test is deterministic."""
|
|
|
|
def understand(self, turn, history=(), budget=None, prior_frame=None):
|
|
return QueryFrame(
|
|
turn_type="dosing_calc", drugs=("paracetamol_acetaminophen",),
|
|
needs_clarify=True, clarify_reason="Bé nặng bao nhiêu kg?",
|
|
)
|
|
|
|
|
|
def test_clarify_loop_is_hard_stopped_after_max_consecutive_turns():
|
|
answers = GroundedAnswerService(routing=None)
|
|
agent = RagAgent(_AlwaysClarifyUnderstander(), _FixedRetrieval({}), answers)
|
|
|
|
replies = [
|
|
agent.handle(f"turn {i}", conversation_id="stuck")
|
|
for i in range(MAX_CONSECUTIVE_CLARIFY)
|
|
]
|
|
|
|
# Every turn up to the last stays a genuine clarify...
|
|
for reply in replies[:-1]:
|
|
assert reply.decision == "clarify"
|
|
# ...the Nth forces a hard stop instead of asking again.
|
|
assert replies[-1].decision == "abstain"
|
|
assert replies[-1].reason == "clarify_loop_exhausted"
|
|
assert replies[-1].answer is not None
|
|
|
|
|
|
def test_clarify_streak_resets_after_the_hard_stop_so_a_new_attempt_can_proceed():
|
|
"""Confirms the breaker is a bounded pause, not a permanent lockout of
|
|
the conversation id — the very next turn gets a fresh streak."""
|
|
answers = GroundedAnswerService(routing=None)
|
|
agent = RagAgent(_AlwaysClarifyUnderstander(), _FixedRetrieval({}), answers)
|
|
|
|
for i in range(MAX_CONSECUTIVE_CLARIFY):
|
|
agent.handle(f"turn {i}", conversation_id="stuck")
|
|
reply = agent.handle("one more try", conversation_id="stuck")
|
|
|
|
assert reply.decision == "clarify"
|
|
|
|
|
|
def test_a_resolved_turn_resets_the_clarify_streak():
|
|
"""An answerable turn in between two clarify runs must not let their
|
|
streaks combine — only genuinely consecutive clarifies count."""
|
|
result = RetrievalResult(
|
|
EvidenceDecision.ANSWERABLE, "grounded_evidence_available",
|
|
(_evidence("Liều 500 mg."),),
|
|
)
|
|
clarify_frame = QueryFrame(
|
|
turn_type="dosing_calc", drugs=("paracetamol_acetaminophen",),
|
|
needs_clarify=True, clarify_reason="Bé nặng bao nhiêu kg?",
|
|
)
|
|
resolved_frame = QueryFrame(
|
|
turn_type="drug_attribute", drugs=("metformin",),
|
|
attribute="lieu_luong_va_cach_dung",
|
|
)
|
|
# (MAX-1) clarifies, one resolved turn, then (MAX-1) clarifies again —
|
|
# scripted explicitly so the test asserts the streak reset, not an
|
|
# incidental side effect of some other call-counting scheme.
|
|
script = (
|
|
[clarify_frame] * (MAX_CONSECUTIVE_CLARIFY - 1)
|
|
+ [resolved_frame]
|
|
+ [clarify_frame] * (MAX_CONSECUTIVE_CLARIFY - 1)
|
|
)
|
|
|
|
class _ScriptedUnderstander:
|
|
def __init__(self, frames):
|
|
self._frames = iter(frames)
|
|
|
|
def understand(self, turn, history=(), budget=None, prior_frame=None):
|
|
return next(self._frames)
|
|
|
|
answers = GroundedAnswerService(routing=None)
|
|
agent = RagAgent(
|
|
_ScriptedUnderstander(script), _FixedRetrieval({"metformin": result}), answers,
|
|
)
|
|
|
|
for i in range(MAX_CONSECUTIVE_CLARIFY - 1):
|
|
reply = agent.handle(f"clarify {i}", conversation_id="c1")
|
|
assert reply.decision == "clarify"
|
|
resolved = agent.handle("answerable turn", conversation_id="c1")
|
|
assert resolved.decision == "answerable"
|
|
|
|
# Streak was reset by the resolved turn -- this run of clarifies must
|
|
# not be treated as a continuation of the earlier (pre-reset) run.
|
|
for i in range(MAX_CONSECUTIVE_CLARIFY - 1):
|
|
reply = agent.handle(f"clarify again {i}", conversation_id="c1")
|
|
assert reply.decision == "clarify"
|
|
|
|
|
|
def test_prior_frame_is_threaded_from_the_last_turn_to_the_understander():
|
|
received: list[QueryFrame | None] = []
|
|
|
|
class _RecordingUnderstander:
|
|
def understand(self, turn, history=(), budget=None, prior_frame=None):
|
|
received.append(prior_frame)
|
|
return QueryFrame(
|
|
turn_type="dosing_calc", drugs=("paracetamol_acetaminophen",),
|
|
needs_clarify=True, clarify_reason="Bé nặng bao nhiêu kg?",
|
|
)
|
|
|
|
answers = GroundedAnswerService(routing=None)
|
|
agent = RagAgent(_RecordingUnderstander(), _FixedRetrieval({}), answers)
|
|
|
|
agent.handle("liều paracetamol cho bé", conversation_id="c1")
|
|
agent.handle("20kg", conversation_id="c1")
|
|
|
|
assert received[0] is None
|
|
assert received[1] is not None
|
|
assert received[1].clarify_reason == "Bé nặng bao nhiêu kg?"
|