"""`rag/agent.py` — the new orchestrator wired live in F-03. Codex's 2026-08-06 review (F-03/F-10) flagged that this module had zero test coverage despite being built to replace the live front end. This is the first coverage: routing branches with fakes, not an exhaustive golden set (that is F-10's job — a production-path regression suite against real Qdrant/Bedrock fixtures). """ from __future__ import annotations from rag.agent import MAX_CONSECUTIVE_CLARIFY, RagAgent from rag.answer import GroundedAnswerService from rag.metrics import GENERATION_REJECTED, InMemoryMetrics from rag.models import Evidence, EvidenceDecision, RetrievalResult, SourceRef from rag.understanding import QueryFrame SOURCE = SourceRef(physical_page=100, precision="region", printed_page=100) def _evidence(text: str, requires_visual_check: bool = False) -> Evidence: return Evidence( evidence_id="e0", matched_doc_id="e0", kind="prose", text=text, score=1.0, source_refs=(SOURCE,), hydrated_from_parent=False, requires_visual_check=requires_visual_check, ) class _FixedUnderstander: def __init__(self, frame: QueryFrame) -> None: self._frame = frame def understand(self, turn, history=(), budget=None, prior_frame=None): return self._frame class _FixedRetrieval: """Stands in for `RetrievalService.retrieve_framed` — a canned result per drug_id regardless of section/query, so these tests assert routing, not retrieval (that's `test_retrieval_service.py`'s job).""" def __init__( self, results: dict[str, RetrievalResult], indication_results: dict[str, RetrievalResult] | None = None, ) -> None: self._results = results self._indication_results = indication_results or {} self.calls: list[tuple[str, str | None, str]] = [] def retrieve_framed(self, drug_id, section_key, query, is_overview=False): self.calls.append((drug_id, section_key, query)) return self._results.get( drug_id, RetrievalResult(EvidenceDecision.ABSTAIN, "not_configured") ) def retrieve_by_indication(self, indication_text): return self._indication_results.get( indication_text, RetrievalResult(EvidenceDecision.ABSTAIN, "no_indication_match") ) def decide(self, evidence): # Mirrors `RetrievalService.decide`'s real policy (not a stub that # always says ANSWERABLE) so `_interaction`'s use of it is actually # under test, not just its own call site. if not evidence: return RetrievalResult(EvidenceDecision.ABSTAIN, "parent_hydration_failed") if any(item.requires_visual_check for item in evidence): return RetrievalResult(EvidenceDecision.VERIFY_PDF, "visual_verification_required", evidence) return RetrievalResult(EvidenceDecision.ANSWERABLE, "grounded_evidence_available", evidence) def _agent( frame: QueryFrame, results: dict[str, RetrievalResult] | None = None, indication_results: dict[str, RetrievalResult] | None = None, ) -> RagAgent: # `routing=None`: `answer_from_result` (the only method this path calls) # never touches it — see `rag/answer.py`. answers = GroundedAnswerService(routing=None) return RagAgent( _FixedUnderstander(frame), _FixedRetrieval(results or {}, indication_results), answers, ) def test_smalltalk_does_not_touch_retrieval(): agent = _agent(QueryFrame(turn_type="smalltalk")) reply = agent.handle("chào bạn") assert reply.decision == "answerable" assert reply.turn_type == "smalltalk" assert "Dược thư" in reply.answer def test_out_of_scope_turn_type_abstains(): agent = _agent(QueryFrame(turn_type="out_of_scope")) reply = agent.handle("cách tiêm truyền tĩnh mạch") assert reply.decision == "abstain" assert reply.reason == "out_of_scope" def test_veterinary_phrase_abstains_even_if_the_model_missed_it(): # The model returned an ordinary-looking frame; the keyword backstop # (`rag.policy.looks_non_human`, the same one F-02 wired server-side) # still catches it before any retrieval happens. agent = _agent(QueryFrame(turn_type="drug_attribute", drugs=("metformin",))) reply = agent.handle("liều metformin cho chó bao nhiêu") assert reply.decision == "abstain" assert reply.reason == "out_of_scope" def test_unknown_drug_name_is_reported_not_substituted(): agent = _agent(QueryFrame(turn_type="drug_attribute", unknown_drugs=("aspirinol",))) reply = agent.handle("liều aspirinol") assert reply.decision == "abstain" assert reply.reason == "drug_not_in_formulary" assert "aspirinol" in reply.answer def test_no_drug_named_asks_which_one(): agent = _agent(QueryFrame(turn_type="drug_attribute")) reply = agent.handle("liều dùng bao nhiêu") assert reply.decision == "clarify" assert reply.reason == "no_drug" def test_drug_attribute_without_an_attribute_does_not_fall_into_overview_retrieval(): retrieval = _FixedRetrieval({}) answers = GroundedAnswerService(routing=None) agent = RagAgent( _FixedUnderstander(QueryFrame( turn_type="drug_attribute", drugs=("paracetamol_acetaminophen",) )), retrieval, answers, ) reply = agent.handle("paracetamol thì sao?") assert reply.decision == "clarify" assert reply.reason == "missing_attribute" assert retrieval.calls == [] # --- the pediatric dosing gate. This code path gained its first test # coverage on 2026-08-11, after driving production reproduced the same # behaviour 5/5: the clarify question asked for both age and weight every # time, including the field the user had just supplied ("Bé 18 ký ...", # "Bé nặng 18 kg ...", "Trẻ 5 tuổi ..." all received "Bé bao nhiêu tuổi và # cân nặng bao nhiêu kg?"). Requiring BOTH fields is deliberate and stays — # the formulary bands paracetamol by age *and* by mg/kg — so these tests pin # the question text without loosening the requirement. -- def _pediatric_agent(**frame_kwargs) -> RagAgent: return _agent(QueryFrame( turn_type="dosing_calc", drugs=("paracetamol_acetaminophen",), population="tre_em", **frame_kwargs, )) def test_pediatric_gate_with_a_known_weight_asks_only_for_the_age(): reply = _pediatric_agent(weight_kg=18.0).handle("bé 18 ký sốt cao uống paracetamol liều bao nhiêu") assert reply.decision == "clarify" assert reply.reason == "missing_pediatric_age_or_weight" assert "tuổi" in reply.clarification # The weight is echoed back so the user can see it was received (and # catch a mis-parse), but is never asked for again. assert "18 kg" in reply.clarification assert "bao nhiêu kg" not in reply.clarification def test_pediatric_gate_with_a_known_age_asks_only_for_the_weight(): reply = _pediatric_agent(age_text="5 tuổi").handle("trẻ 5 tuổi sốt cao uống paracetamol liều bao nhiêu") assert reply.decision == "clarify" assert "kg" in reply.clarification assert "bao nhiêu tuổi" not in reply.clarification def test_pediatric_gate_with_neither_field_still_asks_for_both(): reply = _pediatric_agent().handle("liều paracetamol cho trẻ em") assert reply.decision == "clarify" assert "tuổi" in reply.clarification assert "kg" in reply.clarification def test_pediatric_gate_still_requires_both_fields_before_answering(): """The safety property, asserted directly: knowing only one of the two must NOT be treated as enough to pick a regimen. Both single-field cases above stop at `clarify` — this states the invariant so a future change that "helpfully" answers with weight alone fails here loudly.""" for kwargs in ({"weight_kg": 18.0}, {"age_text": "5 tuổi"}, {}): reply = _pediatric_agent(**kwargs).handle("liều paracetamol cho bé") assert reply.decision == "clarify", kwargs assert reply.reason == "missing_pediatric_age_or_weight", kwargs def test_a_whole_number_weight_is_not_echoed_with_a_trailing_zero(): reply = _pediatric_agent(weight_kg=18.0).handle("liều paracetamol cho bé 18 ký") assert "18 kg" in reply.clarification assert "18.0" not in reply.clarification def test_the_models_own_clarify_question_still_wins_over_the_generated_one(): """Unchanged precedence: when understanding.py produced a question of its own it is still preferred, because it can see phrasing/context this code-level fallback cannot.""" reply = _pediatric_agent( weight_kg=18.0, needs_clarify=True, clarify_reason="Bé mấy tháng tuổi rồi ạ?", ).handle("liều paracetamol cho bé 18 ký") assert reply.clarification == "Bé mấy tháng tuổi rồi ạ?" def test_needs_clarify_frame_is_surfaced_directly(): agent = _agent(QueryFrame( turn_type="dosing_calc", drugs=("paracetamol",), needs_clarify=True, clarify_reason="Bé mấy tuổi, cân nặng bao nhiêu kg?", )) reply = agent.handle("liều paracetamol cho trẻ em") assert reply.decision == "clarify" assert reply.clarification == "Bé mấy tuổi, cân nặng bao nhiêu kg?" # No quick_replies on the frame -> none surfaced (a specific weight has # no clean short options; must not be fabricated downstream). assert reply.quick_replies == () def test_understanding_provider_failure_is_an_abstain_not_a_fake_clarification(): agent = _agent(QueryFrame( turn_type="out_of_scope", needs_clarify=True, clarify_reason="Dịch vụ đang gặp sự cố tạm thời.", system_error="understanding_provider_unavailable", )) reply = agent.handle("liều paracetamol") assert reply.decision == "abstain" assert reply.reason == "understanding_provider_unavailable" assert reply.answer == "Dịch vụ đang gặp sự cố tạm thời." assert reply.clarification is None def test_needs_clarify_frame_carries_quick_replies_through(): """This is the path real traffic actually hits (checked live): the understanding LLM call itself sets needs_clarify/clarify_reason before retrieval ever runs, short-circuiting `_route` — quick_replies must survive that same short-circuit, not just the sufficiency-check path inside `GroundedAnswerService`.""" agent = _agent(QueryFrame( turn_type="drug_attribute", drugs=("paracetamol",), needs_clarify=True, clarify_reason="Người lớn hay trẻ em?", quick_replies=("Người lớn", "Trẻ em"), )) reply = agent.handle("liều paracetamol") assert reply.decision == "clarify" assert reply.quick_replies == ("Người lớn", "Trẻ em") def test_dosing_without_route_asks_only_when_evidence_is_ambiguous(): result = RetrievalResult( EvidenceDecision.ANSWERABLE, "grounded_evidence_available", ( _evidence("Đường uống, người lớn: 500 mg mỗi lần."), _evidence("Đặt trực tràng, người lớn: 500 mg mỗi lần."), ), resolved_drug_id="paracetamol_acetaminophen", ) class _Generator: def generate(self, system, user, schema): return ( '{"claims": [], "evidence_sufficient": false, ' '"clarifying_question": "Anh/chị muốn dùng đường nào?", ' '"quick_replies": ["Uống", "Đặt trực tràng"]}' ) retrieval = _FixedRetrieval({"paracetamol_acetaminophen": result}) answers = GroundedAnswerService(routing=None, generator=_Generator()) agent = RagAgent( _FixedUnderstander(QueryFrame( turn_type="dosing_calc", drugs=("paracetamol_acetaminophen",), population="nguoi_lon", )), retrieval, answers, ) reply = agent.handle("Người lớn", conversation_id="dose-route") assert reply.decision == "clarify" assert reply.reason == "needs_more_info" assert reply.quick_replies == ("Uống", "Đặt trực tràng") assert len(retrieval.calls) == 1 remembered = agent._last_frame["dose-route"] assert remembered.needs_clarify is True assert remembered.population == "nguoi_lon" assert remembered.clarify_reason == reply.clarification def test_dosing_without_route_answers_directly_when_evidence_has_one_route(): result = RetrievalResult( EvidenceDecision.ANSWERABLE, "grounded_evidence_available", (_evidence("Đường uống, người lớn: 500 mg mỗi lần."),), resolved_drug_id="paracetamol_acetaminophen", ) class _Generator: def generate(self, system, user, schema): if "entailed" in schema.get("properties", {}): return '{"entailed": true, "unsupported": []}' return ( '{"claims": [{"text": "Đường uống, người lớn: 500 mg mỗi lần.", ' '"citations": [1]}], "evidence_sufficient": true, ' '"clarifying_question": null, "quick_replies": []}' ) retrieval = _FixedRetrieval({"paracetamol_acetaminophen": result}) answers = GroundedAnswerService(routing=None, generator=_Generator()) agent = RagAgent( _FixedUnderstander(QueryFrame( turn_type="dosing_calc", drugs=("paracetamol_acetaminophen",), population="nguoi_lon", )), retrieval, answers, ) reply = agent.handle("Liều Paracetamol cho người lớn") assert reply.decision == "answerable" assert reply.quick_replies == () assert "500 mg" in reply.answer assert len(retrieval.calls) == 1 def test_general_dosage_section_survey_does_not_force_population_chip(): result = RetrievalResult( EvidenceDecision.ANSWERABLE, "grounded_evidence_available", (_evidence("Người lớn: uống 10 mg. Trẻ em: liều theo cân nặng."),), resolved_drug_id="example", ) retrieval = _FixedRetrieval({"example": result}) agent = RagAgent( _FixedUnderstander(QueryFrame( turn_type="dosing_calc", drugs=("example",), needs_clarify=True, clarify_reason="Người lớn hay trẻ em?", )), retrieval, GroundedAnswerService(routing=None), ) reply = agent.handle( "Dược thư hướng dẫn dùng Example thế nào: đường dùng và các liều nếu có?" ) assert reply.decision == "answerable" assert reply.quick_replies == () assert "tra cứu tổng quan toàn mục" in retrieval.calls[0][2] def test_clear_precaution_section_survey_overrides_model_overclarification(): result = RetrievalResult( EvidenceDecision.ANSWERABLE, "grounded_evidence_available", (_evidence("Theo dõi chức năng thận và điện giải."),), resolved_drug_id="example", ) agent = _agent( QueryFrame( turn_type="drug_attribute", drugs=("example",), attribute="than_trong", needs_clarify=True, clarify_reason="Muốn hỏi thận trọng hay chống chỉ định?", ), {"example": result}, ) reply = agent.handle("Những tình huống nào cần thận trọng khi dùng Example?") assert reply.decision == "answerable" assert reply.quick_replies == () def test_complete_adult_dosing_core_ignores_an_irrelevant_weight_reask(): result = RetrievalResult( EvidenceDecision.ANSWERABLE, "grounded_evidence_available", (_evidence("Đường uống, người lớn: 500 mg mỗi lần."),), resolved_drug_id="paracetamol_acetaminophen", ) retrieval = _FixedRetrieval({"paracetamol_acetaminophen": result}) answers = GroundedAnswerService(routing=None) agent = RagAgent( _FixedUnderstander(QueryFrame( turn_type="dosing_calc", drugs=("paracetamol_acetaminophen",), population="nguoi_lon", route="uong", needs_clarify=True, clarify_reason="Cân nặng của người lớn là bao nhiêu kg?", )), retrieval, answers, ) reply = agent.handle("Uống") assert reply.decision == "answerable" assert len(retrieval.calls) == 1 assert retrieval.calls[0][1] == "lieu_luong_va_cach_dung" def test_single_drug_attribute_retrieves_and_answers(): result = RetrievalResult( EvidenceDecision.ANSWERABLE, "grounded_evidence_available", (_evidence("Liều 500 mg mỗi ngày."),), resolved_drug_id="metformin", ) agent = _agent( QueryFrame(turn_type="drug_attribute", drugs=("metformin",), attribute="lieu_luong_va_cach_dung"), {"metformin": result}, ) reply = agent.handle("liều metformin") assert reply.decision == "answerable" assert reply.drugs == ("metformin",) assert "500 mg" in reply.answer def test_context_resolved_across_turns_is_folded_into_the_query(): """The 2026-08-07 P0 (named in the 2026-08-06 audit): population/weight/ age/route are extracted by understanding.py but were never passed into retrieval or generation — so a reply like "Uống" three turns into a dose conversation reached `GroundedAnswerService` as literally just "Uống", with no notion that population=adult was already established. Fixed via `_synthesize_query`; this asserts the synthesized text — not the bare turn — is what retrieval and generation actually see.""" result = RetrievalResult( EvidenceDecision.ANSWERABLE, "grounded_evidence_available", (_evidence("Liều uống người lớn: 500 mg."), _evidence("Liều tiêm người lớn: 1 g.")), resolved_drug_id="paracetamol_acetaminophen", ) retrieval = _FixedRetrieval({"paracetamol_acetaminophen": result}) agent = RagAgent( _FixedUnderstander(QueryFrame( turn_type="dosing_calc", drugs=("paracetamol_acetaminophen",), attribute="lieu_luong_va_cach_dung", population="nguoi_lon", route="uong", needs_clarify=False, )), retrieval, GroundedAnswerService(routing=None), ) agent.handle("Uống") assert len(retrieval.calls) == 1 _, _, query = retrieval.calls[0] assert query.startswith("Uống") assert "người lớn" in query assert "uống" in query.lower() def test_context_synthesis_is_a_no_op_when_the_frame_has_no_resolved_fields(): """A fresh, fully-specified single-shot question already states its own context — synthesis must not alter it or introduce redundant noise.""" result = RetrievalResult( EvidenceDecision.ANSWERABLE, "grounded_evidence_available", (_evidence("x"), _evidence("y")), resolved_drug_id="metformin", ) retrieval = _FixedRetrieval({"metformin": result}) agent = RagAgent( _FixedUnderstander(QueryFrame( turn_type="drug_attribute", drugs=("metformin",), attribute="chong_chi_dinh", )), retrieval, GroundedAnswerService(routing=None), ) agent.handle("Chống chỉ định của metformin là gì?") assert retrieval.calls[0][2] == "Chống chỉ định của metformin là gì?" def test_interaction_combines_both_drugs_evidence(): warfarin = RetrievalResult( EvidenceDecision.ANSWERABLE, "grounded_evidence_available", (_evidence("Tương tác với aspirin làm tăng nguy cơ chảy máu."),), ) aspirin = RetrievalResult( EvidenceDecision.ANSWERABLE, "grounded_evidence_available", (_evidence("Tương tác với warfarin làm tăng nguy cơ chảy máu."),), ) agent = _agent( QueryFrame(turn_type="interaction", drugs=("warfarin", "aspirin")), {"warfarin": warfarin, "aspirin": aspirin}, ) reply = agent.handle("warfarin với aspirin có dùng chung được không") assert reply.decision == "answerable" assert reply.turn_type == "interaction" assert "chảy máu" in reply.answer def test_interaction_with_one_drug_quarantined_never_answers_confidently(): """The P0 the 2026-08-06 audit found: `_interaction` used to keep only `ANSWERABLE` parts, so a quarantined drug's evidence (and the "table exists, verify PDF" notice the quarantine contract requires) was silently dropped — a confident interaction answer could omit a real unverified contraindication table for one of the two drugs. Fixed via `RetrievalService.decide` applied to the combined pool, the same policy the single-drug path already uses.""" warfarin = RetrievalResult( EvidenceDecision.ANSWERABLE, "grounded_evidence_available", (_evidence("Tương tác với aspirin làm tăng nguy cơ chảy máu."),), ) aspirin = RetrievalResult( EvidenceDecision.VERIFY_PDF, "visual_verification_required", (_evidence("Bảng tương tác cần đối chiếu PDF.", requires_visual_check=True),), ) agent = _agent( QueryFrame(turn_type="interaction", drugs=("warfarin", "aspirin")), {"warfarin": warfarin, "aspirin": aspirin}, ) reply = agent.handle("warfarin với aspirin có dùng chung được không") # Must NOT be a confident "answerable" that silently omits aspirin's # quarantined table — must ask for PDF verification instead. assert reply.decision == "verify_pdf" # Both drugs' evidence must still be present (as citations), not dropped. assert len(reply.citations) == 2 def test_interaction_with_no_evidence_abstains_and_does_not_imply_safety(): agent = _agent( QueryFrame(turn_type="interaction", drugs=("drug_a", "drug_b")), {}, ) reply = agent.handle("drug_a và drug_b dùng chung được không") assert reply.decision == "abstain" assert reply.reason == "no_interaction_evidence" assert "KHÔNG có nghĩa là an toàn" in reply.answer def test_symptom_to_drug_with_no_indication_extracted_asks_for_one(): agent = _agent(QueryFrame(turn_type="symptom_to_drug")) reply = agent.handle("có thuốc gì không") assert reply.decision == "clarify" assert reply.reason == "no_indication" def test_symptom_to_drug_with_no_match_abstains_not_silently_safe(): agent = _agent( QueryFrame(turn_type="symptom_to_drug", indication="bệnh hiếm gặp x"), indication_results={ "bệnh hiếm gặp x": RetrievalResult(EvidenceDecision.ABSTAIN, "no_indication_match"), }, ) reply = agent.handle("thuốc gì trị bệnh hiếm gặp x") assert reply.decision == "abstain" assert "bệnh hiếm gặp x" in reply.answer assert "KHÔNG" in reply.answer def test_symptom_to_drug_returns_the_matched_drugs_not_frame_drugs(): """`frame.drugs` is empty by construction for this turn_type (the router only reaches `_symptom_to_drug` with no named drug) — the reply's `drugs` must come from what retrieval actually found.""" result = RetrievalResult( EvidenceDecision.ANSWERABLE, "grounded_evidence_available", (_evidence("Paracetamol chỉ định hạ sốt."), _evidence("Ibuprofen chỉ định hạ sốt, giảm đau.")), ) # Overwrite matched_doc_id per evidence to simulate two different drugs # (the shared `_evidence` helper always uses "e0" — construct directly). ev_a = Evidence( evidence_id="paracetamol_acetaminophen__chi_dinh__0", matched_doc_id="paracetamol_acetaminophen__chi_dinh__0", kind="prose", text="Paracetamol chỉ định hạ sốt.", score=1.0, source_refs=(SOURCE,), hydrated_from_parent=False, requires_visual_check=False, ) ev_b = Evidence( evidence_id="ibuprofen__chi_dinh__0", matched_doc_id="ibuprofen__chi_dinh__0", kind="prose", text="Ibuprofen chỉ định hạ sốt, giảm đau.", score=1.0, source_refs=(SOURCE,), hydrated_from_parent=False, requires_visual_check=False, ) result = RetrievalResult( EvidenceDecision.ANSWERABLE, "grounded_evidence_available", (ev_a, ev_b), ) agent = _agent( QueryFrame(turn_type="symptom_to_drug", indication="sốt"), indication_results={"sốt": result}, ) reply = agent.handle("sốt thì uống thuốc gì") assert reply.decision == "answerable" assert reply.drugs == ("paracetamol_acetaminophen", "ibuprofen") def test_history_is_passed_to_the_understander_on_the_next_turn(): received_history: list[tuple[str, ...]] = [] class _RecordingUnderstander: def understand(self, turn, history=(), budget=None, prior_frame=None): received_history.append(tuple(history)) return QueryFrame(turn_type="smalltalk") answers = GroundedAnswerService(routing=None) agent = RagAgent(_RecordingUnderstander(), _FixedRetrieval({}), answers) agent.handle("chào bạn", conversation_id="c1") agent.handle("còn liều thì sao?", conversation_id="c1") assert received_history[0] == () assert any("chào bạn" in line for line in received_history[1]) def test_history_is_isolated_per_conversation_id(): received_history: list[tuple[str, ...]] = [] class _RecordingUnderstander: def understand(self, turn, history=(), budget=None, prior_frame=None): received_history.append(tuple(history)) return QueryFrame(turn_type="smalltalk") answers = GroundedAnswerService(routing=None) agent = RagAgent(_RecordingUnderstander(), _FixedRetrieval({}), answers) agent.handle("xin chào", conversation_id="a") agent.handle("liều dùng bao nhiêu", conversation_id="b") # The second call, on a different conversation id, must not see "a"'s turn. assert received_history[1] == () # --- F-10: `conversation_id` presence/absence must not change the safety # decision on a fresh (first) turn — only whether the turn is remembered # afterward ------------------------------------------------------------- def test_conversation_id_presence_or_absence_reaches_the_same_decision(): """A single-turn call (no `conversation_id`) and the first turn of a fresh multi-turn conversation must resolve identically — both see empty history, so nothing about `conversation_id` itself may become a second, undocumented safety signal.""" agent = _agent(QueryFrame( turn_type="drug_attribute", unknown_drugs=("aspirinol",), )) without_id = agent.handle("liều aspirinol") with_id = agent.handle("liều aspirinol", conversation_id="fresh-conv") assert without_id.decision == with_id.decision == "abstain" assert without_id.reason == with_id.reason == "drug_not_in_formulary" def test_autocomplete_delegates_to_the_configured_source(): class _Source: def complete(self, prefix, k): return ["metformin_id", "metoprolol_id"] answers = GroundedAnswerService(routing=None) agent = RagAgent( _FixedUnderstander(QueryFrame(turn_type="smalltalk")), _FixedRetrieval({}), answers, autocomplete=_Source(), ) assert agent.complete("met") == ["Metformin Id", "Metoprolol Id"] def test_autocomplete_with_no_source_configured_returns_empty(): answers = GroundedAnswerService(routing=None) agent = RagAgent( _FixedUnderstander(QueryFrame(turn_type="smalltalk")), _FixedRetrieval({}), answers, ) assert agent.complete("met") == [] # --- F-08: a per-turn budget actually stops real provider calls once spent, # it isn't just bookkeeping --------------------------------------------- def test_a_budget_exhausted_during_understand_blocks_every_later_call(): """`max_llm_calls_per_turn=1` means the (simulated) `understand()` call spends the entire turn's budget — sufficiency and generate must never reach the generator at all, not just receive an error from it. Proves the budget is threaded end to end through `RagAgent`, not only present at the one call site each unit test exercises in isolation.""" class _BudgetSpendingUnderstander: """Stands in for the real `LlmQueryUnderstander`, which calls `budget.require()` once before its own LLM call — simulated here so this test doesn't need a live-shaped LLM fake for the understand step, only for the reply's calls to fail budget.""" def understand(self, turn, history=(), budget=None, prior_frame=None): if budget is not None: budget.require() return QueryFrame( turn_type="drug_attribute", drugs=("metformin",), attribute="chi_dinh", ) result = RetrievalResult( EvidenceDecision.ANSWERABLE, "grounded_evidence_available", (_evidence("đoạn 0"), _evidence("đoạn 1")), ) generator_calls: list[dict] = [] class _CountingGenerator: def generate(self, system, user, schema): generator_calls.append(schema) return '{"answer": "unused", "evidence_sufficient": true, "clarifying_question": null}' metrics = InMemoryMetrics() answers = GroundedAnswerService( routing=None, generator=_CountingGenerator(), metrics=metrics ) agent = RagAgent( _BudgetSpendingUnderstander(), _FixedRetrieval({"metformin": result}), answers, max_llm_calls_per_turn=1, ) reply = agent.handle("liều metformin") # The real point of F-08: no more actual provider calls happen once the # budget is spent — not "the generator returned an error", the generator # is never invoked at all. assert generator_calls == [] assert reply.decision == "abstain" assert reply.answer is None assert metrics.total(GENERATION_REJECTED, reason="request_budget_exhausted") >= 1 def test_a_generous_budget_does_not_change_normal_behaviour(): """Control for the test above: with the default (generous) budget, the same setup answers normally — proves the previous test's tiny budget is what caused the block, not some other change to the fixtures.""" class _BudgetSpendingUnderstander: def understand(self, turn, history=(), budget=None, prior_frame=None): if budget is not None: budget.require() return QueryFrame( turn_type="drug_attribute", drugs=("metformin",), attribute="chi_dinh", ) result = RetrievalResult( EvidenceDecision.ANSWERABLE, "grounded_evidence_available", (_evidence("Liều 500 mg mỗi ngày."), _evidence("đoạn 1")), ) class _Generator: def generate(self, system, user, schema): if "sufficient" in schema.get("properties", {}): return '{"sufficient": true, "clarifying_question": null, "quick_replies": []}' if "entailed" in schema.get("properties", {}): return '{"entailed": true, "unsupported": []}' return ( '{"claims": [{"text": "Liều 500 mg", "citations": [1]}], ' '"evidence_sufficient": true, "clarifying_question": null}' ) answers = GroundedAnswerService(routing=None, generator=_Generator()) agent = RagAgent( _BudgetSpendingUnderstander(), _FixedRetrieval({"metformin": result}), answers, ) reply = agent.handle("liều metformin") assert reply.decision == "answerable" assert reply.answer == "Liều 500 mg" assert reply.blocks[0].claims[0].source_ids # --- durable conversation history (ADR 0008's named gap): an optional # `ConversationStore` replaces the in-process dict when configured -------- class _FakeStore: def __init__(self) -> None: self.lines: dict[str, list[str]] = {} self.fail_reads = False self.fail_writes = False def recent(self, conversation_id, limit): if self.fail_reads: raise ConnectionError("store unreachable") return self.lines.get(conversation_id, [])[-limit:] def append(self, conversation_id, line): if self.fail_writes: raise ConnectionError("store unreachable") self.lines.setdefault(conversation_id, []).append(line) def test_history_round_trips_through_a_configured_store(): received_history: list[tuple[str, ...]] = [] class _RecordingUnderstander: def understand(self, turn, history=(), budget=None, prior_frame=None): received_history.append(tuple(history)) return QueryFrame(turn_type="smalltalk") store = _FakeStore() answers = GroundedAnswerService(routing=None) agent = RagAgent(_RecordingUnderstander(), _FixedRetrieval({}), answers, store=store) agent.handle("chào bạn", conversation_id="c1") agent.handle("còn liều thì sao?", conversation_id="c1") assert received_history[0] == () assert any("chào bạn" in line for line in received_history[1]) assert store.lines["c1"] # actually persisted, not just read back in-process def test_store_read_failure_fails_open_to_fresh_history_not_a_crash(): store = _FakeStore() store.lines["c1"] = ["Người dùng: câu cũ", "Trợ lý: trả lời cũ"] store.fail_reads = True answers = GroundedAnswerService(routing=None) agent = RagAgent( _FixedUnderstander(QueryFrame(turn_type="smalltalk")), _FixedRetrieval({}), answers, store=store, ) # Must not raise -- degrades to no history for this turn. reply = agent.handle("chào bạn", conversation_id="c1") assert reply.decision == "answerable" def test_store_write_failure_fails_open_the_response_still_returns(): store = _FakeStore() store.fail_writes = True answers = GroundedAnswerService(routing=None) agent = RagAgent( _FixedUnderstander(QueryFrame(turn_type="smalltalk")), _FixedRetrieval({}), answers, store=store, ) # Must not raise, even though persisting this turn silently fails. reply = agent.handle("chào bạn", conversation_id="c1") assert reply.decision == "answerable" assert reply.answer is not None # --- F-11: the clarify-loop circuit breaker. Found live 2026-08-07 # (50-question hand-typed browser audit): the understanding LLM can keep # deciding needs_clarify=true forever with no natural exit — reproduced 3 # times independently, one case never converged after 5 real answered turns. # `understanding.py`'s prior-frame merge fixes most of the underlying cause, # but this is the code-level bound that guarantees a user is never stuck. -- class _AlwaysClarifyUnderstander: """Simulates a model stuck re-asking regardless of what the user answers — exactly the observed live failure, isolated from any real LLM's actual (variable) behaviour so this test is deterministic.""" def understand(self, turn, history=(), budget=None, prior_frame=None): return QueryFrame( turn_type="dosing_calc", drugs=("paracetamol_acetaminophen",), needs_clarify=True, clarify_reason="Bé nặng bao nhiêu kg?", ) def test_clarify_loop_is_hard_stopped_after_max_consecutive_turns(): answers = GroundedAnswerService(routing=None) agent = RagAgent(_AlwaysClarifyUnderstander(), _FixedRetrieval({}), answers) replies = [ agent.handle(f"turn {i}", conversation_id="stuck") for i in range(MAX_CONSECUTIVE_CLARIFY) ] # Every turn up to the last stays a genuine clarify... for reply in replies[:-1]: assert reply.decision == "clarify" # ...the Nth forces a hard stop instead of asking again. assert replies[-1].decision == "abstain" assert replies[-1].reason == "clarify_loop_exhausted" assert replies[-1].answer is not None def test_clarify_streak_resets_after_the_hard_stop_so_a_new_attempt_can_proceed(): """Confirms the breaker is a bounded pause, not a permanent lockout of the conversation id — the very next turn gets a fresh streak.""" answers = GroundedAnswerService(routing=None) agent = RagAgent(_AlwaysClarifyUnderstander(), _FixedRetrieval({}), answers) for i in range(MAX_CONSECUTIVE_CLARIFY): agent.handle(f"turn {i}", conversation_id="stuck") reply = agent.handle("one more try", conversation_id="stuck") assert reply.decision == "clarify" def test_a_resolved_turn_resets_the_clarify_streak(): """An answerable turn in between two clarify runs must not let their streaks combine — only genuinely consecutive clarifies count.""" result = RetrievalResult( EvidenceDecision.ANSWERABLE, "grounded_evidence_available", (_evidence("Liều 500 mg."),), ) clarify_frame = QueryFrame( turn_type="dosing_calc", drugs=("paracetamol_acetaminophen",), needs_clarify=True, clarify_reason="Bé nặng bao nhiêu kg?", ) resolved_frame = QueryFrame( turn_type="drug_attribute", drugs=("metformin",), attribute="lieu_luong_va_cach_dung", ) # (MAX-1) clarifies, one resolved turn, then (MAX-1) clarifies again — # scripted explicitly so the test asserts the streak reset, not an # incidental side effect of some other call-counting scheme. script = ( [clarify_frame] * (MAX_CONSECUTIVE_CLARIFY - 1) + [resolved_frame] + [clarify_frame] * (MAX_CONSECUTIVE_CLARIFY - 1) ) class _ScriptedUnderstander: def __init__(self, frames): self._frames = iter(frames) def understand(self, turn, history=(), budget=None, prior_frame=None): return next(self._frames) answers = GroundedAnswerService(routing=None) agent = RagAgent( _ScriptedUnderstander(script), _FixedRetrieval({"metformin": result}), answers, ) for i in range(MAX_CONSECUTIVE_CLARIFY - 1): reply = agent.handle(f"clarify {i}", conversation_id="c1") assert reply.decision == "clarify" resolved = agent.handle("answerable turn", conversation_id="c1") assert resolved.decision == "answerable" # Streak was reset by the resolved turn -- this run of clarifies must # not be treated as a continuation of the earlier (pre-reset) run. for i in range(MAX_CONSECUTIVE_CLARIFY - 1): reply = agent.handle(f"clarify again {i}", conversation_id="c1") assert reply.decision == "clarify" def test_prior_frame_is_threaded_from_the_last_turn_to_the_understander(): received: list[QueryFrame | None] = [] class _RecordingUnderstander: def understand(self, turn, history=(), budget=None, prior_frame=None): received.append(prior_frame) return QueryFrame( turn_type="dosing_calc", drugs=("paracetamol_acetaminophen",), needs_clarify=True, clarify_reason="Bé nặng bao nhiêu kg?", ) answers = GroundedAnswerService(routing=None) agent = RagAgent(_RecordingUnderstander(), _FixedRetrieval({}), answers) agent.handle("liều paracetamol cho bé", conversation_id="c1") agent.handle("20kg", conversation_id="c1") assert received[0] is None assert received[1] is not None assert received[1].clarify_reason == "Bé nặng bao nhiêu kg?"