Fix ai-service Dockerfile: bake in drug_entities.json, override its path
This commit is contained in:
@@ -8,18 +8,20 @@ Qdrant/Bedrock fixtures).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from rag.agent import RagAgent
|
||||
from rag.agent import MAX_CONSECUTIVE_CLARIFY, RagAgent
|
||||
from rag.answer import GroundedAnswerService
|
||||
from rag.metrics import GENERATION_REJECTED, InMemoryMetrics
|
||||
from rag.models import Evidence, EvidenceDecision, RetrievalResult, SourceRef
|
||||
from rag.understanding import QueryFrame
|
||||
|
||||
SOURCE = SourceRef(physical_page=100, precision="region", printed_page=100)
|
||||
|
||||
|
||||
def _evidence(text: str) -> Evidence:
|
||||
def _evidence(text: str, requires_visual_check: bool = False) -> Evidence:
|
||||
return Evidence(
|
||||
evidence_id="e0", matched_doc_id="e0", kind="prose", text=text, score=1.0,
|
||||
source_refs=(SOURCE,), hydrated_from_parent=False, requires_visual_check=False,
|
||||
source_refs=(SOURCE,), hydrated_from_parent=False,
|
||||
requires_visual_check=requires_visual_check,
|
||||
)
|
||||
|
||||
|
||||
@@ -27,7 +29,7 @@ class _FixedUnderstander:
|
||||
def __init__(self, frame: QueryFrame) -> None:
|
||||
self._frame = frame
|
||||
|
||||
def understand(self, turn, history=()):
|
||||
def understand(self, turn, history=(), budget=None, prior_frame=None):
|
||||
return self._frame
|
||||
|
||||
|
||||
@@ -36,20 +38,50 @@ class _FixedRetrieval:
|
||||
drug_id regardless of section/query, so these tests assert routing, not
|
||||
retrieval (that's `test_retrieval_service.py`'s job)."""
|
||||
|
||||
def __init__(self, results: dict[str, RetrievalResult]) -> None:
|
||||
def __init__(
|
||||
self,
|
||||
results: dict[str, RetrievalResult],
|
||||
indication_results: dict[str, RetrievalResult] | None = None,
|
||||
) -> None:
|
||||
self._results = results
|
||||
self._indication_results = indication_results or {}
|
||||
self.calls: list[tuple[str, str | None, str]] = []
|
||||
|
||||
def retrieve_framed(self, drug_id, section_key, query, is_overview=False):
|
||||
self.calls.append((drug_id, section_key, query))
|
||||
return self._results.get(
|
||||
drug_id, RetrievalResult(EvidenceDecision.ABSTAIN, "not_configured")
|
||||
)
|
||||
|
||||
def retrieve_by_indication(self, indication_text):
|
||||
return self._indication_results.get(
|
||||
indication_text, RetrievalResult(EvidenceDecision.ABSTAIN, "no_indication_match")
|
||||
)
|
||||
|
||||
def _agent(frame: QueryFrame, results: dict[str, RetrievalResult] | None = None) -> RagAgent:
|
||||
def decide(self, evidence):
|
||||
# Mirrors `RetrievalService.decide`'s real policy (not a stub that
|
||||
# always says ANSWERABLE) so `_interaction`'s use of it is actually
|
||||
# under test, not just its own call site.
|
||||
if not evidence:
|
||||
return RetrievalResult(EvidenceDecision.ABSTAIN, "parent_hydration_failed")
|
||||
if any(item.requires_visual_check for item in evidence):
|
||||
return RetrievalResult(EvidenceDecision.VERIFY_PDF, "visual_verification_required", evidence)
|
||||
return RetrievalResult(EvidenceDecision.ANSWERABLE, "grounded_evidence_available", evidence)
|
||||
|
||||
|
||||
def _agent(
|
||||
frame: QueryFrame,
|
||||
results: dict[str, RetrievalResult] | None = None,
|
||||
indication_results: dict[str, RetrievalResult] | None = None,
|
||||
) -> RagAgent:
|
||||
# `routing=None`: `answer_from_result` (the only method this path calls)
|
||||
# never touches it — see `rag/answer.py`.
|
||||
answers = GroundedAnswerService(routing=None)
|
||||
return RagAgent(_FixedUnderstander(frame), _FixedRetrieval(results or {}), answers)
|
||||
return RagAgent(
|
||||
_FixedUnderstander(frame),
|
||||
_FixedRetrieval(results or {}, indication_results),
|
||||
answers,
|
||||
)
|
||||
|
||||
|
||||
def test_smalltalk_does_not_touch_retrieval():
|
||||
@@ -100,6 +132,25 @@ def test_needs_clarify_frame_is_surfaced_directly():
|
||||
reply = agent.handle("liều paracetamol cho trẻ em")
|
||||
assert reply.decision == "clarify"
|
||||
assert reply.clarification == "Bé mấy tuổi, cân nặng bao nhiêu kg?"
|
||||
# No quick_replies on the frame -> none surfaced (a specific weight has
|
||||
# no clean short options; must not be fabricated downstream).
|
||||
assert reply.quick_replies == ()
|
||||
|
||||
|
||||
def test_needs_clarify_frame_carries_quick_replies_through():
|
||||
"""This is the path real traffic actually hits (checked live): the
|
||||
understanding LLM call itself sets needs_clarify/clarify_reason before
|
||||
retrieval ever runs, short-circuiting `_route` — quick_replies must
|
||||
survive that same short-circuit, not just the sufficiency-check path
|
||||
inside `GroundedAnswerService`."""
|
||||
agent = _agent(QueryFrame(
|
||||
turn_type="drug_attribute", drugs=("paracetamol",),
|
||||
needs_clarify=True, clarify_reason="Người lớn hay trẻ em?",
|
||||
quick_replies=("Người lớn", "Trẻ em"),
|
||||
))
|
||||
reply = agent.handle("liều paracetamol")
|
||||
assert reply.decision == "clarify"
|
||||
assert reply.quick_replies == ("Người lớn", "Trẻ em")
|
||||
|
||||
|
||||
def test_single_drug_attribute_retrieves_and_answers():
|
||||
@@ -118,6 +169,58 @@ def test_single_drug_attribute_retrieves_and_answers():
|
||||
assert "500 mg" in reply.answer
|
||||
|
||||
|
||||
def test_context_resolved_across_turns_is_folded_into_the_query():
|
||||
"""The 2026-08-07 P0 (named in the 2026-08-06 audit): population/weight/
|
||||
age/route are extracted by understanding.py but were never passed into
|
||||
retrieval or generation — so a reply like "Uống" three turns into a dose
|
||||
conversation reached `GroundedAnswerService` as literally just "Uống",
|
||||
with no notion that population=adult was already established. Fixed via
|
||||
`_synthesize_query`; this asserts the synthesized text — not the bare
|
||||
turn — is what retrieval and generation actually see."""
|
||||
result = RetrievalResult(
|
||||
EvidenceDecision.ANSWERABLE, "grounded_evidence_available",
|
||||
(_evidence("Liều uống người lớn: 500 mg."),
|
||||
_evidence("Liều tiêm người lớn: 1 g.")),
|
||||
resolved_drug_id="paracetamol_acetaminophen",
|
||||
)
|
||||
retrieval = _FixedRetrieval({"paracetamol_acetaminophen": result})
|
||||
agent = RagAgent(
|
||||
_FixedUnderstander(QueryFrame(
|
||||
turn_type="dosing_calc", drugs=("paracetamol_acetaminophen",),
|
||||
attribute="lieu_luong_va_cach_dung", population="nguoi_lon",
|
||||
route="uong", needs_clarify=False,
|
||||
)),
|
||||
retrieval,
|
||||
GroundedAnswerService(routing=None),
|
||||
)
|
||||
agent.handle("Uống")
|
||||
assert len(retrieval.calls) == 1
|
||||
_, _, query = retrieval.calls[0]
|
||||
assert query.startswith("Uống")
|
||||
assert "người lớn" in query
|
||||
assert "uống" in query.lower()
|
||||
|
||||
|
||||
def test_context_synthesis_is_a_no_op_when_the_frame_has_no_resolved_fields():
|
||||
"""A fresh, fully-specified single-shot question already states its own
|
||||
context — synthesis must not alter it or introduce redundant noise."""
|
||||
result = RetrievalResult(
|
||||
EvidenceDecision.ANSWERABLE, "grounded_evidence_available",
|
||||
(_evidence("x"), _evidence("y")), resolved_drug_id="metformin",
|
||||
)
|
||||
retrieval = _FixedRetrieval({"metformin": result})
|
||||
agent = RagAgent(
|
||||
_FixedUnderstander(QueryFrame(
|
||||
turn_type="drug_attribute", drugs=("metformin",),
|
||||
attribute="chong_chi_dinh",
|
||||
)),
|
||||
retrieval,
|
||||
GroundedAnswerService(routing=None),
|
||||
)
|
||||
agent.handle("Chống chỉ định của metformin là gì?")
|
||||
assert retrieval.calls[0][2] == "Chống chỉ định của metformin là gì?"
|
||||
|
||||
|
||||
def test_interaction_combines_both_drugs_evidence():
|
||||
warfarin = RetrievalResult(
|
||||
EvidenceDecision.ANSWERABLE, "grounded_evidence_available",
|
||||
@@ -137,6 +240,34 @@ def test_interaction_combines_both_drugs_evidence():
|
||||
assert "chảy máu" in reply.answer
|
||||
|
||||
|
||||
def test_interaction_with_one_drug_quarantined_never_answers_confidently():
|
||||
"""The P0 the 2026-08-06 audit found: `_interaction` used to keep only
|
||||
`ANSWERABLE` parts, so a quarantined drug's evidence (and the "table
|
||||
exists, verify PDF" notice the quarantine contract requires) was
|
||||
silently dropped — a confident interaction answer could omit a real
|
||||
unverified contraindication table for one of the two drugs. Fixed via
|
||||
`RetrievalService.decide` applied to the combined pool, the same policy
|
||||
the single-drug path already uses."""
|
||||
warfarin = RetrievalResult(
|
||||
EvidenceDecision.ANSWERABLE, "grounded_evidence_available",
|
||||
(_evidence("Tương tác với aspirin làm tăng nguy cơ chảy máu."),),
|
||||
)
|
||||
aspirin = RetrievalResult(
|
||||
EvidenceDecision.VERIFY_PDF, "visual_verification_required",
|
||||
(_evidence("Bảng tương tác cần đối chiếu PDF.", requires_visual_check=True),),
|
||||
)
|
||||
agent = _agent(
|
||||
QueryFrame(turn_type="interaction", drugs=("warfarin", "aspirin")),
|
||||
{"warfarin": warfarin, "aspirin": aspirin},
|
||||
)
|
||||
reply = agent.handle("warfarin với aspirin có dùng chung được không")
|
||||
# Must NOT be a confident "answerable" that silently omits aspirin's
|
||||
# quarantined table — must ask for PDF verification instead.
|
||||
assert reply.decision == "verify_pdf"
|
||||
# Both drugs' evidence must still be present (as citations), not dropped.
|
||||
assert len(reply.citations) == 2
|
||||
|
||||
|
||||
def test_interaction_with_no_evidence_abstains_and_does_not_imply_safety():
|
||||
agent = _agent(
|
||||
QueryFrame(turn_type="interaction", drugs=("drug_a", "drug_b")), {},
|
||||
@@ -147,18 +278,65 @@ def test_interaction_with_no_evidence_abstains_and_does_not_imply_safety():
|
||||
assert "KHÔNG có nghĩa là an toàn" in reply.answer
|
||||
|
||||
|
||||
def test_symptom_to_drug_without_a_drug_name_asks_honestly_not_wired_yet():
|
||||
agent = _agent(QueryFrame(turn_type="symptom_to_drug", indication="sốt cao"))
|
||||
reply = agent.handle("sốt cao uống thuốc gì")
|
||||
def test_symptom_to_drug_with_no_indication_extracted_asks_for_one():
|
||||
agent = _agent(QueryFrame(turn_type="symptom_to_drug"))
|
||||
reply = agent.handle("có thuốc gì không")
|
||||
assert reply.decision == "clarify"
|
||||
assert reply.reason == "reverse_lookup_not_ready"
|
||||
assert reply.reason == "no_indication"
|
||||
|
||||
|
||||
def test_symptom_to_drug_with_no_match_abstains_not_silently_safe():
|
||||
agent = _agent(
|
||||
QueryFrame(turn_type="symptom_to_drug", indication="bệnh hiếm gặp x"),
|
||||
indication_results={
|
||||
"bệnh hiếm gặp x": RetrievalResult(EvidenceDecision.ABSTAIN, "no_indication_match"),
|
||||
},
|
||||
)
|
||||
reply = agent.handle("thuốc gì trị bệnh hiếm gặp x")
|
||||
assert reply.decision == "abstain"
|
||||
assert "bệnh hiếm gặp x" in reply.answer
|
||||
assert "KHÔNG" in reply.answer
|
||||
|
||||
|
||||
def test_symptom_to_drug_returns_the_matched_drugs_not_frame_drugs():
|
||||
"""`frame.drugs` is empty by construction for this turn_type (the router
|
||||
only reaches `_symptom_to_drug` with no named drug) — the reply's
|
||||
`drugs` must come from what retrieval actually found."""
|
||||
result = RetrievalResult(
|
||||
EvidenceDecision.ANSWERABLE, "grounded_evidence_available",
|
||||
(_evidence("Paracetamol chỉ định hạ sốt."),
|
||||
_evidence("Ibuprofen chỉ định hạ sốt, giảm đau.")),
|
||||
)
|
||||
# Overwrite matched_doc_id per evidence to simulate two different drugs
|
||||
# (the shared `_evidence` helper always uses "e0" — construct directly).
|
||||
ev_a = Evidence(
|
||||
evidence_id="paracetamol_acetaminophen__chi_dinh__0",
|
||||
matched_doc_id="paracetamol_acetaminophen__chi_dinh__0",
|
||||
kind="prose", text="Paracetamol chỉ định hạ sốt.", score=1.0,
|
||||
source_refs=(SOURCE,), hydrated_from_parent=False, requires_visual_check=False,
|
||||
)
|
||||
ev_b = Evidence(
|
||||
evidence_id="ibuprofen__chi_dinh__0", matched_doc_id="ibuprofen__chi_dinh__0",
|
||||
kind="prose", text="Ibuprofen chỉ định hạ sốt, giảm đau.", score=1.0,
|
||||
source_refs=(SOURCE,), hydrated_from_parent=False, requires_visual_check=False,
|
||||
)
|
||||
result = RetrievalResult(
|
||||
EvidenceDecision.ANSWERABLE, "grounded_evidence_available", (ev_a, ev_b),
|
||||
)
|
||||
agent = _agent(
|
||||
QueryFrame(turn_type="symptom_to_drug", indication="sốt"),
|
||||
indication_results={"sốt": result},
|
||||
)
|
||||
reply = agent.handle("sốt thì uống thuốc gì")
|
||||
assert reply.decision == "answerable"
|
||||
assert reply.drugs == ("paracetamol_acetaminophen", "ibuprofen")
|
||||
|
||||
|
||||
def test_history_is_passed_to_the_understander_on_the_next_turn():
|
||||
received_history: list[tuple[str, ...]] = []
|
||||
|
||||
class _RecordingUnderstander:
|
||||
def understand(self, turn, history=()):
|
||||
def understand(self, turn, history=(), budget=None, prior_frame=None):
|
||||
received_history.append(tuple(history))
|
||||
return QueryFrame(turn_type="smalltalk")
|
||||
|
||||
@@ -176,7 +354,7 @@ def test_history_is_isolated_per_conversation_id():
|
||||
received_history: list[tuple[str, ...]] = []
|
||||
|
||||
class _RecordingUnderstander:
|
||||
def understand(self, turn, history=()):
|
||||
def understand(self, turn, history=(), budget=None, prior_frame=None):
|
||||
received_history.append(tuple(history))
|
||||
return QueryFrame(turn_type="smalltalk")
|
||||
|
||||
@@ -190,6 +368,26 @@ def test_history_is_isolated_per_conversation_id():
|
||||
assert received_history[1] == ()
|
||||
|
||||
|
||||
# --- F-10: `conversation_id` presence/absence must not change the safety
|
||||
# decision on a fresh (first) turn — only whether the turn is remembered
|
||||
# afterward -------------------------------------------------------------
|
||||
|
||||
|
||||
def test_conversation_id_presence_or_absence_reaches_the_same_decision():
|
||||
"""A single-turn call (no `conversation_id`) and the first turn of a
|
||||
fresh multi-turn conversation must resolve identically — both see empty
|
||||
history, so nothing about `conversation_id` itself may become a second,
|
||||
undocumented safety signal."""
|
||||
agent = _agent(QueryFrame(
|
||||
turn_type="drug_attribute", unknown_drugs=("aspirinol",),
|
||||
))
|
||||
without_id = agent.handle("liều aspirinol")
|
||||
with_id = agent.handle("liều aspirinol", conversation_id="fresh-conv")
|
||||
|
||||
assert without_id.decision == with_id.decision == "abstain"
|
||||
assert without_id.reason == with_id.reason == "drug_not_in_formulary"
|
||||
|
||||
|
||||
def test_autocomplete_delegates_to_the_configured_source():
|
||||
class _Source:
|
||||
def complete(self, prefix, k):
|
||||
@@ -210,3 +408,291 @@ def test_autocomplete_with_no_source_configured_returns_empty():
|
||||
_FixedRetrieval({}), answers,
|
||||
)
|
||||
assert agent.complete("met") == []
|
||||
|
||||
|
||||
# --- F-08: a per-turn budget actually stops real provider calls once spent,
|
||||
# it isn't just bookkeeping ---------------------------------------------
|
||||
|
||||
|
||||
def test_a_budget_exhausted_during_understand_blocks_every_later_call():
|
||||
"""`max_llm_calls_per_turn=1` means the (simulated) `understand()` call
|
||||
spends the entire turn's budget — sufficiency and generate must never
|
||||
reach the generator at all, not just receive an error from it. Proves
|
||||
the budget is threaded end to end through `RagAgent`, not only present
|
||||
at the one call site each unit test exercises in isolation."""
|
||||
class _BudgetSpendingUnderstander:
|
||||
"""Stands in for the real `LlmQueryUnderstander`, which calls
|
||||
`budget.require()` once before its own LLM call — simulated here so
|
||||
this test doesn't need a live-shaped LLM fake for the understand
|
||||
step, only for the reply's calls to fail budget."""
|
||||
|
||||
def understand(self, turn, history=(), budget=None, prior_frame=None):
|
||||
if budget is not None:
|
||||
budget.require()
|
||||
return QueryFrame(
|
||||
turn_type="drug_attribute", drugs=("metformin",),
|
||||
attribute="chi_dinh",
|
||||
)
|
||||
|
||||
result = RetrievalResult(
|
||||
EvidenceDecision.ANSWERABLE, "grounded_evidence_available",
|
||||
(_evidence("đoạn 0"), _evidence("đoạn 1")),
|
||||
)
|
||||
|
||||
generator_calls: list[dict] = []
|
||||
|
||||
class _CountingGenerator:
|
||||
def generate(self, system, user, schema):
|
||||
generator_calls.append(schema)
|
||||
return '{"answer": "unused", "evidence_sufficient": true, "clarifying_question": null}'
|
||||
|
||||
metrics = InMemoryMetrics()
|
||||
answers = GroundedAnswerService(
|
||||
routing=None, generator=_CountingGenerator(), metrics=metrics
|
||||
)
|
||||
agent = RagAgent(
|
||||
_BudgetSpendingUnderstander(), _FixedRetrieval({"metformin": result}), answers,
|
||||
max_llm_calls_per_turn=1,
|
||||
)
|
||||
|
||||
reply = agent.handle("liều metformin")
|
||||
|
||||
# The real point of F-08: no more actual provider calls happen once the
|
||||
# budget is spent — not "the generator returned an error", the generator
|
||||
# is never invoked at all.
|
||||
assert generator_calls == []
|
||||
assert reply.decision == "abstain"
|
||||
assert reply.answer is None
|
||||
assert metrics.total(GENERATION_REJECTED, reason="request_budget_exhausted") >= 1
|
||||
|
||||
|
||||
def test_a_generous_budget_does_not_change_normal_behaviour():
|
||||
"""Control for the test above: with the default (generous) budget, the
|
||||
same setup answers normally — proves the previous test's tiny budget is
|
||||
what caused the block, not some other change to the fixtures."""
|
||||
class _BudgetSpendingUnderstander:
|
||||
def understand(self, turn, history=(), budget=None, prior_frame=None):
|
||||
if budget is not None:
|
||||
budget.require()
|
||||
return QueryFrame(
|
||||
turn_type="drug_attribute", drugs=("metformin",),
|
||||
attribute="chi_dinh",
|
||||
)
|
||||
|
||||
result = RetrievalResult(
|
||||
EvidenceDecision.ANSWERABLE, "grounded_evidence_available",
|
||||
(_evidence("Liều 500 mg mỗi ngày."), _evidence("đoạn 1")),
|
||||
)
|
||||
|
||||
class _Generator:
|
||||
def generate(self, system, user, schema):
|
||||
if "sufficient" in schema.get("properties", {}):
|
||||
return '{"sufficient": true, "clarifying_question": null, "quick_replies": []}'
|
||||
if "entailed" in schema.get("properties", {}):
|
||||
return '{"entailed": true, "unsupported": []}'
|
||||
return '{"answer": "Liều 500 mg [1].", "evidence_sufficient": true, "clarifying_question": null}'
|
||||
|
||||
answers = GroundedAnswerService(routing=None, generator=_Generator())
|
||||
agent = RagAgent(
|
||||
_BudgetSpendingUnderstander(), _FixedRetrieval({"metformin": result}), answers,
|
||||
)
|
||||
|
||||
reply = agent.handle("liều metformin")
|
||||
|
||||
assert reply.decision == "answerable"
|
||||
assert reply.answer == "Liều 500 mg [1]."
|
||||
|
||||
|
||||
# --- durable conversation history (ADR 0008's named gap): an optional
|
||||
# `ConversationStore` replaces the in-process dict when configured --------
|
||||
|
||||
|
||||
class _FakeStore:
|
||||
def __init__(self) -> None:
|
||||
self.lines: dict[str, list[str]] = {}
|
||||
self.fail_reads = False
|
||||
self.fail_writes = False
|
||||
|
||||
def recent(self, conversation_id, limit):
|
||||
if self.fail_reads:
|
||||
raise ConnectionError("store unreachable")
|
||||
return self.lines.get(conversation_id, [])[-limit:]
|
||||
|
||||
def append(self, conversation_id, line):
|
||||
if self.fail_writes:
|
||||
raise ConnectionError("store unreachable")
|
||||
self.lines.setdefault(conversation_id, []).append(line)
|
||||
|
||||
|
||||
def test_history_round_trips_through_a_configured_store():
|
||||
received_history: list[tuple[str, ...]] = []
|
||||
|
||||
class _RecordingUnderstander:
|
||||
def understand(self, turn, history=(), budget=None, prior_frame=None):
|
||||
received_history.append(tuple(history))
|
||||
return QueryFrame(turn_type="smalltalk")
|
||||
|
||||
store = _FakeStore()
|
||||
answers = GroundedAnswerService(routing=None)
|
||||
agent = RagAgent(_RecordingUnderstander(), _FixedRetrieval({}), answers, store=store)
|
||||
|
||||
agent.handle("chào bạn", conversation_id="c1")
|
||||
agent.handle("còn liều thì sao?", conversation_id="c1")
|
||||
|
||||
assert received_history[0] == ()
|
||||
assert any("chào bạn" in line for line in received_history[1])
|
||||
assert store.lines["c1"] # actually persisted, not just read back in-process
|
||||
|
||||
|
||||
def test_store_read_failure_fails_open_to_fresh_history_not_a_crash():
|
||||
store = _FakeStore()
|
||||
store.lines["c1"] = ["Người dùng: câu cũ", "Trợ lý: trả lời cũ"]
|
||||
store.fail_reads = True
|
||||
|
||||
answers = GroundedAnswerService(routing=None)
|
||||
agent = RagAgent(
|
||||
_FixedUnderstander(QueryFrame(turn_type="smalltalk")),
|
||||
_FixedRetrieval({}), answers, store=store,
|
||||
)
|
||||
|
||||
# Must not raise -- degrades to no history for this turn.
|
||||
reply = agent.handle("chào bạn", conversation_id="c1")
|
||||
assert reply.decision == "answerable"
|
||||
|
||||
|
||||
def test_store_write_failure_fails_open_the_response_still_returns():
|
||||
store = _FakeStore()
|
||||
store.fail_writes = True
|
||||
|
||||
answers = GroundedAnswerService(routing=None)
|
||||
agent = RagAgent(
|
||||
_FixedUnderstander(QueryFrame(turn_type="smalltalk")),
|
||||
_FixedRetrieval({}), answers, store=store,
|
||||
)
|
||||
|
||||
# Must not raise, even though persisting this turn silently fails.
|
||||
reply = agent.handle("chào bạn", conversation_id="c1")
|
||||
assert reply.decision == "answerable"
|
||||
assert reply.answer is not None
|
||||
|
||||
|
||||
# --- F-11: the clarify-loop circuit breaker. Found live 2026-08-07
|
||||
# (50-question hand-typed browser audit): the understanding LLM can keep
|
||||
# deciding needs_clarify=true forever with no natural exit — reproduced 3
|
||||
# times independently, one case never converged after 5 real answered turns.
|
||||
# `understanding.py`'s prior-frame merge fixes most of the underlying cause,
|
||||
# but this is the code-level bound that guarantees a user is never stuck. --
|
||||
|
||||
|
||||
class _AlwaysClarifyUnderstander:
|
||||
"""Simulates a model stuck re-asking regardless of what the user
|
||||
answers — exactly the observed live failure, isolated from any real
|
||||
LLM's actual (variable) behaviour so this test is deterministic."""
|
||||
|
||||
def understand(self, turn, history=(), budget=None, prior_frame=None):
|
||||
return QueryFrame(
|
||||
turn_type="dosing_calc", drugs=("paracetamol_acetaminophen",),
|
||||
needs_clarify=True, clarify_reason="Bé nặng bao nhiêu kg?",
|
||||
)
|
||||
|
||||
|
||||
def test_clarify_loop_is_hard_stopped_after_max_consecutive_turns():
|
||||
answers = GroundedAnswerService(routing=None)
|
||||
agent = RagAgent(_AlwaysClarifyUnderstander(), _FixedRetrieval({}), answers)
|
||||
|
||||
replies = [
|
||||
agent.handle(f"turn {i}", conversation_id="stuck")
|
||||
for i in range(MAX_CONSECUTIVE_CLARIFY)
|
||||
]
|
||||
|
||||
# Every turn up to the last stays a genuine clarify...
|
||||
for reply in replies[:-1]:
|
||||
assert reply.decision == "clarify"
|
||||
# ...the Nth forces a hard stop instead of asking again.
|
||||
assert replies[-1].decision == "abstain"
|
||||
assert replies[-1].reason == "clarify_loop_exhausted"
|
||||
assert replies[-1].answer is not None
|
||||
|
||||
|
||||
def test_clarify_streak_resets_after_the_hard_stop_so_a_new_attempt_can_proceed():
|
||||
"""Confirms the breaker is a bounded pause, not a permanent lockout of
|
||||
the conversation id — the very next turn gets a fresh streak."""
|
||||
answers = GroundedAnswerService(routing=None)
|
||||
agent = RagAgent(_AlwaysClarifyUnderstander(), _FixedRetrieval({}), answers)
|
||||
|
||||
for i in range(MAX_CONSECUTIVE_CLARIFY):
|
||||
agent.handle(f"turn {i}", conversation_id="stuck")
|
||||
reply = agent.handle("one more try", conversation_id="stuck")
|
||||
|
||||
assert reply.decision == "clarify"
|
||||
|
||||
|
||||
def test_a_resolved_turn_resets_the_clarify_streak():
|
||||
"""An answerable turn in between two clarify runs must not let their
|
||||
streaks combine — only genuinely consecutive clarifies count."""
|
||||
result = RetrievalResult(
|
||||
EvidenceDecision.ANSWERABLE, "grounded_evidence_available",
|
||||
(_evidence("Liều 500 mg."),),
|
||||
)
|
||||
clarify_frame = QueryFrame(
|
||||
turn_type="dosing_calc", drugs=("paracetamol_acetaminophen",),
|
||||
needs_clarify=True, clarify_reason="Bé nặng bao nhiêu kg?",
|
||||
)
|
||||
resolved_frame = QueryFrame(
|
||||
turn_type="drug_attribute", drugs=("metformin",),
|
||||
attribute="lieu_luong_va_cach_dung",
|
||||
)
|
||||
# (MAX-1) clarifies, one resolved turn, then (MAX-1) clarifies again —
|
||||
# scripted explicitly so the test asserts the streak reset, not an
|
||||
# incidental side effect of some other call-counting scheme.
|
||||
script = (
|
||||
[clarify_frame] * (MAX_CONSECUTIVE_CLARIFY - 1)
|
||||
+ [resolved_frame]
|
||||
+ [clarify_frame] * (MAX_CONSECUTIVE_CLARIFY - 1)
|
||||
)
|
||||
|
||||
class _ScriptedUnderstander:
|
||||
def __init__(self, frames):
|
||||
self._frames = iter(frames)
|
||||
|
||||
def understand(self, turn, history=(), budget=None, prior_frame=None):
|
||||
return next(self._frames)
|
||||
|
||||
answers = GroundedAnswerService(routing=None)
|
||||
agent = RagAgent(
|
||||
_ScriptedUnderstander(script), _FixedRetrieval({"metformin": result}), answers,
|
||||
)
|
||||
|
||||
for i in range(MAX_CONSECUTIVE_CLARIFY - 1):
|
||||
reply = agent.handle(f"clarify {i}", conversation_id="c1")
|
||||
assert reply.decision == "clarify"
|
||||
resolved = agent.handle("answerable turn", conversation_id="c1")
|
||||
assert resolved.decision == "answerable"
|
||||
|
||||
# Streak was reset by the resolved turn -- this run of clarifies must
|
||||
# not be treated as a continuation of the earlier (pre-reset) run.
|
||||
for i in range(MAX_CONSECUTIVE_CLARIFY - 1):
|
||||
reply = agent.handle(f"clarify again {i}", conversation_id="c1")
|
||||
assert reply.decision == "clarify"
|
||||
|
||||
|
||||
def test_prior_frame_is_threaded_from_the_last_turn_to_the_understander():
|
||||
received: list[QueryFrame | None] = []
|
||||
|
||||
class _RecordingUnderstander:
|
||||
def understand(self, turn, history=(), budget=None, prior_frame=None):
|
||||
received.append(prior_frame)
|
||||
return QueryFrame(
|
||||
turn_type="dosing_calc", drugs=("paracetamol_acetaminophen",),
|
||||
needs_clarify=True, clarify_reason="Bé nặng bao nhiêu kg?",
|
||||
)
|
||||
|
||||
answers = GroundedAnswerService(routing=None)
|
||||
agent = RagAgent(_RecordingUnderstander(), _FixedRetrieval({}), answers)
|
||||
|
||||
agent.handle("liều paracetamol cho bé", conversation_id="c1")
|
||||
agent.handle("20kg", conversation_id="c1")
|
||||
|
||||
assert received[0] is None
|
||||
assert received[1] is not None
|
||||
assert received[1].clarify_reason == "Bé nặng bao nhiêu kg?"
|
||||
|
||||
Reference in New Issue
Block a user