281 lines
11 KiB
Python
281 lines
11 KiB
Python
"""Two answer-UX fixes, pinned:
|
|
|
|
- citations shown = only the sources the answer cited, not every retrieved chunk;
|
|
- a bare drug name is introduced, not restated section-by-section.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
|
|
from rag.answer import GroundedAnswerService
|
|
from rag.models import (
|
|
Evidence,
|
|
EvidenceDecision,
|
|
QueryIntent,
|
|
RetrievalResult,
|
|
SourceRef,
|
|
SubjectScope,
|
|
)
|
|
from rag.ports import AnswerGenerationUnavailable
|
|
from rag.prompt import build_request
|
|
|
|
|
|
def _evidence(i: int, page: int) -> Evidence:
|
|
return Evidence(
|
|
evidence_id=f"drug::sec::{i}",
|
|
matched_doc_id=f"drug::sec::{i}",
|
|
kind="prose",
|
|
text=f"đoạn bằng chứng {i}",
|
|
score=1.0,
|
|
source_refs=(SourceRef(physical_page=page, precision="exact", printed_page=page),),
|
|
hydrated_from_parent=False,
|
|
requires_visual_check=False,
|
|
)
|
|
|
|
|
|
class _Routing:
|
|
def __init__(self, result: RetrievalResult) -> None:
|
|
self._result = result
|
|
|
|
def retrieve(self, query, subject_scope, intent): # noqa: ARG002
|
|
return self._result
|
|
|
|
|
|
class _Generator:
|
|
def __init__(
|
|
self, payload: dict, entailment_payload: dict | None = None,
|
|
sufficiency_payload: dict | None = None,
|
|
) -> None:
|
|
self._payload = payload
|
|
self._entailment_payload = entailment_payload or {
|
|
"entailed": True,
|
|
"unsupported": [],
|
|
}
|
|
self._sufficiency_payload = sufficiency_payload
|
|
|
|
def generate(self, system: str, user: str, schema: dict) -> str: # noqa: ARG002
|
|
# `_generate` also runs a post-generation entailment check, and
|
|
# `_check_sufficiency` runs its own separate call before that — tell
|
|
# the three request shapes apart by schema so a test asserting on one
|
|
# doesn't have to also shape a payload for the others.
|
|
props = schema.get("properties", {})
|
|
if "entailed" in props:
|
|
payload = self._entailment_payload
|
|
elif "sufficient" in props and self._sufficiency_payload is not None:
|
|
payload = self._sufficiency_payload
|
|
else:
|
|
payload = self._payload
|
|
if isinstance(payload, BaseException):
|
|
raise payload
|
|
return json.dumps(payload, ensure_ascii=False)
|
|
|
|
|
|
def _answerable(*evidence: Evidence, is_overview: bool = False) -> RetrievalResult:
|
|
return RetrievalResult(
|
|
EvidenceDecision.ANSWERABLE,
|
|
"grounded_evidence_available",
|
|
tuple(evidence),
|
|
resolved_drug_id="drug",
|
|
is_drug_overview=is_overview,
|
|
)
|
|
|
|
|
|
def test_only_cited_sources_are_returned():
|
|
result = _answerable(_evidence(0, 100), _evidence(1, 200), _evidence(2, 300))
|
|
service = GroundedAnswerService(
|
|
_Routing(result),
|
|
_Generator({"claims": [{"text": "Chỉ dùng đoạn hai", "citations": [2]}],
|
|
"evidence_sufficient": True}),
|
|
)
|
|
|
|
grounded = service.answer("q", SubjectScope.HUMAN, QueryIntent.FACT_LOOKUP)
|
|
|
|
assert grounded.generated is True
|
|
assert len(grounded.citations) == 1
|
|
assert grounded.citations[0].printed_page_start == 200
|
|
|
|
|
|
def test_answer_citing_nothing_is_rejected_not_dressed_up_with_borrowed_citations():
|
|
result = _answerable(_evidence(0, 100), _evidence(1, 200))
|
|
service = GroundedAnswerService(
|
|
_Routing(result),
|
|
# no [n] marker at all: grounding.verify rejects this outright (an
|
|
# uncited claim, per F-01). A generator is configured, so the
|
|
# rejection abstains — it must not attach every retrieved citation
|
|
# to dress an uncited generation up as sourced (the old behavior),
|
|
# and it must not silently degrade to a raw extractive quote either
|
|
# (owner correction, 2026-08-06: no fallback to the retired
|
|
# offline-extractive shape when a real generator is configured).
|
|
_Generator({"claims": [{"text": "Không có trích dẫn.", "citations": []}],
|
|
"evidence_sufficient": True}),
|
|
)
|
|
|
|
grounded = service.answer("q", SubjectScope.HUMAN, QueryIntent.FACT_LOOKUP)
|
|
|
|
assert grounded.generated is False
|
|
assert grounded.answer is None
|
|
assert grounded.citations == ()
|
|
assert grounded.result.decision == EvidenceDecision.ABSTAIN
|
|
|
|
|
|
def test_underspecified_dose_asks_instead_of_dumping():
|
|
"""The reasoning step: a dose question spanning bands with no age/weight is
|
|
turned into a clarification, not the whole section."""
|
|
result = _answerable(_evidence(0, 100), _evidence(1, 200))
|
|
gen = _Generator(
|
|
{"sufficient": False,
|
|
"clarifying_question": "Bé mấy tuổi, cân nặng bao nhiêu kg?"}
|
|
)
|
|
service = GroundedAnswerService(_Routing(result), gen)
|
|
|
|
g = service.answer("paracetamol cho trẻ em", SubjectScope.HUMAN, QueryIntent.FACT_LOOKUP)
|
|
|
|
assert g.clarification is not None
|
|
assert "tuổi" in g.clarification
|
|
assert g.answer == g.clarification
|
|
assert g.generated is False
|
|
assert g.quick_replies == ()
|
|
|
|
|
|
def test_underspecified_dose_carries_quick_replies_when_the_model_offers_them():
|
|
result = _answerable(_evidence(0, 100), _evidence(1, 200))
|
|
gen = _Generator(
|
|
{"sufficient": False,
|
|
"clarifying_question": "Người lớn hay trẻ em?",
|
|
"quick_replies": ["Người lớn", "Trẻ em"]}
|
|
)
|
|
service = GroundedAnswerService(_Routing(result), gen)
|
|
|
|
g = service.answer("paracetamol", SubjectScope.HUMAN, QueryIntent.FACT_LOOKUP)
|
|
|
|
assert g.quick_replies == ("Người lớn", "Trẻ em")
|
|
|
|
|
|
def test_sufficiency_check_outage_fails_open_to_generation_not_abstain():
|
|
"""F-10: `_check_sufficiency` documents (and this pins) a deliberate
|
|
fail-OPEN on provider outage — unlike every other failure mode in this
|
|
service, a sufficiency-check outage does not abstain, it just skips the
|
|
clarify heuristic and lets grounding/entailment (tested elsewhere) be
|
|
the real safety net on whatever gets generated next."""
|
|
result = _answerable(_evidence(0, 100), _evidence(1, 200))
|
|
gen = _Generator(
|
|
{"claims": [{"text": "Đoạn bằng chứng 0", "citations": [1]}],
|
|
"evidence_sufficient": True, "clarifying_question": None},
|
|
sufficiency_payload=AnswerGenerationUnavailable("Bedrock unreachable"),
|
|
)
|
|
service = GroundedAnswerService(_Routing(result), gen)
|
|
|
|
g = service.answer("liều người lớn", SubjectScope.HUMAN, QueryIntent.FACT_LOOKUP)
|
|
|
|
assert g.clarification is None
|
|
assert g.result.decision == EvidenceDecision.ANSWERABLE
|
|
assert g.generated is True
|
|
|
|
|
|
def test_sufficient_query_is_not_turned_into_a_clarification():
|
|
result = _answerable(_evidence(0, 100), _evidence(1, 200))
|
|
gen = _Generator({"sufficient": True, "clarifying_question": None})
|
|
service = GroundedAnswerService(_Routing(result), gen)
|
|
|
|
g = service.answer("liều người lớn", SubjectScope.HUMAN, QueryIntent.FACT_LOOKUP)
|
|
|
|
# Sufficiency passes; generation then runs (its payload lacks answer keys,
|
|
# so it fails closed) — the point is no clarification fired.
|
|
assert g.clarification is None
|
|
|
|
|
|
def test_bare_name_builds_an_intro_prompt():
|
|
intro = build_request("PARACETAMOL", ("đoạn A", "đoạn B"), intro=True)
|
|
assert "GIỚI THIỆU" in intro.user
|
|
assert "CÂU HỎI:" not in intro.user
|
|
|
|
normal = build_request("Liều?", ("đoạn A",), intro=False)
|
|
assert "CÂU HỎI:" in normal.user
|
|
assert "GIỚI THIỆU" not in normal.user
|
|
|
|
|
|
def test_list_mode_prompt_instructs_enumerating_every_drug_not_ranking():
|
|
"""2026-08-07 finding: without an explicit instruction, the model picked
|
|
one drug out of 8 real symptom_to_drug matches and silently dropped the
|
|
rest — verified live. `list_mode` closes that."""
|
|
request = build_request("thuốc gì trị sốt", ("chỉ định A", "chỉ định B"), list_mode=True)
|
|
assert "LIỆT KÊ TẤT CẢ" in request.user
|
|
assert "không xếp hạng" in request.user or "KHÔNG" in request.user
|
|
assert "CÂU HỎI:" in request.user
|
|
|
|
|
|
def test_list_mode_skips_the_sufficiency_clarify():
|
|
"""If the sufficiency call were NOT actually skipped, it would read
|
|
`sufficiency_payload` (`sufficient=False`) and clarify. `list_mode=True`
|
|
must never call it at all, so only the real answer payload is ever read."""
|
|
result = _answerable(_evidence(0, 100), _evidence(1, 200))
|
|
gen = _Generator(
|
|
{"claims": [
|
|
{"text": "Đoạn bằng chứng 0", "citations": [1]},
|
|
{"text": "Đoạn bằng chứng 1", "citations": [2]},
|
|
], "evidence_sufficient": True, "clarifying_question": None},
|
|
sufficiency_payload={
|
|
"sufficient": False, "clarifying_question": "Loại nào?", "quick_replies": [],
|
|
},
|
|
)
|
|
service = GroundedAnswerService(_Routing(result), gen)
|
|
|
|
g = service.answer_from_result("thuốc gì trị sốt", result, list_mode=True)
|
|
|
|
assert g.clarification is None
|
|
assert g.answer is not None
|
|
assert g.generated is True
|
|
|
|
|
|
def test_list_mode_rejects_a_generated_drug_outside_candidate_set():
|
|
evidence = Evidence(
|
|
evidence_id="a__chi_dinh__0",
|
|
matched_doc_id="a__chi_dinh__0",
|
|
kind="prose",
|
|
text="Thuốc A được chỉ định điều trị bệnh X.",
|
|
score=1.0,
|
|
source_refs=(SourceRef(physical_page=100, precision="exact", printed_page=101),),
|
|
hydrated_from_parent=False,
|
|
requires_visual_check=False,
|
|
drug_id="a",
|
|
drug_name="A",
|
|
section_key="chi_dinh",
|
|
)
|
|
result = _answerable(evidence)
|
|
gen = _Generator({
|
|
"claims": [{"text": "Thuốc D điều trị bệnh X.", "citations": [1], "drug_id": "d"}],
|
|
"evidence_sufficient": True,
|
|
"clarifying_question": None,
|
|
"quick_replies": [],
|
|
})
|
|
service = GroundedAnswerService(_Routing(result), gen)
|
|
|
|
grounded = service.answer_from_result(
|
|
"Bệnh X dùng thuốc gì?",
|
|
result,
|
|
list_mode=True,
|
|
candidate_drug_ids=("a",),
|
|
)
|
|
|
|
assert grounded.answer is None
|
|
assert grounded.result.decision == EvidenceDecision.ABSTAIN
|
|
assert grounded.result.reason == "unsupported_drug"
|
|
|
|
|
|
def test_without_list_mode_the_same_evidence_does_ask_for_clarification():
|
|
"""Control for the test above: the same sufficiency payload, without
|
|
`list_mode`, must actually clarify — proving the previous test's "not
|
|
skipped" branch is reachable and would have failed loudly."""
|
|
result = _answerable(_evidence(0, 100), _evidence(1, 200))
|
|
gen = _Generator(
|
|
{"answer": "unused", "evidence_sufficient": True, "clarifying_question": None},
|
|
sufficiency_payload={
|
|
"sufficient": False, "clarifying_question": "Loại nào?", "quick_replies": [],
|
|
},
|
|
)
|
|
service = GroundedAnswerService(_Routing(result), gen)
|
|
|
|
g = service.answer_from_result("thuốc gì trị sốt", result, list_mode=False)
|
|
|
|
assert g.clarification == "Loại nào?"
|