464 lines
18 KiB
Python
464 lines
18 KiB
Python
"""The answer layer may rephrase evidence; it may not add to it.
|
|
|
|
Every test here is a fabrication the generator could plausibly produce, and
|
|
the assertion is that the clinician never sees it. The dose figures are taken
|
|
from the real METFORMIN and PARACETAMOL sections in `duocthu_v1`.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
|
|
import pytest
|
|
|
|
from rag import grounding
|
|
from rag.answer import GroundedAnswerService
|
|
from rag.metrics import GENERATION_REJECTED, GENERATION_SERVED, InMemoryMetrics
|
|
from rag.models import (
|
|
Evidence,
|
|
EvidenceDecision,
|
|
QueryIntent,
|
|
RetrievalResult,
|
|
SourceRef,
|
|
SubjectScope,
|
|
)
|
|
from rag.ports import AnswerGenerationUnavailable
|
|
from rag.prompt import build_request
|
|
|
|
SOURCE = SourceRef(
|
|
physical_page=812,
|
|
precision="region",
|
|
printed_page_range=(714, 714),
|
|
)
|
|
|
|
EVIDENCE_TEXT = (
|
|
"Người lớn: uống 500 mg metformin hydroclorid, 2 lần mỗi ngày. "
|
|
"Liều tối đa 2 g mỗi ngày, chia làm nhiều lần."
|
|
)
|
|
|
|
|
|
def _result(text: str = EVIDENCE_TEXT) -> RetrievalResult:
|
|
return RetrievalResult(
|
|
EvidenceDecision.ANSWERABLE,
|
|
"grounded_evidence_available",
|
|
(
|
|
Evidence(
|
|
evidence_id="metformin::lieu::0",
|
|
matched_doc_id="metformin::lieu::0",
|
|
kind="prose",
|
|
text=text,
|
|
score=1.0,
|
|
source_refs=(SOURCE,),
|
|
hydrated_from_parent=False,
|
|
requires_visual_check=False,
|
|
),
|
|
),
|
|
resolved_drug_id="metformin",
|
|
)
|
|
|
|
|
|
class _FixedRouting:
|
|
def __init__(self, result: RetrievalResult) -> None:
|
|
self._result = result
|
|
|
|
def retrieve(self, query, subject_scope, intent):
|
|
return self._result
|
|
|
|
|
|
class _Generator:
|
|
"""Returns whatever payload the test wants the model to have produced.
|
|
|
|
`_generate` can make a main answer call (a lone
|
|
`evidence_sufficient: false` retries once) and one fail-closed entailment
|
|
call. The legacy direct-answer path may also make a sufficiency call when
|
|
several evidence blocks need disambiguation; the structured agent path
|
|
skips that duplicate judgment. They're told apart by schema, so a test
|
|
that only cares about one call doesn't have to fake the others; `payload`
|
|
and `entailment_payload` each take either a fixed value or a list for a
|
|
different answer on each successive call to that schema.
|
|
"""
|
|
|
|
def __init__(self, payload, entailment_payload=None) -> None:
|
|
payloads = payload
|
|
self._payloads = list(payloads) if isinstance(payloads, list) else [payloads]
|
|
self._call = 0
|
|
default = {"entailed": True, "unsupported": []}
|
|
e_payloads = entailment_payload if entailment_payload is not None else default
|
|
self._entailment_payloads = (
|
|
list(e_payloads) if isinstance(e_payloads, list) else [e_payloads]
|
|
)
|
|
self._entailment_call = 0
|
|
|
|
def generate(self, system: str, user: str, schema: dict) -> str:
|
|
if "entailed" in schema.get("properties", {}):
|
|
index = min(self._entailment_call, len(self._entailment_payloads) - 1)
|
|
payload = self._entailment_payloads[index]
|
|
self._entailment_call += 1
|
|
else:
|
|
index = min(self._call, len(self._payloads) - 1)
|
|
payload = self._payloads[index]
|
|
self._call += 1
|
|
if isinstance(payload, BaseException):
|
|
raise payload
|
|
if isinstance(payload, str):
|
|
return payload
|
|
return json.dumps(payload, ensure_ascii=False)
|
|
|
|
|
|
def _answer(payload, result: RetrievalResult | None = None, entailment_payload=None):
|
|
metrics = InMemoryMetrics()
|
|
service = GroundedAnswerService(
|
|
_FixedRouting(result or _result()),
|
|
_Generator(payload, entailment_payload),
|
|
metrics,
|
|
)
|
|
grounded = service.answer("Liều Metformin?", SubjectScope.HUMAN, QueryIntent.FACT_LOOKUP)
|
|
return grounded, metrics
|
|
|
|
|
|
# --- the guardrail's whole reason to exist ------------------------------------
|
|
|
|
|
|
def test_invented_dose_is_refused_and_never_reaches_the_answer():
|
|
grounded, metrics = _answer(
|
|
{"claims": [{"text": "Người lớn uống 850 mg, 2 lần mỗi ngày", "citations": [1]}],
|
|
"evidence_sufficient": True}
|
|
)
|
|
|
|
assert grounded.generated is False
|
|
# A generator is configured, so a rejected generation abstains — it does
|
|
# NOT silently degrade to a raw source dump (owner correction, 2026-08-06:
|
|
# this is a real LLM chatbot, not the retired offline-extractive build).
|
|
assert grounded.answer is None
|
|
assert grounded.result.decision == EvidenceDecision.ABSTAIN
|
|
# The specific check that rejected it, not a generic catch-all — found
|
|
# live 2026-08-07: every rejection reason used to collapse into
|
|
# "generation_unavailable" by the time it reached the API response,
|
|
# making a real provider outage indistinguishable from ordinary
|
|
# entailment noise without reading server metrics by hand.
|
|
assert grounded.result.reason == "ungrounded_number"
|
|
assert metrics.total(GENERATION_REJECTED, reason="ungrounded_number") == 1
|
|
|
|
|
|
def test_a_rounded_figure_counts_as_invented():
|
|
"""`2 g` is in the source; `2000 mg` is a conversion, and conversions are
|
|
where unit errors live. The prompt forbids it and the check enforces it."""
|
|
grounded, metrics = _answer(
|
|
{"claims": [{"text": "Liều tối đa 2000 mg mỗi ngày", "citations": [1]}],
|
|
"evidence_sufficient": True}
|
|
)
|
|
|
|
assert grounded.generated is False
|
|
assert metrics.total(GENERATION_REJECTED, reason="ungrounded_number") == 1
|
|
|
|
|
|
def test_citation_pointing_at_nothing_is_refused():
|
|
grounded, metrics = _answer(
|
|
{"claims": [{"text": "Người lớn uống 500 mg", "citations": [3]}],
|
|
"evidence_sufficient": True}
|
|
)
|
|
|
|
assert grounded.generated is False
|
|
# [3] is out of range with one evidence block, so "500" has no valid
|
|
# citation to bind to — grounding.verify now flags it as unsupported
|
|
# rather than letting it pass because 500 happens to exist somewhere in
|
|
# the (single) evidence block anyway. ungrounded_number takes priority
|
|
# over invalid_citation in GroundingReport.reason; both are present.
|
|
assert metrics.total(GENERATION_REJECTED, reason="ungrounded_number") == 1
|
|
|
|
|
|
def test_faithful_rewrite_is_served():
|
|
grounded, metrics = _answer(
|
|
{"claims": [{"text": "Người lớn: 500 mg, 2 lần/ngày; tối đa 2 g/ngày", "citations": [1]}],
|
|
"evidence_sufficient": True}
|
|
)
|
|
|
|
assert grounded.generated is True
|
|
assert grounded.answer == "Người lớn: 500 mg, 2 lần/ngày; tối đa 2 g/ngày"
|
|
assert grounded.blocks[0].claims[0].source_ids == ("metformin::lieu::0",)
|
|
assert metrics.total(GENERATION_SERVED) == 1
|
|
assert metrics.total(GENERATION_REJECTED) == 0
|
|
|
|
|
|
# --- the entailment pass: catches what number/citation checks structurally can't -----
|
|
|
|
|
|
def test_fabricated_nonnumeric_claim_with_a_valid_citation_is_rejected():
|
|
"""Reproduces `claim_bia` from the Codex 2026-08-06 review end to end:
|
|
right drug, syntactically valid citation, fabricated indication.
|
|
grounding.verify alone cannot see this (no number, citation in range) —
|
|
the entailment pass, told the model judged evidence 1 does not support
|
|
it, is what rejects the generation."""
|
|
grounded, metrics = _answer(
|
|
{"claims": [{"text": "Metformin chữa ung thư", "citations": [1]}], "evidence_sufficient": True},
|
|
entailment_payload={"entailed": False, "unsupported": [1]},
|
|
)
|
|
|
|
assert grounded.generated is False
|
|
assert grounded.answer is None
|
|
assert grounded.result.decision == EvidenceDecision.ABSTAIN
|
|
assert metrics.total(GENERATION_REJECTED, reason="unsupported_claim") == 1
|
|
|
|
|
|
def test_entailment_check_running_and_passing_still_serves_the_answer():
|
|
grounded, metrics = _answer(
|
|
{"claims": [{"text": "Metformin dùng điều trị đái tháo đường", "citations": [1]}],
|
|
"evidence_sufficient": True},
|
|
entailment_payload={"entailed": True, "unsupported": []},
|
|
)
|
|
|
|
assert grounded.generated is True
|
|
assert metrics.total(GENERATION_SERVED) == 1
|
|
|
|
|
|
def test_entailment_rejects_after_one_fail_closed_semantic_pass():
|
|
"""The verifier is one semantic pass after deterministic grounding.
|
|
|
|
Repeating an identical temperature-0 prompt against the same model is a
|
|
correlated retry, not an independent vote, and doubled the hot-path model
|
|
latency for every valid answer.
|
|
"""
|
|
grounded, metrics = _answer(
|
|
{"claims": [{"text": "Metformin chữa ung thư", "citations": [1]}],
|
|
"evidence_sufficient": True},
|
|
entailment_payload=[
|
|
{"entailed": False, "unsupported": [1]},
|
|
{"entailed": True, "unsupported": []},
|
|
],
|
|
)
|
|
|
|
assert grounded.generated is False
|
|
assert grounded.answer is None
|
|
assert metrics.total(GENERATION_REJECTED, reason="unsupported_claim") == 1
|
|
|
|
|
|
def test_supported_but_incomplete_answer_is_rejected_against_full_raw_evidence():
|
|
grounded, metrics = _answer(
|
|
{
|
|
"claims": [{"text": "Người lớn uống 500 mg", "citations": [1]}],
|
|
"evidence_sufficient": True,
|
|
},
|
|
entailment_payload={
|
|
"entailed": True,
|
|
"unsupported": [],
|
|
"complete": False,
|
|
"missing_evidence": [{
|
|
"description": "2 lần mỗi ngày và liều tối đa 2 g mỗi ngày",
|
|
"evidence_quote": EVIDENCE_TEXT,
|
|
}],
|
|
},
|
|
)
|
|
|
|
assert grounded.answer is None
|
|
assert grounded.result.reason == "incomplete_answer"
|
|
assert metrics.total(GENERATION_REJECTED, reason="incomplete_answer") == 1
|
|
|
|
|
|
def test_completeness_judge_cannot_claim_its_own_quoted_fact_is_missing():
|
|
grounded, _ = _answer(
|
|
{
|
|
"claims": [{
|
|
"text": "Chảy máu giữa vòng kinh (rất hay gặp trong 3 tháng đầu dùng thuốc theo đường tiêm).",
|
|
"citations": [1],
|
|
}],
|
|
"evidence_sufficient": True,
|
|
},
|
|
result=_result(
|
|
"Chảy máu giữa vòng kinh (rất hay gặp trong 3 tháng đầu dùng thuốc theo đường tiêm)."
|
|
),
|
|
entailment_payload={
|
|
"entailed": True,
|
|
"unsupported": [],
|
|
"complete": False,
|
|
"missing_evidence": [{
|
|
"description": "Không ghi nhận 'rất hay gặp trong 3 tháng đầu dùng thuốc theo đường tiêm'",
|
|
"evidence_quote": "rất hay gặp trong 3 tháng đầu dùng thuốc theo đường tiêm",
|
|
}],
|
|
},
|
|
)
|
|
|
|
assert grounded.generated is True
|
|
assert grounded.answer is not None
|
|
|
|
|
|
def test_completeness_objection_without_a_real_source_quote_is_ignored():
|
|
grounded, _ = _answer(
|
|
{
|
|
"claims": [{
|
|
"text": "Người lớn: uống 500 mg metformin hydroclorid, 2 lần mỗi ngày. Liều tối đa 2 g mỗi ngày, chia làm nhiều lần.",
|
|
"citations": [1],
|
|
}],
|
|
"evidence_sufficient": True,
|
|
},
|
|
entailment_payload={
|
|
"entailed": True,
|
|
"unsupported": [],
|
|
"complete": False,
|
|
"missing_evidence": [{
|
|
"description": "Không nêu điều kiện độ ẩm",
|
|
"evidence_quote": "độ ẩm",
|
|
}],
|
|
},
|
|
)
|
|
|
|
assert grounded.generated is True
|
|
assert grounded.answer is not None
|
|
|
|
|
|
def test_entailment_accepts_after_one_semantic_pass():
|
|
grounded, metrics = _answer(
|
|
{"claims": [{"text": "Metformin dùng điều trị đái tháo đường", "citations": [1]}],
|
|
"evidence_sufficient": True},
|
|
entailment_payload=[
|
|
{"entailed": True, "unsupported": []},
|
|
{"entailed": False, "unsupported": [1]}, # never consulted
|
|
],
|
|
)
|
|
|
|
assert grounded.generated is True
|
|
assert metrics.total(GENERATION_SERVED) == 1
|
|
|
|
|
|
def test_entailment_provider_outage_fails_closed_to_abstain():
|
|
grounded, metrics = _answer(
|
|
{"claims": [{"text": "Người lớn: 500 mg, 2 lần/ngày", "citations": [1]}],
|
|
"evidence_sufficient": True},
|
|
entailment_payload=AnswerGenerationUnavailable(),
|
|
)
|
|
|
|
assert grounded.generated is False
|
|
assert grounded.answer is None
|
|
assert grounded.result.decision == EvidenceDecision.ABSTAIN
|
|
assert metrics.total(GENERATION_REJECTED, reason="unsupported_claim") == 1
|
|
|
|
|
|
def test_entailment_check_is_skipped_when_there_are_no_claims():
|
|
"""No claims at all (2026-08-10: the structured-claims schema makes a
|
|
claim's `text` a required, non-empty field, so the old "answer is
|
|
nothing but a bare citation marker" scenario can no longer occur — the
|
|
analogous edge case is an empty `claims` list) has nothing for an
|
|
entailment pass to check against — `_verify_entailment` must not call
|
|
the provider at all. Proven by making that call raise: if the skip
|
|
didn't fire, this would reject rather than serve the answer."""
|
|
grounded, metrics = _answer(
|
|
{"claims": [], "evidence_sufficient": True},
|
|
entailment_payload=AnswerGenerationUnavailable(),
|
|
)
|
|
|
|
assert grounded.generated is True
|
|
assert metrics.total(GENERATION_REJECTED) == 0
|
|
|
|
|
|
def test_citations_survive_generation():
|
|
"""Provenance is the point; a prettier answer must not cost the folio."""
|
|
grounded, _ = _answer(
|
|
{"claims": [{"text": "Người lớn: 500 mg", "citations": [1]}],
|
|
"evidence_sufficient": True}
|
|
)
|
|
|
|
assert grounded.generated is True
|
|
assert len(grounded.citations) == 1
|
|
assert grounded.citations[0].printed_page_start == 714
|
|
|
|
|
|
# --- a configured generator that fails abstains, never a raw source dump -----
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"payload, reason",
|
|
[
|
|
(AnswerGenerationUnavailable("revoked"), "provider_unavailable"),
|
|
("not json at all", "malformed_output"),
|
|
({"claims": [{"text": "500 mg", "citations": [1]}]}, "malformed_output"),
|
|
({"claims": 500, "evidence_sufficient": True}, "malformed_output"),
|
|
({"claims": [], "evidence_sufficient": False}, "evidence_insufficient"),
|
|
],
|
|
)
|
|
def test_every_generation_failure_abstains_instead_of_a_raw_source_dump(payload, reason):
|
|
grounded, metrics = _answer(payload)
|
|
|
|
assert grounded.generated is False
|
|
assert grounded.answer is None
|
|
assert grounded.result.decision == EvidenceDecision.ABSTAIN
|
|
# The API/trace-visible reason must match the specific check that
|
|
# failed, not a generic "generation_unavailable" for every cause —
|
|
# otherwise a real outage and ordinary model noise are indistinguishable
|
|
# from the outside (the exact gap a live report 2026-08-07 named).
|
|
assert grounded.result.reason == reason
|
|
assert metrics.total(GENERATION_REJECTED, reason=reason) == 1
|
|
|
|
|
|
def test_evidence_insufficient_retries_once_and_recovers():
|
|
"""Found live 2026-08-07 via a 50-question adversarial sample: a real
|
|
section that plainly contains the answer (confirmed by re-asking the
|
|
identical question 3/3 times successfully right after) still drew an
|
|
`evidence_sufficient: false` self-judgment once — the same noisy-judge
|
|
pattern already known for entailment, just on a different field of the
|
|
same call. A lone insufficient verdict must not be final."""
|
|
grounded, metrics = _answer([
|
|
{"claims": [], "evidence_sufficient": False},
|
|
{"claims": [{"text": "Metformin dùng điều trị đái tháo đường", "citations": [1]}],
|
|
"evidence_sufficient": True},
|
|
])
|
|
|
|
assert grounded.generated is True
|
|
assert grounded.answer == "Metformin dùng điều trị đái tháo đường"
|
|
assert metrics.total(GENERATION_SERVED) == 1
|
|
assert metrics.total(GENERATION_REJECTED) == 0
|
|
|
|
|
|
def test_evidence_insufficient_twice_still_abstains():
|
|
grounded, metrics = _answer([
|
|
{"claims": [], "evidence_sufficient": False},
|
|
{"claims": [], "evidence_sufficient": False},
|
|
])
|
|
|
|
assert grounded.generated is False
|
|
assert grounded.answer is None
|
|
# Both attempts are the same noisy self-judgment on the same evidence —
|
|
# a real, reliable "insufficient" must still discard exactly once.
|
|
assert metrics.total(GENERATION_REJECTED, reason="evidence_insufficient") == 1
|
|
|
|
|
|
def test_no_generator_configured_still_answers():
|
|
service = GroundedAnswerService(_FixedRouting(_result()))
|
|
|
|
grounded = service.answer(
|
|
"Liều Metformin?", SubjectScope.HUMAN, QueryIntent.FACT_LOOKUP
|
|
)
|
|
|
|
assert grounded.generated is False
|
|
assert grounded.answer.startswith(EVIDENCE_TEXT)
|
|
|
|
|
|
# --- the comparison rule itself ----------------------------------------------
|
|
|
|
|
|
def test_decimal_separators_are_not_interchangeable():
|
|
"""`7,5` and `7.5` differ, and so do `7,5` and `75`. Normalising them
|
|
together is how a tenfold dose error scores as a match."""
|
|
source = ("Sơ sinh: 7,5 mg/kg cách 8 giờ/lần.",)
|
|
|
|
assert grounding.verify("7,5 mg/kg [1]", source).grounded is True
|
|
assert grounding.verify("7.5 mg/kg [1]", source).grounded is False
|
|
assert grounding.verify("75 mg/kg [1]", source).grounded is False
|
|
|
|
|
|
def test_citation_markers_are_not_read_as_quantities():
|
|
report = grounding.verify("Không dùng cho người suy thận [1].", ("Suy thận.",))
|
|
|
|
assert report.grounded is True
|
|
assert report.cited_indices == (1,)
|
|
|
|
|
|
def test_prompt_numbers_evidence_from_one():
|
|
request = build_request("Liều?", ("đoạn A", "đoạn B"))
|
|
|
|
assert "[1] đoạn A" in request.user
|
|
assert "[2] đoạn B" in request.user
|
|
assert "CHÉP NGUYÊN VĂN" in request.system
|
|
|
|
|
|
def test_prompt_refuses_to_build_without_evidence():
|
|
with pytest.raises(ValueError):
|
|
build_request("Liều?", ())
|