336 lines
12 KiB
Python
336 lines
12 KiB
Python
"""The answer layer may rephrase evidence; it may not add to it.
|
|
|
|
Every test here is a fabrication the generator could plausibly produce, and
|
|
the assertion is that the clinician never sees it. The dose figures are taken
|
|
from the real METFORMIN and PARACETAMOL sections in `duocthu_v1`.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
|
|
import pytest
|
|
|
|
from rag import grounding
|
|
from rag.answer import GroundedAnswerService
|
|
from rag.metrics import GENERATION_REJECTED, GENERATION_SERVED, InMemoryMetrics
|
|
from rag.models import (
|
|
Evidence,
|
|
EvidenceDecision,
|
|
QueryIntent,
|
|
RetrievalResult,
|
|
SourceRef,
|
|
SubjectScope,
|
|
)
|
|
from rag.ports import AnswerGenerationUnavailable
|
|
from rag.prompt import build_request
|
|
|
|
SOURCE = SourceRef(
|
|
physical_page=812,
|
|
precision="region",
|
|
printed_page_range=(714, 714),
|
|
)
|
|
|
|
EVIDENCE_TEXT = (
|
|
"Người lớn: uống 500 mg metformin hydroclorid, 2 lần mỗi ngày. "
|
|
"Liều tối đa 2 g mỗi ngày, chia làm nhiều lần."
|
|
)
|
|
|
|
|
|
def _result(text: str = EVIDENCE_TEXT) -> RetrievalResult:
|
|
return RetrievalResult(
|
|
EvidenceDecision.ANSWERABLE,
|
|
"grounded_evidence_available",
|
|
(
|
|
Evidence(
|
|
evidence_id="metformin::lieu::0",
|
|
matched_doc_id="metformin::lieu::0",
|
|
kind="prose",
|
|
text=text,
|
|
score=1.0,
|
|
source_refs=(SOURCE,),
|
|
hydrated_from_parent=False,
|
|
requires_visual_check=False,
|
|
),
|
|
),
|
|
resolved_drug_id="metformin",
|
|
)
|
|
|
|
|
|
class _FixedRouting:
|
|
def __init__(self, result: RetrievalResult) -> None:
|
|
self._result = result
|
|
|
|
def retrieve(self, query, subject_scope, intent):
|
|
return self._result
|
|
|
|
|
|
class _Generator:
|
|
"""Returns whatever payload the test wants the model to have produced.
|
|
|
|
`_generate` now makes up to four calls through this port: the main
|
|
answer, a sufficiency check (skipped here — one evidence block), and up
|
|
to two entailment calls (a reject retries once — live probing found the
|
|
judge noisy on an identical claim/evidence pair). They're told apart by
|
|
schema, so a test that only cares about the main answer doesn't also
|
|
have to fake an entailment response by hand; `entailment_payload`
|
|
overrides it when a test wants the entailment pass to reject. Pass a
|
|
list of payloads to get a different answer on each successive
|
|
entailment call (e.g. `[reject, accept]` for the retry-recovers case).
|
|
"""
|
|
|
|
def __init__(self, payload, entailment_payload=None) -> None:
|
|
self._payload = payload
|
|
default = {"entailed": True, "unsupported": []}
|
|
payloads = entailment_payload if entailment_payload is not None else default
|
|
self._entailment_payloads = (
|
|
list(payloads) if isinstance(payloads, list) else [payloads]
|
|
)
|
|
self._entailment_call = 0
|
|
|
|
def generate(self, system: str, user: str, schema: dict) -> str:
|
|
if "entailed" in schema.get("properties", {}):
|
|
index = min(self._entailment_call, len(self._entailment_payloads) - 1)
|
|
payload = self._entailment_payloads[index]
|
|
self._entailment_call += 1
|
|
else:
|
|
payload = self._payload
|
|
if isinstance(payload, BaseException):
|
|
raise payload
|
|
if isinstance(payload, str):
|
|
return payload
|
|
return json.dumps(payload, ensure_ascii=False)
|
|
|
|
|
|
def _answer(payload, result: RetrievalResult | None = None, entailment_payload=None):
|
|
metrics = InMemoryMetrics()
|
|
service = GroundedAnswerService(
|
|
_FixedRouting(result or _result()),
|
|
_Generator(payload, entailment_payload),
|
|
metrics,
|
|
)
|
|
grounded = service.answer("Liều Metformin?", SubjectScope.HUMAN, QueryIntent.FACT_LOOKUP)
|
|
return grounded, metrics
|
|
|
|
|
|
# --- the guardrail's whole reason to exist ------------------------------------
|
|
|
|
|
|
def test_invented_dose_is_refused_and_never_reaches_the_answer():
|
|
grounded, metrics = _answer(
|
|
{"answer": "Người lớn uống 850 mg, 2 lần mỗi ngày [1].",
|
|
"evidence_sufficient": True}
|
|
)
|
|
|
|
assert grounded.generated is False
|
|
# A generator is configured, so a rejected generation abstains — it does
|
|
# NOT silently degrade to a raw source dump (owner correction, 2026-08-06:
|
|
# this is a real LLM chatbot, not the retired offline-extractive build).
|
|
assert grounded.answer is None
|
|
assert grounded.result.decision == EvidenceDecision.ABSTAIN
|
|
assert grounded.result.reason == "generation_unavailable"
|
|
assert metrics.total(GENERATION_REJECTED, reason="ungrounded_number") == 1
|
|
|
|
|
|
def test_a_rounded_figure_counts_as_invented():
|
|
"""`2 g` is in the source; `2000 mg` is a conversion, and conversions are
|
|
where unit errors live. The prompt forbids it and the check enforces it."""
|
|
grounded, metrics = _answer(
|
|
{"answer": "Liều tối đa 2000 mg mỗi ngày [1].", "evidence_sufficient": True}
|
|
)
|
|
|
|
assert grounded.generated is False
|
|
assert metrics.total(GENERATION_REJECTED, reason="ungrounded_number") == 1
|
|
|
|
|
|
def test_citation_pointing_at_nothing_is_refused():
|
|
grounded, metrics = _answer(
|
|
{"answer": "Người lớn uống 500 mg [3].", "evidence_sufficient": True}
|
|
)
|
|
|
|
assert grounded.generated is False
|
|
# [3] is out of range with one evidence block, so "500" has no valid
|
|
# citation to bind to — grounding.verify now flags it as unsupported
|
|
# rather than letting it pass because 500 happens to exist somewhere in
|
|
# the (single) evidence block anyway. ungrounded_number takes priority
|
|
# over invalid_citation in GroundingReport.reason; both are present.
|
|
assert metrics.total(GENERATION_REJECTED, reason="ungrounded_number") == 1
|
|
|
|
|
|
def test_faithful_rewrite_is_served():
|
|
grounded, metrics = _answer(
|
|
{"answer": "Người lớn: 500 mg, 2 lần/ngày; tối đa 2 g/ngày [1].",
|
|
"evidence_sufficient": True}
|
|
)
|
|
|
|
assert grounded.generated is True
|
|
assert grounded.answer == "Người lớn: 500 mg, 2 lần/ngày; tối đa 2 g/ngày [1]."
|
|
assert metrics.total(GENERATION_SERVED) == 1
|
|
assert metrics.total(GENERATION_REJECTED) == 0
|
|
|
|
|
|
# --- the entailment pass: catches what number/citation checks structurally can't -----
|
|
|
|
|
|
def test_fabricated_nonnumeric_claim_with_a_valid_citation_is_rejected():
|
|
"""Reproduces `claim_bia` from the Codex 2026-08-06 review end to end:
|
|
right drug, syntactically valid citation, fabricated indication.
|
|
grounding.verify alone cannot see this (no number, citation in range) —
|
|
the entailment pass, told the model judged evidence 1 does not support
|
|
it, is what rejects the generation."""
|
|
grounded, metrics = _answer(
|
|
{"answer": "Metformin chữa ung thư [1].", "evidence_sufficient": True},
|
|
entailment_payload={"entailed": False, "unsupported": [1]},
|
|
)
|
|
|
|
assert grounded.generated is False
|
|
assert grounded.answer is None
|
|
assert grounded.result.decision == EvidenceDecision.ABSTAIN
|
|
assert metrics.total(GENERATION_REJECTED, reason="unsupported_claim") == 1
|
|
|
|
|
|
def test_entailment_check_running_and_passing_still_serves_the_answer():
|
|
grounded, metrics = _answer(
|
|
{"answer": "Metformin dùng điều trị đái tháo đường [1].",
|
|
"evidence_sufficient": True},
|
|
entailment_payload={"entailed": True, "unsupported": []},
|
|
)
|
|
|
|
assert grounded.generated is True
|
|
assert metrics.total(GENERATION_SERVED) == 1
|
|
|
|
|
|
def test_entailment_retries_once_after_a_reject_and_a_later_accept_serves():
|
|
"""Reproduces the 2026-08-06 live finding: the same claim/evidence pair,
|
|
called three times through the real judge, came back entailed twice and
|
|
rejected once — a single noisy reject must not discard a correct,
|
|
well-cited answer."""
|
|
grounded, metrics = _answer(
|
|
{"answer": "Metformin dùng điều trị đái tháo đường [1].",
|
|
"evidence_sufficient": True},
|
|
entailment_payload=[
|
|
{"entailed": False, "unsupported": [1]},
|
|
{"entailed": True, "unsupported": []},
|
|
],
|
|
)
|
|
|
|
assert grounded.generated is True
|
|
assert metrics.total(GENERATION_SERVED) == 1
|
|
|
|
|
|
def test_entailment_two_agreeing_rejects_still_discard():
|
|
grounded, metrics = _answer(
|
|
{"answer": "Metformin chữa ung thư [1].", "evidence_sufficient": True},
|
|
entailment_payload=[
|
|
{"entailed": False, "unsupported": [1]},
|
|
{"entailed": False, "unsupported": [1]},
|
|
],
|
|
)
|
|
|
|
assert grounded.generated is False
|
|
assert grounded.answer is None
|
|
assert metrics.total(GENERATION_REJECTED, reason="unsupported_claim") == 1
|
|
|
|
|
|
def test_entailment_provider_outage_fails_closed_to_abstain():
|
|
grounded, metrics = _answer(
|
|
{"answer": "Người lớn: 500 mg, 2 lần/ngày [1].", "evidence_sufficient": True},
|
|
entailment_payload=AnswerGenerationUnavailable(),
|
|
)
|
|
|
|
assert grounded.generated is False
|
|
assert grounded.answer is None
|
|
assert grounded.result.decision == EvidenceDecision.ABSTAIN
|
|
assert metrics.total(GENERATION_REJECTED, reason="unsupported_claim") == 1
|
|
|
|
|
|
def test_entailment_check_is_skipped_when_the_answer_has_no_claim_text():
|
|
"""An answer that is nothing but a citation marker has no claim text for
|
|
an entailment pass to check against — `_verify_entailment` must not call
|
|
the provider at all. Proven by making that call raise: if the skip
|
|
didn't fire, this would reject rather than serve the answer."""
|
|
grounded, metrics = _answer(
|
|
{"answer": "[1]", "evidence_sufficient": True},
|
|
entailment_payload=AnswerGenerationUnavailable(),
|
|
)
|
|
|
|
assert grounded.generated is True
|
|
assert metrics.total(GENERATION_REJECTED) == 0
|
|
|
|
|
|
def test_citations_survive_generation():
|
|
"""Provenance is the point; a prettier answer must not cost the folio."""
|
|
grounded, _ = _answer(
|
|
{"answer": "Người lớn: 500 mg [1].", "evidence_sufficient": True}
|
|
)
|
|
|
|
assert grounded.generated is True
|
|
assert len(grounded.citations) == 1
|
|
assert grounded.citations[0].printed_page_start == 714
|
|
|
|
|
|
# --- a configured generator that fails abstains, never a raw source dump -----
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"payload, reason",
|
|
[
|
|
(AnswerGenerationUnavailable("revoked"), "provider_unavailable"),
|
|
("not json at all", "malformed_output"),
|
|
({"answer": "500 mg [1]"}, "malformed_output"),
|
|
({"answer": 500, "evidence_sufficient": True}, "malformed_output"),
|
|
({"answer": "...", "evidence_sufficient": False}, "evidence_insufficient"),
|
|
],
|
|
)
|
|
def test_every_generation_failure_abstains_instead_of_a_raw_source_dump(payload, reason):
|
|
grounded, metrics = _answer(payload)
|
|
|
|
assert grounded.generated is False
|
|
assert grounded.answer is None
|
|
assert grounded.result.decision == EvidenceDecision.ABSTAIN
|
|
assert grounded.result.reason == "generation_unavailable"
|
|
assert metrics.total(GENERATION_REJECTED, reason=reason) == 1
|
|
|
|
|
|
def test_no_generator_configured_still_answers():
|
|
service = GroundedAnswerService(_FixedRouting(_result()))
|
|
|
|
grounded = service.answer(
|
|
"Liều Metformin?", SubjectScope.HUMAN, QueryIntent.FACT_LOOKUP
|
|
)
|
|
|
|
assert grounded.generated is False
|
|
assert grounded.answer.startswith(EVIDENCE_TEXT)
|
|
|
|
|
|
# --- the comparison rule itself ----------------------------------------------
|
|
|
|
|
|
def test_decimal_separators_are_not_interchangeable():
|
|
"""`7,5` and `7.5` differ, and so do `7,5` and `75`. Normalising them
|
|
together is how a tenfold dose error scores as a match."""
|
|
source = ("Sơ sinh: 7,5 mg/kg cách 8 giờ/lần.",)
|
|
|
|
assert grounding.verify("7,5 mg/kg [1]", source).grounded is True
|
|
assert grounding.verify("7.5 mg/kg [1]", source).grounded is False
|
|
assert grounding.verify("75 mg/kg [1]", source).grounded is False
|
|
|
|
|
|
def test_citation_markers_are_not_read_as_quantities():
|
|
report = grounding.verify("Không dùng cho người suy thận [1].", ("Suy thận.",))
|
|
|
|
assert report.grounded is True
|
|
assert report.cited_indices == (1,)
|
|
|
|
|
|
def test_prompt_numbers_evidence_from_one():
|
|
request = build_request("Liều?", ("đoạn A", "đoạn B"))
|
|
|
|
assert "[1] đoạn A" in request.user
|
|
assert "[2] đoạn B" in request.user
|
|
assert "CHÉP NGUYÊN VĂN" in request.system
|
|
|
|
|
|
def test_prompt_refuses_to_build_without_evidence():
|
|
with pytest.raises(ValueError):
|
|
build_request("Liều?", ())
|