"""The answer layer may rephrase evidence; it may not add to it. Every test here is a fabrication the generator could plausibly produce, and the assertion is that the clinician never sees it. The dose figures are taken from the real METFORMIN and PARACETAMOL sections in `duocthu_v1`. """ from __future__ import annotations import json import pytest from rag import grounding from rag.answer import GroundedAnswerService from rag.metrics import GENERATION_REJECTED, GENERATION_SERVED, InMemoryMetrics from rag.models import ( Evidence, EvidenceDecision, QueryIntent, RetrievalResult, SourceRef, SubjectScope, ) from rag.ports import AnswerGenerationUnavailable from rag.prompt import build_request SOURCE = SourceRef( physical_page=812, precision="region", printed_page_range=(714, 714), ) EVIDENCE_TEXT = ( "Người lớn: uống 500 mg metformin hydroclorid, 2 lần mỗi ngày. " "Liều tối đa 2 g mỗi ngày, chia làm nhiều lần." ) def _result(text: str = EVIDENCE_TEXT) -> RetrievalResult: return RetrievalResult( EvidenceDecision.ANSWERABLE, "grounded_evidence_available", ( Evidence( evidence_id="metformin::lieu::0", matched_doc_id="metformin::lieu::0", kind="prose", text=text, score=1.0, source_refs=(SOURCE,), hydrated_from_parent=False, requires_visual_check=False, ), ), resolved_drug_id="metformin", ) class _FixedRouting: def __init__(self, result: RetrievalResult) -> None: self._result = result def retrieve(self, query, subject_scope, intent): return self._result class _Generator: """Returns whatever payload the test wants the model to have produced. `_generate` now makes up to four calls through this port: the main answer, a sufficiency check (skipped here — one evidence block), and up to two entailment calls (a reject retries once — live probing found the judge noisy on an identical claim/evidence pair). They're told apart by schema, so a test that only cares about the main answer doesn't also have to fake an entailment response by hand; `entailment_payload` overrides it when a test wants the entailment pass to reject. Pass a list of payloads to get a different answer on each successive entailment call (e.g. `[reject, accept]` for the retry-recovers case). """ def __init__(self, payload, entailment_payload=None) -> None: self._payload = payload default = {"entailed": True, "unsupported": []} payloads = entailment_payload if entailment_payload is not None else default self._entailment_payloads = ( list(payloads) if isinstance(payloads, list) else [payloads] ) self._entailment_call = 0 def generate(self, system: str, user: str, schema: dict) -> str: if "entailed" in schema.get("properties", {}): index = min(self._entailment_call, len(self._entailment_payloads) - 1) payload = self._entailment_payloads[index] self._entailment_call += 1 else: payload = self._payload if isinstance(payload, BaseException): raise payload if isinstance(payload, str): return payload return json.dumps(payload, ensure_ascii=False) def _answer(payload, result: RetrievalResult | None = None, entailment_payload=None): metrics = InMemoryMetrics() service = GroundedAnswerService( _FixedRouting(result or _result()), _Generator(payload, entailment_payload), metrics, ) grounded = service.answer("Liều Metformin?", SubjectScope.HUMAN, QueryIntent.FACT_LOOKUP) return grounded, metrics # --- the guardrail's whole reason to exist ------------------------------------ def test_invented_dose_is_refused_and_never_reaches_the_answer(): grounded, metrics = _answer( {"answer": "Người lớn uống 850 mg, 2 lần mỗi ngày [1].", "evidence_sufficient": True} ) assert grounded.generated is False # A generator is configured, so a rejected generation abstains — it does # NOT silently degrade to a raw source dump (owner correction, 2026-08-06: # this is a real LLM chatbot, not the retired offline-extractive build). assert grounded.answer is None assert grounded.result.decision == EvidenceDecision.ABSTAIN assert grounded.result.reason == "generation_unavailable" assert metrics.total(GENERATION_REJECTED, reason="ungrounded_number") == 1 def test_a_rounded_figure_counts_as_invented(): """`2 g` is in the source; `2000 mg` is a conversion, and conversions are where unit errors live. The prompt forbids it and the check enforces it.""" grounded, metrics = _answer( {"answer": "Liều tối đa 2000 mg mỗi ngày [1].", "evidence_sufficient": True} ) assert grounded.generated is False assert metrics.total(GENERATION_REJECTED, reason="ungrounded_number") == 1 def test_citation_pointing_at_nothing_is_refused(): grounded, metrics = _answer( {"answer": "Người lớn uống 500 mg [3].", "evidence_sufficient": True} ) assert grounded.generated is False # [3] is out of range with one evidence block, so "500" has no valid # citation to bind to — grounding.verify now flags it as unsupported # rather than letting it pass because 500 happens to exist somewhere in # the (single) evidence block anyway. ungrounded_number takes priority # over invalid_citation in GroundingReport.reason; both are present. assert metrics.total(GENERATION_REJECTED, reason="ungrounded_number") == 1 def test_faithful_rewrite_is_served(): grounded, metrics = _answer( {"answer": "Người lớn: 500 mg, 2 lần/ngày; tối đa 2 g/ngày [1].", "evidence_sufficient": True} ) assert grounded.generated is True assert grounded.answer == "Người lớn: 500 mg, 2 lần/ngày; tối đa 2 g/ngày [1]." assert metrics.total(GENERATION_SERVED) == 1 assert metrics.total(GENERATION_REJECTED) == 0 # --- the entailment pass: catches what number/citation checks structurally can't ----- def test_fabricated_nonnumeric_claim_with_a_valid_citation_is_rejected(): """Reproduces `claim_bia` from the Codex 2026-08-06 review end to end: right drug, syntactically valid citation, fabricated indication. grounding.verify alone cannot see this (no number, citation in range) — the entailment pass, told the model judged evidence 1 does not support it, is what rejects the generation.""" grounded, metrics = _answer( {"answer": "Metformin chữa ung thư [1].", "evidence_sufficient": True}, entailment_payload={"entailed": False, "unsupported": [1]}, ) assert grounded.generated is False assert grounded.answer is None assert grounded.result.decision == EvidenceDecision.ABSTAIN assert metrics.total(GENERATION_REJECTED, reason="unsupported_claim") == 1 def test_entailment_check_running_and_passing_still_serves_the_answer(): grounded, metrics = _answer( {"answer": "Metformin dùng điều trị đái tháo đường [1].", "evidence_sufficient": True}, entailment_payload={"entailed": True, "unsupported": []}, ) assert grounded.generated is True assert metrics.total(GENERATION_SERVED) == 1 def test_entailment_retries_once_after_a_reject_and_a_later_accept_serves(): """Reproduces the 2026-08-06 live finding: the same claim/evidence pair, called three times through the real judge, came back entailed twice and rejected once — a single noisy reject must not discard a correct, well-cited answer.""" grounded, metrics = _answer( {"answer": "Metformin dùng điều trị đái tháo đường [1].", "evidence_sufficient": True}, entailment_payload=[ {"entailed": False, "unsupported": [1]}, {"entailed": True, "unsupported": []}, ], ) assert grounded.generated is True assert metrics.total(GENERATION_SERVED) == 1 def test_entailment_two_agreeing_rejects_still_discard(): grounded, metrics = _answer( {"answer": "Metformin chữa ung thư [1].", "evidence_sufficient": True}, entailment_payload=[ {"entailed": False, "unsupported": [1]}, {"entailed": False, "unsupported": [1]}, ], ) assert grounded.generated is False assert grounded.answer is None assert metrics.total(GENERATION_REJECTED, reason="unsupported_claim") == 1 def test_entailment_provider_outage_fails_closed_to_abstain(): grounded, metrics = _answer( {"answer": "Người lớn: 500 mg, 2 lần/ngày [1].", "evidence_sufficient": True}, entailment_payload=AnswerGenerationUnavailable(), ) assert grounded.generated is False assert grounded.answer is None assert grounded.result.decision == EvidenceDecision.ABSTAIN assert metrics.total(GENERATION_REJECTED, reason="unsupported_claim") == 1 def test_entailment_check_is_skipped_when_the_answer_has_no_claim_text(): """An answer that is nothing but a citation marker has no claim text for an entailment pass to check against — `_verify_entailment` must not call the provider at all. Proven by making that call raise: if the skip didn't fire, this would reject rather than serve the answer.""" grounded, metrics = _answer( {"answer": "[1]", "evidence_sufficient": True}, entailment_payload=AnswerGenerationUnavailable(), ) assert grounded.generated is True assert metrics.total(GENERATION_REJECTED) == 0 def test_citations_survive_generation(): """Provenance is the point; a prettier answer must not cost the folio.""" grounded, _ = _answer( {"answer": "Người lớn: 500 mg [1].", "evidence_sufficient": True} ) assert grounded.generated is True assert len(grounded.citations) == 1 assert grounded.citations[0].printed_page_start == 714 # --- a configured generator that fails abstains, never a raw source dump ----- @pytest.mark.parametrize( "payload, reason", [ (AnswerGenerationUnavailable("revoked"), "provider_unavailable"), ("not json at all", "malformed_output"), ({"answer": "500 mg [1]"}, "malformed_output"), ({"answer": 500, "evidence_sufficient": True}, "malformed_output"), ({"answer": "...", "evidence_sufficient": False}, "evidence_insufficient"), ], ) def test_every_generation_failure_abstains_instead_of_a_raw_source_dump(payload, reason): grounded, metrics = _answer(payload) assert grounded.generated is False assert grounded.answer is None assert grounded.result.decision == EvidenceDecision.ABSTAIN assert grounded.result.reason == "generation_unavailable" assert metrics.total(GENERATION_REJECTED, reason=reason) == 1 def test_no_generator_configured_still_answers(): service = GroundedAnswerService(_FixedRouting(_result())) grounded = service.answer( "Liều Metformin?", SubjectScope.HUMAN, QueryIntent.FACT_LOOKUP ) assert grounded.generated is False assert grounded.answer.startswith(EVIDENCE_TEXT) # --- the comparison rule itself ---------------------------------------------- def test_decimal_separators_are_not_interchangeable(): """`7,5` and `7.5` differ, and so do `7,5` and `75`. Normalising them together is how a tenfold dose error scores as a match.""" source = ("Sơ sinh: 7,5 mg/kg cách 8 giờ/lần.",) assert grounding.verify("7,5 mg/kg [1]", source).grounded is True assert grounding.verify("7.5 mg/kg [1]", source).grounded is False assert grounding.verify("75 mg/kg [1]", source).grounded is False def test_citation_markers_are_not_read_as_quantities(): report = grounding.verify("Không dùng cho người suy thận [1].", ("Suy thận.",)) assert report.grounded is True assert report.cited_indices == (1,) def test_prompt_numbers_evidence_from_one(): request = build_request("Liều?", ("đoạn A", "đoạn B")) assert "[1] đoạn A" in request.user assert "[2] đoạn B" in request.user assert "CHÉP NGUYÊN VĂN" in request.system def test_prompt_refuses_to_build_without_evidence(): with pytest.raises(ValueError): build_request("Liều?", ())