Soften the tone of the docs and comments written today

2026-08-11 10:12:25 +07:00
parent 97cb6d16f4
commit 6b8f7584ed
18 changed files with 921 additions and 64 deletions
@@ -12,6 +12,7 @@ import pytest

 from rag import grounding
 from rag.answer import GroundedAnswerService
+from rag.budget import RequestBudgetExhausted
 from rag.metrics import GENERATION_REJECTED, GENERATION_SERVED, InMemoryMetrics
 from rag.models import (
    Evidence,
@@ -319,6 +320,16 @@ def test_entailment_accepts_after_one_semantic_pass():


 def test_entailment_provider_outage_fails_closed_to_abstain():
+    """Fail-closed is unchanged; only the label it fails closed *under* is.
+
+    This previously asserted `unsupported_claim`, which reports a claim the
+    evidence did not support, in a case where the judge was never reachable.
+    `apps/web/app/api/chat/route.ts` renders that as "bước đối chiếu chưa
+    xác nhận được câu trả lời khớp với nguồn", describing the answer rather
+    than the outage, and places it in the content-failure bucket that the
+    failure taxonomy in `docs/current-rag-pipeline-audit.md` §4 keeps
+    separate from availability.
+    """
    grounded, metrics = _answer(
        {"claims": [{"text": "Người lớn: 500 mg, 2 lần/ngày", "citations": [1]}],
         "evidence_sufficient": True},
@@ -328,7 +339,125 @@ def test_entailment_provider_outage_fails_closed_to_abstain():
    assert grounded.generated is False
    assert grounded.answer is None
    assert grounded.result.decision == EvidenceDecision.ABSTAIN
+    assert grounded.result.reason == "provider_unavailable"
+    assert metrics.total(GENERATION_REJECTED, reason="provider_unavailable") == 1
+    # And specifically NOT counted as a content failure.
+    assert metrics.total(GENERATION_REJECTED, reason="unsupported_claim") == 0
+
+
+def test_entailment_budget_exhaustion_is_reported_as_a_timeout_not_a_bad_claim():
+    """Observed live 2026-08-11 against production.
+
+    `RequestBudgetExhausted` subclasses `AnswerGenerationUnavailable`, so it
+    has to be caught first to be distinguishable from an ordinary outage;
+    previously both arrived as `unsupported_claim`. The user-facing string
+    for `request_budget_exhausted` already exists in the BFF mapping, so no
+    new reason code is introduced here.
+    """
+    grounded, metrics = _answer(
+        {"claims": [{"text": "Người lớn: 500 mg, 2 lần/ngày", "citations": [1]}],
+         "evidence_sufficient": True},
+        entailment_payload=RequestBudgetExhausted(),
+    )
+
+    assert grounded.generated is False
+    assert grounded.answer is None
+    assert grounded.result.decision == EvidenceDecision.ABSTAIN
+    assert grounded.result.reason == "request_budget_exhausted"
+    assert metrics.total(GENERATION_REJECTED, reason="request_budget_exhausted") == 1
+    assert metrics.total(GENERATION_REJECTED, reason="unsupported_claim") == 0
+    assert metrics.total(GENERATION_REJECTED, reason="provider_unavailable") == 0
+
+
+def test_unparseable_judge_reply_is_reported_as_malformed_not_as_a_bad_claim():
+    """A judge reply this code cannot read is not a verdict against the answer."""
+    grounded, metrics = _answer(
+        {"claims": [{"text": "Người lớn: 500 mg, 2 lần/ngày", "citations": [1]}],
+         "evidence_sufficient": True},
+        entailment_payload="{not json at all",
+    )
+
+    assert grounded.generated is False
+    assert grounded.answer is None
+    assert grounded.result.reason == "malformed_output"
+    assert metrics.total(GENERATION_REJECTED, reason="unsupported_claim") == 0
+
+
+def test_budget_running_out_during_completeness_repair_is_not_called_incomplete():
+    """Pins the production case observed live 2026-08-11.
+
+    "Liều dùng của Isosorbid dinitrat theo Dược thư là gì?" took 40.3s
+    against a 40s budget and returned `incomplete_answer`, whose user-facing
+    text says the answer was cancelled because the source had information it
+    left out — while what actually happened is that the repair generation did
+    not run to completion. The completeness repair roughly doubles a turn's
+    model calls, so it is the likeliest place to exhaust the budget, and it
+    reports that the same way the first attempt does.
+    """
+    grounded, metrics = _answer(
+        [
+            {"claims": [{"text": "Người lớn uống 500 mg", "citations": [1]}],
+             "evidence_sufficient": True},
+            RequestBudgetExhausted(),
+        ],
+        entailment_payload={
+            "entailed": True,
+            "unsupported": [],
+            "complete": False,
+            "missing_evidence": [{
+                "description": "2 lần mỗi ngày và liều tối đa 2 g mỗi ngày",
+                "evidence_quote": EVIDENCE_TEXT,
+            }],
+        },
+    )
+
+    assert grounded.answer is None
+    assert grounded.result.reason == "request_budget_exhausted"
+    assert metrics.total(GENERATION_REJECTED, reason="request_budget_exhausted") == 1
+    # The user must not be told their answer was missing source information
+    # when the repair simply ran out of time.
+    assert metrics.total(GENERATION_REJECTED, reason="incomplete_answer") == 0
+
+
+def test_a_genuinely_incomplete_repair_is_still_called_incomplete():
+    """Guards the other side of the split above: when the repair really does
+    run and still comes back incomplete, `incomplete_answer` must survive."""
+    incomplete_verdict = {
+        "entailed": True,
+        "unsupported": [],
+        "complete": False,
+        "missing_evidence": [{
+            "description": "2 lần mỗi ngày và liều tối đa 2 g mỗi ngày",
+            "evidence_quote": EVIDENCE_TEXT,
+        }],
+    }
+    grounded, metrics = _answer(
+        {"claims": [{"text": "Người lớn uống 500 mg", "citations": [1]}],
+         "evidence_sufficient": True},
+        entailment_payload=[incomplete_verdict, incomplete_verdict],
+    )
+
+    assert grounded.answer is None
+    assert grounded.result.reason == "incomplete_answer"
+    assert metrics.total(GENERATION_REJECTED, reason="incomplete_answer") == 1
+    assert metrics.total(GENERATION_REJECTED, reason="request_budget_exhausted") == 0
+
+
+def test_a_real_negative_verdict_is_still_an_unsupported_claim():
+    """The counterpart to the three tests above: when the judge DID run and
+    said no, the reason must stay a content failure. Splitting the
+    availability cases out must not quietly reclassify genuine rejections."""
+    grounded, metrics = _answer(
+        {"claims": [{"text": "Metformin chữa ung thư", "citations": [1]}],
+         "evidence_sufficient": True},
+        entailment_payload={"entailed": False, "unsupported": [1]},
+    )
+
+    assert grounded.generated is False
+    assert grounded.result.reason == "unsupported_claim"
    assert metrics.total(GENERATION_REJECTED, reason="unsupported_claim") == 1
+    assert metrics.total(GENERATION_REJECTED, reason="provider_unavailable") == 0
+    assert metrics.total(GENERATION_REJECTED, reason="request_budget_exhausted") == 0


 def test_entailment_check_is_skipped_when_there_are_no_claims():