Checkpoint frontend UI/UX overhaul and ingestion embed benchmark work

2026-08-06 17:21:21 +07:00
parent 1e8cbdb586
commit a4b8e1c4db
78 changed files with 6761 additions and 654 deletions
@@ -65,23 +65,48 @@ class _FixedRouting:


 class _Generator:
-    """Returns whatever payload the test wants the model to have produced."""
+    """Returns whatever payload the test wants the model to have produced.

-    def __init__(self, payload) -> None:
+    `_generate` now makes up to four calls through this port: the main
+    answer, a sufficiency check (skipped here — one evidence block), and up
+    to two entailment calls (a reject retries once — live probing found the
+    judge noisy on an identical claim/evidence pair). They're told apart by
+    schema, so a test that only cares about the main answer doesn't also
+    have to fake an entailment response by hand; `entailment_payload`
+    overrides it when a test wants the entailment pass to reject. Pass a
+    list of payloads to get a different answer on each successive
+    entailment call (e.g. `[reject, accept]` for the retry-recovers case).
+    """
+
+    def __init__(self, payload, entailment_payload=None) -> None:
        self._payload = payload
+        default = {"entailed": True, "unsupported": []}
+        payloads = entailment_payload if entailment_payload is not None else default
+        self._entailment_payloads = (
+            list(payloads) if isinstance(payloads, list) else [payloads]
+        )
+        self._entailment_call = 0

    def generate(self, system: str, user: str, schema: dict) -> str:
-        if isinstance(self._payload, BaseException):
-            raise self._payload
-        if isinstance(self._payload, str):
-            return self._payload
-        return json.dumps(self._payload, ensure_ascii=False)
+        if "entailed" in schema.get("properties", {}):
+            index = min(self._entailment_call, len(self._entailment_payloads) - 1)
+            payload = self._entailment_payloads[index]
+            self._entailment_call += 1
+        else:
+            payload = self._payload
+        if isinstance(payload, BaseException):
+            raise payload
+        if isinstance(payload, str):
+            return payload
+        return json.dumps(payload, ensure_ascii=False)


-def _answer(payload, result: RetrievalResult | None = None):
+def _answer(payload, result: RetrievalResult | None = None, entailment_payload=None):
    metrics = InMemoryMetrics()
    service = GroundedAnswerService(
-        _FixedRouting(result or _result()), _Generator(payload), metrics
+        _FixedRouting(result or _result()),
+        _Generator(payload, entailment_payload),
+        metrics,
    )
    grounded = service.answer("Liều Metformin?", SubjectScope.HUMAN, QueryIntent.FACT_LOOKUP)
    return grounded, metrics
@@ -97,8 +122,12 @@ def test_invented_dose_is_refused_and_never_reaches_the_answer():
    )

    assert grounded.generated is False
-    assert "850" not in grounded.answer
-    assert grounded.answer.startswith(EVIDENCE_TEXT)
+    # A generator is configured, so a rejected generation abstains — it does
+    # NOT silently degrade to a raw source dump (owner correction, 2026-08-06:
+    # this is a real LLM chatbot, not the retired offline-extractive build).
+    assert grounded.answer is None
+    assert grounded.result.decision == EvidenceDecision.ABSTAIN
+    assert grounded.result.reason == "generation_unavailable"
    assert metrics.total(GENERATION_REJECTED, reason="ungrounded_number") == 1


@@ -119,7 +148,12 @@ def test_citation_pointing_at_nothing_is_refused():
    )

    assert grounded.generated is False
-    assert metrics.total(GENERATION_REJECTED, reason="invalid_citation") == 1
+    # [3] is out of range with one evidence block, so "500" has no valid
+    # citation to bind to — grounding.verify now flags it as unsupported
+    # rather than letting it pass because 500 happens to exist somewhere in
+    # the (single) evidence block anyway. ungrounded_number takes priority
+    # over invalid_citation in GroundingReport.reason; both are present.
+    assert metrics.total(GENERATION_REJECTED, reason="ungrounded_number") == 1


 def test_faithful_rewrite_is_served():
@@ -134,6 +168,95 @@ def test_faithful_rewrite_is_served():
    assert metrics.total(GENERATION_REJECTED) == 0


+# --- the entailment pass: catches what number/citation checks structurally can't -----
+
+
+def test_fabricated_nonnumeric_claim_with_a_valid_citation_is_rejected():
+    """Reproduces `claim_bia` from the Codex 2026-08-06 review end to end:
+    right drug, syntactically valid citation, fabricated indication.
+    grounding.verify alone cannot see this (no number, citation in range) —
+    the entailment pass, told the model judged evidence 1 does not support
+    it, is what rejects the generation."""
+    grounded, metrics = _answer(
+        {"answer": "Metformin chữa ung thư [1].", "evidence_sufficient": True},
+        entailment_payload={"entailed": False, "unsupported": [1]},
+    )
+
+    assert grounded.generated is False
+    assert grounded.answer is None
+    assert grounded.result.decision == EvidenceDecision.ABSTAIN
+    assert metrics.total(GENERATION_REJECTED, reason="unsupported_claim") == 1
+
+
+def test_entailment_check_running_and_passing_still_serves_the_answer():
+    grounded, metrics = _answer(
+        {"answer": "Metformin dùng điều trị đái tháo đường [1].",
+         "evidence_sufficient": True},
+        entailment_payload={"entailed": True, "unsupported": []},
+    )
+
+    assert grounded.generated is True
+    assert metrics.total(GENERATION_SERVED) == 1
+
+
+def test_entailment_retries_once_after_a_reject_and_a_later_accept_serves():
+    """Reproduces the 2026-08-06 live finding: the same claim/evidence pair,
+    called three times through the real judge, came back entailed twice and
+    rejected once — a single noisy reject must not discard a correct,
+    well-cited answer."""
+    grounded, metrics = _answer(
+        {"answer": "Metformin dùng điều trị đái tháo đường [1].",
+         "evidence_sufficient": True},
+        entailment_payload=[
+            {"entailed": False, "unsupported": [1]},
+            {"entailed": True, "unsupported": []},
+        ],
+    )
+
+    assert grounded.generated is True
+    assert metrics.total(GENERATION_SERVED) == 1
+
+
+def test_entailment_two_agreeing_rejects_still_discard():
+    grounded, metrics = _answer(
+        {"answer": "Metformin chữa ung thư [1].", "evidence_sufficient": True},
+        entailment_payload=[
+            {"entailed": False, "unsupported": [1]},
+            {"entailed": False, "unsupported": [1]},
+        ],
+    )
+
+    assert grounded.generated is False
+    assert grounded.answer is None
+    assert metrics.total(GENERATION_REJECTED, reason="unsupported_claim") == 1
+
+
+def test_entailment_provider_outage_fails_closed_to_abstain():
+    grounded, metrics = _answer(
+        {"answer": "Người lớn: 500 mg, 2 lần/ngày [1].", "evidence_sufficient": True},
+        entailment_payload=AnswerGenerationUnavailable(),
+    )
+
+    assert grounded.generated is False
+    assert grounded.answer is None
+    assert grounded.result.decision == EvidenceDecision.ABSTAIN
+    assert metrics.total(GENERATION_REJECTED, reason="unsupported_claim") == 1
+
+
+def test_entailment_check_is_skipped_when_the_answer_has_no_claim_text():
+    """An answer that is nothing but a citation marker has no claim text for
+    an entailment pass to check against — `_verify_entailment` must not call
+    the provider at all. Proven by making that call raise: if the skip
+    didn't fire, this would reject rather than serve the answer."""
+    grounded, metrics = _answer(
+        {"answer": "[1]", "evidence_sufficient": True},
+        entailment_payload=AnswerGenerationUnavailable(),
+    )
+
+    assert grounded.generated is True
+    assert metrics.total(GENERATION_REJECTED) == 0
+
+
 def test_citations_survive_generation():
    """Provenance is the point; a prettier answer must not cost the folio."""
    grounded, _ = _answer(
@@ -145,7 +268,7 @@ def test_citations_survive_generation():
    assert grounded.citations[0].printed_page_start == 714


-# --- degradation is always to the source, never to an error -------------------
+# --- a configured generator that fails abstains, never a raw source dump -----


@pytest.mark.parametrize(
@@ -158,11 +281,13 @@ def test_citations_survive_generation():
        ({"answer": "...", "evidence_sufficient": False}, "evidence_insufficient"),
    ],
 )
-def test_every_generation_failure_falls_back_to_the_source_text(payload, reason):
+def test_every_generation_failure_abstains_instead_of_a_raw_source_dump(payload, reason):
    grounded, metrics = _answer(payload)

    assert grounded.generated is False
-    assert grounded.answer.startswith(EVIDENCE_TEXT)
+    assert grounded.answer is None
+    assert grounded.result.decision == EvidenceDecision.ABSTAIN
+    assert grounded.result.reason == "generation_unavailable"
    assert metrics.total(GENERATION_REJECTED, reason=reason) == 1