VuQuangBao/duocthu
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Remove corpus counts from chat chrome

Browse Source
This commit is contained in:
BaoVu2k4
2026-08-10 17:26:58 +07:00
parent 46469468bb
commit 97cb6d16f4
31 changed files with 2192 additions and 424 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/ai-service/tests/test_grounded_generation.py
+130 -75
View File
@@ -67,18 +67,14 @@ class _FixedRouting:
class _Generator:
"""Returns whatever payload the test wants the model to have produced.
`_generate` now makes up to five calls through this port: the main
answer (a lone `evidence_sufficient: false` retries once — the same
noisy-judge finding as entailment, live-confirmed 2026-08-07), a
sufficiency check (skipped here — one evidence block), and up to three
entailment calls (widened from two 2026-08-07: live probing found the
judge noisy on an identical claim/evidence pair, and a real adversarial
sample showed a single retry still discarding correct answers on the
unlucky reject-reject draw). They're told apart by schema, so a test
`_generate` can make a main answer call (a lone
`evidence_sufficient: false` retries once) and one fail-closed entailment
call. The legacy direct-answer path may also make a sufficiency call when
several evidence blocks need disambiguation; the structured agent path
skips that duplicate judgment. They're told apart by schema, so a test
that only cares about one call doesn't have to fake the others; `payload`
and `entailment_payload` each take either a fixed value or a list for a
different answer on each successive call to that schema (e.g.
`[reject, reject, accept]` for the third-attempt-recovers case).
different answer on each successive call to that schema.
"""
def __init__(self, payload, entailment_payload=None) -> None:
@@ -124,7 +120,7 @@ def _answer(payload, result: RetrievalResult | None = None, entailment_payload=N
def test_invented_dose_is_refused_and_never_reaches_the_answer():
grounded, metrics = _answer(
{"answer": "Người lớn uống 850 mg, 2 lần mỗi ngày [1].",
{"claims": [{"text": "Người lớn uống 850 mg, 2 lần mỗi ngày", "citations": [1]}],
"evidence_sufficient": True}
)
@@ -147,7 +143,8 @@ def test_a_rounded_figure_counts_as_invented():
"""`2 g` is in the source; `2000 mg` is a conversion, and conversions are
where unit errors live. The prompt forbids it and the check enforces it."""
grounded, metrics = _answer(
{"answer": "Liều tối đa 2000 mg mỗi ngày [1].", "evidence_sufficient": True}
{"claims": [{"text": "Liều tối đa 2000 mg mỗi ngày", "citations": [1]}],
"evidence_sufficient": True}
)
assert grounded.generated is False
@@ -156,7 +153,8 @@ def test_a_rounded_figure_counts_as_invented():
def test_citation_pointing_at_nothing_is_refused():
grounded, metrics = _answer(
{"answer": "Người lớn uống 500 mg [3].", "evidence_sufficient": True}
{"claims": [{"text": "Người lớn uống 500 mg", "citations": [3]}],
"evidence_sufficient": True}
)
assert grounded.generated is False
@@ -170,12 +168,13 @@ def test_citation_pointing_at_nothing_is_refused():
def test_faithful_rewrite_is_served():
grounded, metrics = _answer(
{"answer": "Người lớn: 500 mg, 2 lần/ngày; tối đa 2 g/ngày [1].",
{"claims": [{"text": "Người lớn: 500 mg, 2 lần/ngày; tối đa 2 g/ngày", "citations": [1]}],
"evidence_sufficient": True}
)
assert grounded.generated is True
assert grounded.answer == "Người lớn: 500 mg, 2 lần/ngày; tối đa 2 g/ngày [1]."
assert grounded.answer == "Người lớn: 500 mg, 2 lần/ngày; tối đa 2 g/ngày"
assert grounded.blocks[0].claims[0].source_ids == ("metformin::lieu::0",)
assert metrics.total(GENERATION_SERVED) == 1
assert metrics.total(GENERATION_REJECTED) == 0
@@ -190,7 +189,7 @@ def test_fabricated_nonnumeric_claim_with_a_valid_citation_is_rejected():
the entailment pass, told the model judged evidence 1 does not support
it, is what rejects the generation."""
grounded, metrics = _answer(
{"answer": "Metformin chữa ung thư [1].", "evidence_sufficient": True},
{"claims": [{"text": "Metformin chữa ung thư", "citations": [1]}], "evidence_sufficient": True},
entailment_payload={"entailed": False, "unsupported": [1]},
)
@@ -202,7 +201,7 @@ def test_fabricated_nonnumeric_claim_with_a_valid_citation_is_rejected():
def test_entailment_check_running_and_passing_still_serves_the_answer():
grounded, metrics = _answer(
{"answer": "Metformin dùng điều trị đái tháo đường [1].",
{"claims": [{"text": "Metformin dùng điều trị đái tháo đường", "citations": [1]}],
"evidence_sufficient": True},
entailment_payload={"entailed": True, "unsupported": []},
)
@@ -211,13 +210,15 @@ def test_entailment_check_running_and_passing_still_serves_the_answer():
assert metrics.total(GENERATION_SERVED) == 1
def test_entailment_retries_once_after_a_reject_and_a_later_accept_serves():
"""Reproduces the 2026-08-06 live finding: the same claim/evidence pair,
called three times through the real judge, came back entailed twice and
rejected once — a single noisy reject must not discard a correct,
well-cited answer."""
def test_entailment_rejects_after_one_fail_closed_semantic_pass():
"""The verifier is one semantic pass after deterministic grounding.
Repeating an identical temperature-0 prompt against the same model is a
correlated retry, not an independent vote, and doubled the hot-path model
latency for every valid answer.
"""
grounded, metrics = _answer(
{"answer": "Metformin dùng điều trị đái tháo đường [1].",
{"claims": [{"text": "Metformin chữa ung thư", "citations": [1]}],
"evidence_sufficient": True},
entailment_payload=[
{"entailed": False, "unsupported": [1]},
@@ -225,52 +226,102 @@ def test_entailment_retries_once_after_a_reject_and_a_later_accept_serves():
],
)
assert grounded.generated is True
assert metrics.total(GENERATION_SERVED) == 1
def test_entailment_recovers_on_third_attempt_after_two_rejects():
"""The improvement 2026-08-07 widened the retry from 2 to 3 attempts
after a live 50-question adversarial sample found the 2-attempt policy's
own math (~11% false-discard rate on a genuinely valid claim, from the
noise probed in the docstring above) matched the observed real
abstention rate almost exactly. Two rejects followed by a real accept
must now be served, not discarded."""
grounded, metrics = _answer(
{"answer": "Metformin dùng điều trị đái tháo đường [1].",
"evidence_sufficient": True},
entailment_payload=[
{"entailed": False, "unsupported": [1]},
{"entailed": False, "unsupported": [1]},
{"entailed": True, "unsupported": []},
],
)
assert grounded.generated is True
assert metrics.total(GENERATION_SERVED) == 1
assert metrics.total(GENERATION_REJECTED) == 0
def test_entailment_three_agreeing_rejects_still_discard():
grounded, metrics = _answer(
{"answer": "Metformin chữa ung thư [1].", "evidence_sufficient": True},
entailment_payload=[
{"entailed": False, "unsupported": [1]},
{"entailed": False, "unsupported": [1]},
{"entailed": False, "unsupported": [1]},
],
)
assert grounded.generated is False
assert grounded.answer is None
# All 3 attempts are noisy-judge calls against the SAME claim/evidence —
# a real, reliable rejection must still discard exactly once, not 3x.
assert metrics.total(GENERATION_REJECTED, reason="unsupported_claim") == 1
def test_supported_but_incomplete_answer_is_rejected_against_full_raw_evidence():
grounded, metrics = _answer(
{
"claims": [{"text": "Người lớn uống 500 mg", "citations": [1]}],
"evidence_sufficient": True,
},
entailment_payload={
"entailed": True,
"unsupported": [],
"complete": False,
"missing_evidence": [{
"description": "2 lần mỗi ngày và liều tối đa 2 g mỗi ngày",
"evidence_quote": EVIDENCE_TEXT,
}],
},
)
assert grounded.answer is None
assert grounded.result.reason == "incomplete_answer"
assert metrics.total(GENERATION_REJECTED, reason="incomplete_answer") == 1
def test_completeness_judge_cannot_claim_its_own_quoted_fact_is_missing():
grounded, _ = _answer(
{
"claims": [{
"text": "Chảy máu giữa vòng kinh (rất hay gặp trong 3 tháng đầu dùng thuốc theo đường tiêm).",
"citations": [1],
}],
"evidence_sufficient": True,
},
result=_result(
"Chảy máu giữa vòng kinh (rất hay gặp trong 3 tháng đầu dùng thuốc theo đường tiêm)."
),
entailment_payload={
"entailed": True,
"unsupported": [],
"complete": False,
"missing_evidence": [{
"description": "Không ghi nhận 'rất hay gặp trong 3 tháng đầu dùng thuốc theo đường tiêm'",
"evidence_quote": "rất hay gặp trong 3 tháng đầu dùng thuốc theo đường tiêm",
}],
},
)
assert grounded.generated is True
assert grounded.answer is not None
def test_completeness_objection_without_a_real_source_quote_is_ignored():
grounded, _ = _answer(
{
"claims": [{
"text": "Người lớn: uống 500 mg metformin hydroclorid, 2 lần mỗi ngày. Liều tối đa 2 g mỗi ngày, chia làm nhiều lần.",
"citations": [1],
}],
"evidence_sufficient": True,
},
entailment_payload={
"entailed": True,
"unsupported": [],
"complete": False,
"missing_evidence": [{
"description": "Không nêu điều kiện độ ẩm",
"evidence_quote": "độ ẩm",
}],
},
)
assert grounded.generated is True
assert grounded.answer is not None
def test_entailment_accepts_after_one_semantic_pass():
grounded, metrics = _answer(
{"claims": [{"text": "Metformin dùng điều trị đái tháo đường", "citations": [1]}],
"evidence_sufficient": True},
entailment_payload=[
{"entailed": True, "unsupported": []},
{"entailed": False, "unsupported": [1]}, # never consulted
],
)
assert grounded.generated is True
assert metrics.total(GENERATION_SERVED) == 1
def test_entailment_provider_outage_fails_closed_to_abstain():
grounded, metrics = _answer(
{"answer": "Người lớn: 500 mg, 2 lần/ngày [1].", "evidence_sufficient": True},
{"claims": [{"text": "Người lớn: 500 mg, 2 lần/ngày", "citations": [1]}],
"evidence_sufficient": True},
entailment_payload=AnswerGenerationUnavailable(),
)
@@ -280,13 +331,16 @@ def test_entailment_provider_outage_fails_closed_to_abstain():
assert metrics.total(GENERATION_REJECTED, reason="unsupported_claim") == 1
def test_entailment_check_is_skipped_when_the_answer_has_no_claim_text():
"""An answer that is nothing but a citation marker has no claim text for
an entailment pass to check against — `_verify_entailment` must not call
def test_entailment_check_is_skipped_when_there_are_no_claims():
"""No claims at all (2026-08-10: the structured-claims schema makes a
claim's `text` a required, non-empty field, so the old "answer is
nothing but a bare citation marker" scenario can no longer occur — the
analogous edge case is an empty `claims` list) has nothing for an
entailment pass to check against — `_verify_entailment` must not call
the provider at all. Proven by making that call raise: if the skip
didn't fire, this would reject rather than serve the answer."""
grounded, metrics = _answer(
{"answer": "[1]", "evidence_sufficient": True},
{"claims": [], "evidence_sufficient": True},
entailment_payload=AnswerGenerationUnavailable(),
)
@@ -297,7 +351,8 @@ def test_entailment_check_is_skipped_when_the_answer_has_no_claim_text():
def test_citations_survive_generation():
"""Provenance is the point; a prettier answer must not cost the folio."""
grounded, _ = _answer(
{"answer": "Người lớn: 500 mg [1].", "evidence_sufficient": True}
{"claims": [{"text": "Người lớn: 500 mg", "citations": [1]}],
"evidence_sufficient": True}
)
assert grounded.generated is True
@@ -313,9 +368,9 @@ def test_citations_survive_generation():
[
(AnswerGenerationUnavailable("revoked"), "provider_unavailable"),
("not json at all", "malformed_output"),
({"answer": "500 mg [1]"}, "malformed_output"),
({"answer": 500, "evidence_sufficient": True}, "malformed_output"),
({"answer": "...", "evidence_sufficient": False}, "evidence_insufficient"),
({"claims": [{"text": "500 mg", "citations": [1]}]}, "malformed_output"),
({"claims": 500, "evidence_sufficient": True}, "malformed_output"),
({"claims": [], "evidence_sufficient": False}, "evidence_insufficient"),
],
)
def test_every_generation_failure_abstains_instead_of_a_raw_source_dump(payload, reason):
@@ -340,21 +395,21 @@ def test_evidence_insufficient_retries_once_and_recovers():
pattern already known for entailment, just on a different field of the
same call. A lone insufficient verdict must not be final."""
grounded, metrics = _answer([
{"answer": "...", "evidence_sufficient": False},
{"answer": "Metformin dùng điều trị đái tháo đường [1].",
{"claims": [], "evidence_sufficient": False},
{"claims": [{"text": "Metformin dùng điều trị đái tháo đường", "citations": [1]}],
"evidence_sufficient": True},
])
assert grounded.generated is True
assert grounded.answer == "Metformin dùng điều trị đái tháo đường [1]."
assert grounded.answer == "Metformin dùng điều trị đái tháo đường"
assert metrics.total(GENERATION_SERVED) == 1
assert metrics.total(GENERATION_REJECTED) == 0
def test_evidence_insufficient_twice_still_abstains():
grounded, metrics = _answer([
{"answer": "...", "evidence_sufficient": False},
{"answer": "...", "evidence_sufficient": False},
{"claims": [], "evidence_sufficient": False},
{"claims": [], "evidence_sufficient": False},
])
assert grounded.generated is False