VuQuangBao/duocthu
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Soften the tone of the docs and comments written today

Browse Source
This commit is contained in:
BaoVu2k4
2026-08-11 10:12:25 +07:00
parent 97cb6d16f4
commit 6b8f7584ed
18 changed files with 921 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/ai-service/rag/answer.py
+94 -26
View File
@@ -22,7 +22,13 @@ from .routing import QueryRoutingService
# Repeating the same temperature-0 prompt against the same model is a
# correlated retry, not an independent vote. One fail-closed semantic pass is
# kept after structured-claim parsing and deterministic grounding.
_ENTAILMENT_MAX_ATTEMPTS = 1
#
# This is expressed as a single call rather than a `range(1)` loop on
# purpose: the loop this replaced returned on its first iteration on every
# path, so raising the constant looked like it added retries while silently
# doing nothing. If a future change genuinely wants more passes, they must
# be different prompts (or a different judge) to be independent evidence —
# see this function's own reasoning above.
_QUICK_REPLY_MAX_ITEMS = 4
_QUICK_REPLY_MAX_CHARS = 40
@@ -112,6 +118,29 @@ class GroundedAnswer:
plan: AnswerPlan | None = None
@dataclass(frozen=True)
class _CheckNotRun:
"""The entailment judge could not be consulted at all.
Distinct from a negative verdict on purpose. Fail-closed behaviour is
identical either way — the answer is still discarded — but the reason
code should not report an unsupported claim when the check never ran.
`docs/current-rag-pipeline-audit.md` §4's failure taxonomy keeps
availability and content failures separate for this reason. Observed
live 2026-08-11: a request that ran out of wall-clock budget mid-
verification reached the user as "bước đối chiếu chưa xác nhận được câu
trả lời khớp với nguồn", which describes the answer rather than the
timeout that actually occurred.
`reason` is deliberately one of the codes that already exist and are
already mapped in `apps/web/app/api/chat/route.ts`; a code with no entry
there falls back to wording that reads as "no data in the formulary",
which would misdescribe these cases.
"""
reason: str
@dataclass(frozen=True)
class _GenOutcome:
answer: str | None = None
@@ -714,7 +743,15 @@ class GroundedAnswerService:
verification = self._verify_entailment(
query, attempt.claims, shown_evidence, budget=budget
)
if verification is None or not verification.supported:
if isinstance(verification, _CheckNotRun):
# The judge never ran. Still fail closed, but report why: a
# timeout or outage recorded as "unsupported claim" would sit in
# the content-failure bucket and be hard to spot in metrics.
self._metrics.increment(
metric_names.GENERATION_REJECTED, reason=verification.reason
)
return _GenOutcome(reject_reason=verification.reason)
if not verification.supported:
self._metrics.increment(
metric_names.GENERATION_REJECTED, reason="unsupported_claim"
)
@@ -737,6 +774,25 @@ class GroundedAnswerService:
schema=request.schema,
)
repaired = self._attempt_generation(repair_request, budget)
# The repair roughly doubles a turn's model calls, so it is the
# most likely place to run out of wall-clock budget. Observed
# live 2026-08-11 (Isosorbid dinitrat dosage, 40.3s against a 40s
# budget): running out here fell through to `incomplete_answer`,
# which describes the answer as missing source information rather
# than reporting that the repair did not finish. Report the
# availability failure as itself, as the first attempt above
# already does.
if repaired.budget_exhausted:
self._metrics.increment(
metric_names.GENERATION_REJECTED,
reason="request_budget_exhausted",
)
return _GenOutcome(reject_reason="request_budget_exhausted")
if repaired.outage:
self._metrics.increment(
metric_names.GENERATION_REJECTED, reason="provider_unavailable"
)
return _GenOutcome(reject_reason="provider_unavailable")
if repaired.answer is not None:
repaired_report = grounding.verify(repaired.answer, evidence_texts)
repaired_verification = self._verify_entailment(
@@ -747,9 +803,14 @@ class GroundedAnswerService:
repaired.claims,
repaired_verification,
)
if isinstance(repaired_verification, _CheckNotRun):
self._metrics.increment(
metric_names.GENERATION_REJECTED,
reason=repaired_verification.reason,
)
return _GenOutcome(reject_reason=repaired_verification.reason)
if (
repaired_report.grounded
and repaired_verification is not None
and repaired_verification.supported
and repaired_verification.complete
):
@@ -766,7 +827,7 @@ class GroundedAnswerService:
structured_claims: tuple[tuple[str, tuple[int, ...]], ...],
evidence_texts: tuple[str, ...],
budget: RequestBudget | None = None,
) -> _VerificationOutcome | None:
) -> _VerificationOutcome | _CheckNotRun:
"""A second, adversarial LLM pass over an answer that already passed
`grounding.verify`.
@@ -808,49 +869,56 @@ class GroundedAnswerService:
return _VerificationOutcome(supported=True, complete=True)
request = build_entailment_request(query, claims, evidence_texts)
for _ in range(_ENTAILMENT_MAX_ATTEMPTS):
verdict = self._run_entailment_check(
request, evidence_texts=evidence_texts, budget=budget
)
if verdict is None:
return None
if (
verdict.supported
and not verdict.complete
and _missing_is_already_explicit(verdict.missing, structured_claims)
):
return _VerificationOutcome(supported=True, complete=True)
verdict = self._run_entailment_check(
request, evidence_texts=evidence_texts, budget=budget
)
if isinstance(verdict, _CheckNotRun):
return verdict
return False
if (
verdict.supported
and not verdict.complete
and _missing_is_already_explicit(verdict.missing, structured_claims)
):
return _VerificationOutcome(supported=True, complete=True)
return verdict
def _run_entailment_check(
self,
request,
evidence_texts: tuple[str, ...],
budget: RequestBudget | None = None,
) -> _VerificationOutcome | None:
"""One entailment call. `None` = outage/malformed/budget-exhausted
(fails closed by the caller without a retry); `True`/`False` = the
judge's verdict."""
) -> _VerificationOutcome | _CheckNotRun:
"""One entailment call.
Returns `_CheckNotRun` when the judge could not be consulted at all
(budget exhausted, provider outage, or a reply this code cannot
parse) and a `_VerificationOutcome` when it ran and reached a
verdict. Both make the caller fail closed; they differ only in the
reason reported, which used to collapse into `unsupported_claim`
for all of them.
"""
try:
if budget is not None:
budget.require()
raw = self._generator.generate(request.system, request.user, request.schema)
except RequestBudgetExhausted:
# Must be caught before its parent below — see `budget.py`.
return _CheckNotRun("request_budget_exhausted")
except AnswerGenerationUnavailable:
return None
return _CheckNotRun("provider_unavailable")
try:
payload = json.loads(raw)
entailed = payload["entailed"]
unsupported = payload["unsupported"]
missing_evidence = payload.get("missing_evidence", [])
except (ValueError, TypeError, KeyError):
return None
return _CheckNotRun("malformed_output")
if (
not isinstance(entailed, bool)
or not isinstance(unsupported, list)
or not isinstance(missing_evidence, list)
):
return None
return _CheckNotRun("malformed_output")
if not entailed or unsupported:
return _VerificationOutcome(supported=False, complete=False)
# A completeness objection is itself a factual claim about the raw
@@ -866,11 +934,11 @@ class GroundedAnswerService:
grounded_missing: list[str] = []
for item in missing_evidence:
if not isinstance(item, dict):
return None
return _CheckNotRun("malformed_output")
description = item.get("description")
evidence_quote = item.get("evidence_quote")
if not isinstance(description, str) or not isinstance(evidence_quote, str):
return None
return _CheckNotRun("malformed_output")
description = description.strip()
quote = _normalise_for_coverage(evidence_quote)
if (