84 lines
3.0 KiB
Python
84 lines
3.0 KiB
Python
"""Checks a generated answer against the evidence it was built from.
|
|
|
|
The answer layer may only rephrase retrieved text. This module is what makes
|
|
that a checkable property rather than a promise in a prompt: it recomputes,
|
|
from the evidence alone, whether every number and every citation in a
|
|
generated answer can be traced back to the source. A generation that fails is
|
|
discarded, never shown.
|
|
|
|
Numbers are compared **character for character**, deliberately. "7,5" and
|
|
"7.5" are not treated as equal, and no attempt is made to parse either into a
|
|
quantity. Parsing invites the one error that matters most here: `1.500` is
|
|
1500 under one reading and 1.5 under another, and a normaliser that strips
|
|
separators maps "7,5" and "75" to the same key — a tenfold dose error scored
|
|
as a match. The model is told to copy figures verbatim, so an exact match is
|
|
achievable, and every deviation from it is refused rather than interpreted.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
from dataclasses import dataclass
|
|
|
|
# A digit run with internal separators kept: "500", "7,5", "1.000".
|
|
# Ranges ("4 - 6 giờ") yield two tokens, and each is checked on its own.
|
|
_NUMBER = re.compile(r"\d+(?:[.,]\d+)*")
|
|
|
|
# Citation markers are stripped before number extraction so that "[2]" is
|
|
# never mistaken for the quantity 2.
|
|
_CITATION = re.compile(r"\[(\d+)\]")
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class GroundingReport:
|
|
grounded: bool
|
|
unsupported_numbers: tuple[str, ...]
|
|
invalid_citations: tuple[int, ...]
|
|
cited_indices: tuple[int, ...]
|
|
|
|
@property
|
|
def reason(self) -> str:
|
|
if self.unsupported_numbers:
|
|
return "ungrounded_number"
|
|
if self.invalid_citations:
|
|
return "invalid_citation"
|
|
return "grounded"
|
|
|
|
|
|
def numbers_in(text: str) -> tuple[str, ...]:
|
|
"""Numeric tokens, with citation markers removed first."""
|
|
return tuple(_NUMBER.findall(_CITATION.sub(" ", text)))
|
|
|
|
|
|
def citations_in(text: str) -> tuple[int, ...]:
|
|
return tuple(int(marker) for marker in _CITATION.findall(text))
|
|
|
|
|
|
def verify(answer: str, evidence_texts: tuple[str, ...]) -> GroundingReport:
|
|
"""Whether `answer` states only figures and sources present in evidence.
|
|
|
|
`evidence_texts` is positional: citation `[n]` refers to
|
|
`evidence_texts[n - 1]`, so an out-of-range marker is a defect even when
|
|
the prose around it is faithful — a citation nobody can follow is not a
|
|
citation.
|
|
"""
|
|
source_numbers = set()
|
|
for text in evidence_texts:
|
|
source_numbers.update(numbers_in(text))
|
|
|
|
unsupported = tuple(
|
|
token for token in numbers_in(answer) if token not in source_numbers
|
|
)
|
|
invalid = tuple(
|
|
index
|
|
for index in citations_in(answer)
|
|
if not 1 <= index <= len(evidence_texts)
|
|
)
|
|
cited = tuple(sorted({index for index in citations_in(answer)} - set(invalid)))
|
|
|
|
return GroundingReport(
|
|
grounded=not unsupported and not invalid,
|
|
unsupported_numbers=unsupported,
|
|
invalid_citations=invalid,
|
|
cited_indices=cited,
|
|
)
|