"""Checks a generated answer against the evidence it was built from. The answer layer may only rephrase retrieved text. This module is what makes that a checkable property rather than a promise in a prompt: it recomputes, from the evidence alone, whether every number and every citation in a generated answer can be traced back to the source. A generation that fails is discarded, never shown. Numbers are compared **character for character**, deliberately. "7,5" and "7.5" are not treated as equal, and no attempt is made to parse either into a quantity. Parsing invites the one error that matters most here: `1.500` is 1500 under one reading and 1.5 under another, and a normaliser that strips separators maps "7,5" and "75" to the same key — a tenfold dose error scored as a match. The model is told to copy figures verbatim, so an exact match is achievable, and every deviation from it is refused rather than interpreted. """ from __future__ import annotations import re from dataclasses import dataclass # A digit run with internal separators kept: "500", "7,5", "1.000". # Ranges ("4 - 6 giờ") yield two tokens, and each is checked on its own. _NUMBER = re.compile(r"\d+(?:[.,]\d+)*") # Citation markers are stripped before number extraction so that "[2]" is # never mistaken for the quantity 2. _CITATION = re.compile(r"\[(\d+)\]") @dataclass(frozen=True) class GroundingReport: grounded: bool unsupported_numbers: tuple[str, ...] invalid_citations: tuple[int, ...] cited_indices: tuple[int, ...] @property def reason(self) -> str: if self.unsupported_numbers: return "ungrounded_number" if self.invalid_citations: return "invalid_citation" return "grounded" def numbers_in(text: str) -> tuple[str, ...]: """Numeric tokens, with citation markers removed first.""" return tuple(_NUMBER.findall(_CITATION.sub(" ", text))) def citations_in(text: str) -> tuple[int, ...]: return tuple(int(marker) for marker in _CITATION.findall(text)) def verify(answer: str, evidence_texts: tuple[str, ...]) -> GroundingReport: """Whether `answer` states only figures and sources present in evidence. `evidence_texts` is positional: citation `[n]` refers to `evidence_texts[n - 1]`, so an out-of-range marker is a defect even when the prose around it is faithful — a citation nobody can follow is not a citation. """ source_numbers = set() for text in evidence_texts: source_numbers.update(numbers_in(text)) unsupported = tuple( token for token in numbers_in(answer) if token not in source_numbers ) invalid = tuple( index for index in citations_in(answer) if not 1 <= index <= len(evidence_texts) ) cited = tuple(sorted({index for index in citations_in(answer)} - set(invalid))) return GroundingReport( grounded=not unsupported and not invalid, unsupported_numbers=unsupported, invalid_citations=invalid, cited_indices=cited, )