Wire the guarded conversational RAG answer layer end-to-end

apps/ai-service/rag/grounding.py

@@ -0,0 +1,83 @@
+"""Checks a generated answer against the evidence it was built from.
+
+The answer layer may only rephrase retrieved text. This module is what makes
+that a checkable property rather than a promise in a prompt: it recomputes,
+from the evidence alone, whether every number and every citation in a
+generated answer can be traced back to the source. A generation that fails is
+discarded, never shown.
+
+Numbers are compared **character for character**, deliberately. "7,5" and
+"7.5" are not treated as equal, and no attempt is made to parse either into a
+quantity. Parsing invites the one error that matters most here: `1.500` is
+1500 under one reading and 1.5 under another, and a normaliser that strips
+separators maps "7,5" and "75" to the same key — a tenfold dose error scored
+as a match. The model is told to copy figures verbatim, so an exact match is
+achievable, and every deviation from it is refused rather than interpreted.
+"""
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+# A digit run with internal separators kept: "500", "7,5", "1.000".
+# Ranges ("4 - 6 giờ") yield two tokens, and each is checked on its own.
+_NUMBER = re.compile(r"\d+(?:[.,]\d+)*")
+
+# Citation markers are stripped before number extraction so that "[2]" is
+# never mistaken for the quantity 2.
+_CITATION = re.compile(r"\[(\d+)\]")
+
+
+@dataclass(frozen=True)
+class GroundingReport:
+    grounded: bool
+    unsupported_numbers: tuple[str, ...]
+    invalid_citations: tuple[int, ...]
+    cited_indices: tuple[int, ...]
+
+    @property
+    def reason(self) -> str:
+        if self.unsupported_numbers:
+            return "ungrounded_number"
+        if self.invalid_citations:
+            return "invalid_citation"
+        return "grounded"
+
+
+def numbers_in(text: str) -> tuple[str, ...]:
+    """Numeric tokens, with citation markers removed first."""
+    return tuple(_NUMBER.findall(_CITATION.sub(" ", text)))
+
+
+def citations_in(text: str) -> tuple[int, ...]:
+    return tuple(int(marker) for marker in _CITATION.findall(text))
+
+
+def verify(answer: str, evidence_texts: tuple[str, ...]) -> GroundingReport:
+    """Whether `answer` states only figures and sources present in evidence.
+
+    `evidence_texts` is positional: citation `[n]` refers to
+    `evidence_texts[n - 1]`, so an out-of-range marker is a defect even when
+    the prose around it is faithful — a citation nobody can follow is not a
+    citation.
+    """
+    source_numbers = set()
+    for text in evidence_texts:
+        source_numbers.update(numbers_in(text))
+
+    unsupported = tuple(
+        token for token in numbers_in(answer) if token not in source_numbers
+    )
+    invalid = tuple(
+        index
+        for index in citations_in(answer)
+        if not 1 <= index <= len(evidence_texts)
+    )
+    cited = tuple(sorted({index for index in citations_in(answer)} - set(invalid)))
+
+    return GroundingReport(
+        grounded=not unsupported and not invalid,
+        unsupported_numbers=unsupported,
+        invalid_citations=invalid,
+        cited_indices=cited,
+    )