Checkpoint frontend UI/UX overhaul and ingestion embed benchmark work

apps/ai-service/rag/grounding.py

@@ -2,10 +2,25 @@
 
 The answer layer may only rephrase retrieved text. This module is what makes
 that a checkable property rather than a promise in a prompt: it recomputes,
-from the evidence alone, whether every number and every citation in a
-generated answer can be traced back to the source. A generation that fails is
+from the evidence alone, whether every claim in a generated answer traces
+back to the specific source block it cites. A generation that fails is
 discarded, never shown.
 
+Binding is per-citation, not global. The answer is split at each citation
+marker group (one or more consecutive `[n]`); the text immediately before a
+group is that group's claim, and only the evidence block(s) named in that
+group may support it. A number that is true of evidence block 2 does not
+make a claim citing `[1]` grounded — the old implementation pooled every
+number from every evidence block into one set, which let a number attributed
+to the wrong source pass silently. `evidence_texts` is positional: `[n]`
+refers to `evidence_texts[n - 1]`.
+
+A claim with no valid citation group is rejected outright — a citation
+nobody can follow is not a citation, and an uncited clinical statement is not
+verifiable, numeric or not. This catches a missing-citation defect that the
+old check never looked for at all (it only ever checked numbers already
+carrying a marker).
+
 Numbers are compared **character for character**, deliberately. "7,5" and
 "7.5" are not treated as equal, and no attempt is made to parse either into a
 quantity. Parsing invites the one error that matters most here: `1.500` is
@@ -13,6 +28,13 @@ quantity. Parsing invites the one error that matters most here: `1.500` is
 separators maps "7,5" and "75" to the same key — a tenfold dose error scored
 as a match. The model is told to copy figures verbatim, so an exact match is
 achievable, and every deviation from it is refused rather than interpreted.
+
+What this module still cannot do: confirm that a citation-bearing nonnumeric
+claim is actually *entailed* by the block it cites (e.g. "chữa ung thư [1]"
+where evidence 1 is only about "điều trị đái tháo đường" — same drug name,
+unrelated indication). Regex-level number/citation checking has no notion of
+semantic content. That gap is closed separately by an LLM entailment pass
+(`rag/answer.py`'s post-generation verifier call), not by this module.
 """
 from __future__ import annotations
 
@@ -27,12 +49,23 @@ _NUMBER = re.compile(r"\d+(?:[.,]\d+)*")
 # never mistaken for the quantity 2.
 _CITATION = re.compile(r"\[(\d+)\]")
 
+# One or more consecutive markers ("[1]", "[1][2]") count as a single group:
+# the prompt allows citing more than one source for one claim, and each is
+# checked against the union of just those sources, not all evidence.
+_CITATION_GROUP = re.compile(r"(?:\[\d+\])+")
+
+# Any word character — letter (Vietnamese diacritics included) or digit —
+# used to tell "claim with actual content" apart from bare punctuation or
+# whitespace trailing a citation, which needs no citation of its own.
+_LETTER = re.compile(r"\w", re.UNICODE)
+
 
 @dataclass(frozen=True)
 class GroundingReport:
     grounded: bool
     unsupported_numbers: tuple[str, ...]
     invalid_citations: tuple[int, ...]
+    uncited_claim: bool
     cited_indices: tuple[int, ...]
 
     @property
@@ -41,6 +74,8 @@ class GroundingReport:
             return "ungrounded_number"
         if self.invalid_citations:
             return "invalid_citation"
+        if self.uncited_claim:
+            return "uncited_claim"
         return "grounded"
 
 
@@ -53,31 +88,90 @@ def citations_in(text: str) -> tuple[int, ...]:
     return tuple(int(marker) for marker in _CITATION.findall(text))
 
 
-def verify(answer: str, evidence_texts: tuple[str, ...]) -> GroundingReport:
-    """Whether `answer` states only figures and sources present in evidence.
+def has_content(text: str) -> bool:
+    """True once `text` carries any letter or digit — i.e. more than
+    punctuation or whitespace left over between/after citation markers."""
+    return _LETTER.search(text) is not None
 
-    `evidence_texts` is positional: citation `[n]` refers to
-    `evidence_texts[n - 1]`, so an out-of-range marker is a defect even when
-    the prose around it is faithful — a citation nobody can follow is not a
-    citation.
+
+@dataclass(frozen=True)
+class Claim:
+    """One citation-bounded segment of an answer: the text before a citation
+    group, and the (in-range) evidence indices that group names.
+
+    `indices` is empty for the trailing segment after the last citation
+    group, or for a claim whose only marker(s) were out of range — in both
+    cases there is no evidence block left to check the claim against.
     """
-    source_numbers = set()
-    for text in evidence_texts:
-        source_numbers.update(numbers_in(text))
 
-    unsupported = tuple(
-        token for token in numbers_in(answer) if token not in source_numbers
-    )
-    invalid = tuple(
-        index
-        for index in citations_in(answer)
-        if not 1 <= index <= len(evidence_texts)
-    )
-    cited = tuple(sorted({index for index in citations_in(answer)} - set(invalid)))
+    text: str
+    indices: tuple[int, ...]
+
+
+def split_claims(answer: str, evidence_count: int) -> tuple[Claim, ...]:
+    """The claim segmentation `verify` checks numbers against, exposed so a
+    semantic entailment pass can run the same per-claim binding — each claim
+    checked only against the evidence block(s) it actually cites, never the
+    whole evidence set.
+    """
+    claims: list[Claim] = []
+    cursor = 0
+    for group in _CITATION_GROUP.finditer(answer):
+        text = answer[cursor:group.start()]
+        cursor = group.end()
+        indices = tuple(
+            i for i in citations_in(group.group(0)) if 1 <= i <= evidence_count
+        )
+        claims.append(Claim(text, indices))
+    claims.append(Claim(answer[cursor:], ()))
+    return tuple(claims)
+
+
+def verify(answer: str, evidence_texts: tuple[str, ...]) -> GroundingReport:
+    """Whether `answer` states only figures and sources traceable to the
+    specific evidence block(s) cited immediately after each claim.
+
+    See module docstring for the binding rule and its known limit (no
+    semantic entailment check).
+    """
+    unsupported: list[str] = []
+    invalid: list[int] = []
+    uncited = False
+    cited_all: set[int] = set()
+
+    cursor = 0
+    for group in _CITATION_GROUP.finditer(answer):
+        claim = answer[cursor:group.start()]
+        cursor = group.end()
+
+        indices = citations_in(group.group(0))
+        bad = [i for i in indices if not 1 <= i <= len(evidence_texts)]
+        good = [i for i in indices if i not in bad]
+        invalid.extend(bad)
+        cited_all.update(good)
+
+        claim_numbers = numbers_in(claim)
+        if good:
+            source_numbers: set[str] = set()
+            for index in good:
+                source_numbers.update(numbers_in(evidence_texts[index - 1]))
+            unsupported.extend(n for n in claim_numbers if n not in source_numbers)
+        else:
+            # Every marker in this group was out of range: nothing to bind
+            # the claim to, numeric or not.
+            unsupported.extend(claim_numbers)
+            if has_content(claim):
+                uncited = True
+
+    tail = answer[cursor:]
+    unsupported.extend(numbers_in(tail))
+    if has_content(tail):
+        uncited = True
 
     return GroundingReport(
-        grounded=not unsupported and not invalid,
-        unsupported_numbers=unsupported,
-        invalid_citations=invalid,
-        cited_indices=cited,
+        grounded=not unsupported and not invalid and not uncited,
+        unsupported_numbers=tuple(unsupported),
+        invalid_citations=tuple(invalid),
+        uncited_claim=uncited,
+        cited_indices=tuple(sorted(cited_all)),
     )