VuQuangBao/duocthu
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Checkpoint frontend UI/UX overhaul and ingestion embed benchmark work

Browse Source
This commit is contained in:
BaoVu2k4
2026-08-06 17:21:21 +07:00
parent 1e8cbdb586
commit a4b8e1c4db
78 changed files with 6761 additions and 654 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/ai-service/rag/answer.py
+185 -23
View File
@@ -8,7 +8,7 @@ from . import grounding, metrics as metric_names
from .metrics import Metrics, NullMetrics
from .models import EvidenceDecision, QueryIntent, RetrievalResult, SubjectScope
from .ports import AnswerGenerationUnavailable, AnswerGenerator
from .prompt import build_request
from .prompt import build_entailment_request, build_request, build_sufficiency_request
from .routing import QueryRoutingService
@@ -30,17 +30,38 @@ class GroundedAnswer:
answer: str | None
citations: tuple[Citation, ...] = ()
generated: bool = False
# Set when the model decided the turn is under-specified and asked back
# (e.g. a dose question with no age/weight). The answer field carries the
# question; the caller renders it as a clarification, not a final answer.
clarification: str | None = None
@dataclass(frozen=True)
class _GenOutcome:
answer: str | None = None
clarification: str | None = None
class GroundedAnswerService:
"""Retrieval decides what is true; generation only decides how it reads.
When a generator is configured, its output replaces the extractive text
**only** if `grounding.verify` confirms every figure and citation in it
traces back to the retrieved evidence. Anything else — an unsupported
number, a citation to nothing, a provider outage, malformed output — falls
back to quoting the source verbatim, which is always available because it
was computed first.
Two operating modes, not to be confused with each other:
- **No generator configured** (`generator=None`, e.g. `ANSWER_PROVIDER=
disabled`) is retrieval-only mode, a deliberate and fully supported
way to run this service. It quotes the retrieved source verbatim.
- **A generator IS configured.** Its output replaces the extractive text
only if it clears two independent checks: `grounding.verify` (every
figure and citation traces to the specific evidence block it cites,
and every claim carries one) and `_verify_entailment` (a second LLM
pass confirming each cited claim's *content* — not just its numbers —
is actually stated by that block). If a configured generation fails
any check, or the provider itself is unreachable, or its output is
malformed, the turn **abstains** (`reason="generation_unavailable"`
or the specific `grounding.verify` reason) rather than silently
degrading to a raw source dump — this product is a real LLM chatbot,
and a citation-stapled paragraph of book text is not an acceptable
stand-in for an answer the model was supposed to produce.
"""
def __init__(
@@ -70,6 +91,15 @@ class GroundedAnswerService:
)
else:
result = self._routing.retrieve(query, subject_scope, intent)
return self.answer_from_result(query, result)
def answer_from_result(
self, query: str, result: RetrievalResult
) -> GroundedAnswer:
"""Everything after retrieval — grounding, sufficiency, generation,
citations. Split out so the new understanding-driven orchestrator
(`rag/agent.py`) reuses the safe answer path without going through the
old `QueryRoutingService` text resolution."""
if result.decision == EvidenceDecision.ABSTAIN:
self._metrics.increment(metric_names.ABSTENTION, reason=result.reason)
return GroundedAnswer(result, None)
@@ -102,25 +132,62 @@ class GroundedAnswerService:
f"{text} [{index}]" for index, text in enumerate(evidence_texts, start=1)
)
generated = self._generate(query, evidence_texts, intro=result.is_drug_overview)
answer_text = extractive if generated is None else generated
# Reasoning step BEFORE answering: if the turn is under-specified (a dose
# with several bands and no age/weight/condition), ask instead of dumping.
# A separate focused call is more reliable than folding it into generation.
clarify_q = self._check_sufficiency(query, evidence_texts, result.is_drug_overview)
if clarify_q is not None:
return GroundedAnswer(result, clarify_q, (), clarification=clarify_q)
outcome = self._generate(query, evidence_texts, intro=result.is_drug_overview)
if outcome.clarification is not None:
# The model judged the turn under-specified (a dose with no
# age/weight/renal-function/indication…) and asked back instead of
# listing every band. Return the question, not the whole section.
return GroundedAnswer(
result, outcome.clarification, (), clarification=outcome.clarification
)
if outcome.answer is None:
if self._generator is None:
# No generator configured at all — retrieval-only mode. A
# deliberate operating mode (e.g. ANSWER_PROVIDER=disabled),
# not a failure, so the source is quoted verbatim.
self._metrics.increment(metric_names.ANSWER_EXTRACTIVE)
citations = self._cited_only(indexed, extractive) or all_citations
return GroundedAnswer(result, extractive, citations)
# A generator WAS configured and this specific generation did not
# clear the safety checks (provider outage, malformed output, an
# ungrounded/uncited/unsupported claim). This product is a real
# LLM chatbot, not the retired offline-extractive build — a raw
# source dump is not an acceptable stand-in for a failed
# generation, so this abstains instead of silently degrading to
# one.
self._metrics.increment(
metric_names.ABSTENTION, reason="generation_unavailable"
)
return GroundedAnswer(
replace(
result,
decision=EvidenceDecision.ABSTAIN,
reason="generation_unavailable",
),
None,
)
# Show only the sources the answer actually cited, not every chunk that
# was retrieved — a paragraph that cites [4] must not drag 13 citation
# chips onto the screen. Falls back to all when the text cites nothing.
citations = self._cited_only(indexed, answer_text) or all_citations
if generated is None:
self._metrics.increment(metric_names.ANSWER_EXTRACTIVE)
return GroundedAnswer(result, extractive, citations)
citations = self._cited_only(indexed, outcome.answer) or all_citations
self._metrics.increment(metric_names.GENERATION_SERVED)
return GroundedAnswer(result, generated, citations, generated=True)
return GroundedAnswer(result, outcome.answer, citations, generated=True)
def _generate(
self, query: str, evidence_texts: tuple[str, ...], intro: bool = False
) -> str | None:
"""A verified generation, or None to fall back to the source text."""
) -> "_GenOutcome":
"""A verified generation, a clarifying question, or empty to fall back."""
if self._generator is None or not evidence_texts:
return None
return _GenOutcome()
request = build_request(query, evidence_texts, intro=intro)
try:
@@ -129,7 +196,7 @@ class GroundedAnswerService:
self._metrics.increment(
metric_names.GENERATION_REJECTED, reason="provider_unavailable"
)
return None
return _GenOutcome()
try:
payload = json.loads(raw)
@@ -139,28 +206,123 @@ class GroundedAnswerService:
self._metrics.increment(
metric_names.GENERATION_REJECTED, reason="malformed_output"
)
return None
return _GenOutcome()
# The model asked for a missing detail (age/weight/renal function/
# indication…) instead of listing every band. A clarify is not a grounded
# claim, so it skips the number check — it states no dose.
clarify = payload.get("clarifying_question") if isinstance(payload, dict) else None
if isinstance(clarify, str) and clarify.strip():
return _GenOutcome(clarification=clarify.strip())
if not isinstance(answer, str) or not isinstance(sufficient, bool):
self._metrics.increment(
metric_names.GENERATION_REJECTED, reason="malformed_output"
)
return None
return _GenOutcome()
if not sufficient:
# The model says the evidence does not answer the question. Showing
# the retrieved section verbatim lets the clinician judge that.
self._metrics.increment(
metric_names.GENERATION_REJECTED, reason="evidence_insufficient"
)
return None
return _GenOutcome()
report = grounding.verify(answer, evidence_texts)
if not report.grounded:
self._metrics.increment(
metric_names.GENERATION_REJECTED, reason=report.reason
)
return _GenOutcome()
if not self._verify_entailment(answer, evidence_texts):
self._metrics.increment(
metric_names.GENERATION_REJECTED, reason="unsupported_claim"
)
return _GenOutcome()
return _GenOutcome(answer=answer)
def _verify_entailment(self, answer: str, evidence_texts: tuple[str, ...]) -> bool:
"""A second, adversarial LLM pass over an answer that already passed
`grounding.verify`.
The regex check above only binds numbers and citation indices — it
has no notion of meaning, so "Metformin chữa ung thư [1]" citing an
evidence block about "điều trị đái tháo đường" sails through it
untouched: right drug, right citation shape, fabricated indication.
This call is what catches that: each substantive, validly-cited claim
is checked against only the evidence block(s) it names, by a model
told to compare wording, not to reason about medicine.
Fails closed on an outage or malformed output. A single rejection is
NOT: live probing (2026-08-06) found the judge call itself is noisy
— the identical claim/evidence pair, called three times, came back
entailed twice and rejected once, discarding a correct, well-cited
interaction answer. So a reject triggers one same-claim retry, and
only a second, agreeing reject discards the generation; a single
provider outage/malformed response still fails closed immediately
(that failure mode is reliable, not noisy — no retry needed there).
An answer with no claim text at all (nothing between or after its
citation markers) is vacuously fine — nothing to verify, no call.
"""
claims = [
(claim.text, "\n".join(evidence_texts[i - 1] for i in claim.indices))
for claim in grounding.split_claims(answer, len(evidence_texts))
if claim.indices and grounding.has_content(claim.text)
]
if not claims:
return True
request = build_entailment_request(claims)
first = self._run_entailment_check(request)
if first is None:
return False
if first:
return True
second = self._run_entailment_check(request)
return bool(second)
def _run_entailment_check(self, request) -> bool | None:
"""One entailment call. `None` = outage/malformed (fails closed by the
caller without a retry); `True`/`False` = the judge's verdict."""
try:
raw = self._generator.generate(request.system, request.user, request.schema)
except AnswerGenerationUnavailable:
return None
return answer
try:
payload = json.loads(raw)
entailed = payload["entailed"]
unsupported = payload["unsupported"]
except (ValueError, TypeError, KeyError):
return None
if not isinstance(entailed, bool) or not isinstance(unsupported, list):
return None
return entailed and not unsupported
def _check_sufficiency(
self, query: str, evidence_texts: tuple[str, ...], intro: bool = False
) -> str | None:
"""A focused reasoning call: is the turn specific enough to answer, or
must we ask? Returns a clarifying question, or None to proceed.
Skipped without a model, for a bare-name intro (not a dose), or for a
single evidence block (nothing to disambiguate)."""
if self._generator is None or intro or len(evidence_texts) < 2:
return None
request = build_sufficiency_request(query, evidence_texts)
try:
raw = self._generator.generate(request.system, request.user, request.schema)
except AnswerGenerationUnavailable:
return None
try:
payload = json.loads(raw)
except (ValueError, TypeError):
return None
if isinstance(payload, dict) and payload.get("sufficient") is False:
question = payload.get("clarifying_question")
if isinstance(question, str) and question.strip():
return question.strip()
return None
@staticmethod
def _cited_only(