VuQuangBao/duocthu
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add production condition retrieval smoke test

Browse Source
This commit is contained in:
BaoVu2k4
2026-08-11 14:58:28 +07:00
parent 59e6ad2d0d
commit 7ebbe1f309
38 changed files with 3752 additions and 121 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/ai-service/rag/answer.py
+133 -17
View File
@@ -69,6 +69,11 @@ class Citation:
# handed to the generator/entailment checks, so the UI can show precisely
# what was retrieved rather than a fabricated summary of it.
evidence_text: str = ""
drug_id: str | None = None
drug_name: str | None = None
section_key: str | None = None
section_title: str | None = None
source_document: str = "Dược thư Quốc gia Việt Nam 2018"
@dataclass(frozen=True)
@@ -192,6 +197,7 @@ class _RawAttempt:
outage: bool = False
budget_exhausted: bool = False
malformed: bool = False
unsupported_drug: bool = False
@dataclass(frozen=True)
@@ -201,7 +207,9 @@ class _VerificationOutcome:
missing: tuple[str, ...] = ()
def _parse_claims(raw_claims: list) -> tuple[tuple[str, tuple[int, ...]], ...] | None:
def _parse_claims(
raw_claims: list, *, include_drug_label: bool = False
) -> tuple[tuple[str, tuple[int, ...]], ...] | None:
"""Validate the model's `claims` array into `(text, citation_indices)`
pairs, or `None` on any malformed entry — same fail-closed contract as
every other shape check in `_attempt_generation`."""
@@ -217,13 +225,47 @@ def _parse_claims(raw_claims: list) -> tuple[tuple[str, tuple[int, ...]], ...] |
isinstance(c, int) and not isinstance(c, bool) for c in citations
):
return None
claims.append((text.strip(), tuple(citations)))
cleaned = text.strip()
if include_drug_label:
drug_id = item.get("drug_id")
if not isinstance(drug_id, str) or not drug_id.strip():
return None
label = drug_id.replace("_", " ").upper()
if label.casefold() not in cleaned.casefold():
cleaned = f"{label}: {cleaned}"
claims.append((cleaned, tuple(citations)))
return tuple(claims)
def _candidate_claims_are_valid(
raw_claims: list,
candidate_drug_ids: tuple[str, ...],
evidence_drug_ids: tuple[str | None, ...],
) -> bool:
"""Deterministic generated-candidate subset and citation binding check."""
if not candidate_drug_ids:
return True
allowed = set(candidate_drug_ids)
for item in raw_claims:
if not isinstance(item, dict):
return False
drug_id = item.get("drug_id")
citations = item.get("citations")
if drug_id not in allowed or not isinstance(citations, list) or not citations:
return False
for citation in citations:
if (
not isinstance(citation, int)
or isinstance(citation, bool)
or not 1 <= citation <= len(evidence_drug_ids)
or evidence_drug_ids[citation - 1] != drug_id
):
return False
return True
def _prompt_evidence_texts(evidence: tuple) -> tuple[str, ...]:
"""Evidence text as shown to the generator/entailment judge — labeled with
its source drug ONLY when the evidence set spans more than one drug.
"""Evidence shown to generation/entailment with trusted source metadata.
Found live 2026-08-10: a drug interaction section routinely refers to the
drug it belongs to by pharmacological class rather than by name (e.g.
@@ -238,15 +280,20 @@ def _prompt_evidence_texts(evidence: tuple) -> tuple[str, ...]:
always writes `{drug_id}__{section}__{n}`, so this is not a per-drug
special case) restores that anchor without asking the judge to reason
about pharmacology — it only has to match a name already handed to it.
Single-drug evidence sets are left unlabeled: nothing there was
ambiguous, and every token here is spent on every call this product
makes, so it is not added where the measured bug does not apply.
The same anchor is needed for a single monograph: its interaction or
contraindication prose can use only the pharmacological class while the
answer correctly names the drug from metadata. Label every block so that
naming that source drug is not mistaken for an invented clinical fact.
"""
drug_ids = [item.matched_doc_id.split("__", 1)[0] for item in evidence]
if len(set(drug_ids)) < 2:
return tuple(item.text for item in evidence)
drug_ids = [
item.drug_id or item.matched_doc_id.split("__", 1)[0]
for item in evidence
]
return tuple(
f"(Nguồn: chuyên luận {drug_id.replace('_', ' ').upper()}) {item.text}"
(
f"(drug_id={drug_id}; thuốc={item.drug_name or drug_id.replace('_', ' ')}; "
f"mục={item.section_title or item.section_key or 'không rõ'}) {item.text}"
)
for drug_id, item in zip(drug_ids, evidence, strict=True)
)
@@ -480,6 +527,8 @@ class GroundedAnswerService:
def answer_from_result(
self, query: str, result: RetrievalResult, list_mode: bool = False,
patient_specific: bool = False,
candidate_drug_ids: tuple[str, ...] = (),
budget: RequestBudget | None = None,
prechecked: bool = False,
) -> GroundedAnswer:
@@ -554,10 +603,36 @@ class GroundedAnswerService:
outcome = self._generate(
query, evidence_texts, prompt_evidence_texts,
intro=result.is_drug_overview, list_mode=list_mode, budget=budget,
intro=result.is_drug_overview,
list_mode=list_mode,
patient_specific=patient_specific,
candidate_drug_ids=(candidate_drug_ids if list_mode else ()),
evidence_drug_ids=tuple(
item.drug_id or item.matched_doc_id.split("__", 1)[0]
for item in result.evidence
),
budget=budget,
plan=plan,
)
if outcome.clarification is not None:
if patient_specific:
# A patient-list clarification can smuggle an uncited negative
# corpus claim ("Dược thư không nêu tương tác...") through the
# branch that deliberately skips grounding because ordinary
# input questions contain no clinical assertion. Fail closed;
# the structured candidate statuses carry missing-evidence
# state without inventing a medical conclusion.
self._metrics.increment(
metric_names.ABSTENTION, reason="evidence_insufficient"
)
return GroundedAnswer(
replace(
result,
decision=EvidenceDecision.ABSTAIN,
reason="evidence_insufficient",
),
None,
)
# The model judged the turn under-specified (a dose with no
# age/weight/renal-function/indication…) and asked back instead of
# listing every band. Return the question, not the whole section.
@@ -640,7 +715,12 @@ class GroundedAnswerService:
)
def _attempt_generation(
self, request: "GenerationRequest", budget: RequestBudget | None
self,
request: "GenerationRequest",
budget: RequestBudget | None,
*,
candidate_drug_ids: tuple[str, ...] = (),
evidence_drug_ids: tuple[str | None, ...] = (),
) -> "_RawAttempt":
"""One raw generation call, parsed but not yet metric-counted or
verified — the caller decides whether to retry before charging a
@@ -675,8 +755,14 @@ class GroundedAnswerService:
return _RawAttempt(malformed=True)
if not sufficient:
return _RawAttempt(insufficient=True)
if not _candidate_claims_are_valid(
raw_claims, candidate_drug_ids, evidence_drug_ids
):
return _RawAttempt(unsupported_drug=True)
claims = _parse_claims(raw_claims)
claims = _parse_claims(
raw_claims, include_drug_label=bool(candidate_drug_ids)
)
if claims is None:
return _RawAttempt(malformed=True)
return _RawAttempt(answer=_assemble_answer(claims), claims=claims)
@@ -689,6 +775,9 @@ class GroundedAnswerService:
*,
intro: bool = False,
list_mode: bool = False,
patient_specific: bool = False,
candidate_drug_ids: tuple[str, ...] = (),
evidence_drug_ids: tuple[str | None, ...] = (),
budget: RequestBudget | None = None,
plan: AnswerPlan | None = None,
) -> "_GenOutcome":
@@ -708,8 +797,15 @@ class GroundedAnswerService:
reasoning_mode=plan.reasoning_mode,
show_heading=plan.show_heading,
needs_warning=plan.needs_warning,
patient_specific=patient_specific,
candidate_drug_ids=candidate_drug_ids,
)
attempt = self._attempt_generation(
request,
budget,
candidate_drug_ids=candidate_drug_ids,
evidence_drug_ids=evidence_drug_ids,
)
attempt = self._attempt_generation(request, budget)
if attempt.insufficient:
# Empirically noisy (found live 2026-08-07, reproduced 3/3 on a
# fresh retry): the model's own evidence_sufficient=false
@@ -719,7 +815,12 @@ class GroundedAnswerService:
# its own noisy judge call. Only the terminal "insufficient AND
# no clarifying question" case retries; a legitimate ask-for-
# more-detail clarify is untouched.
attempt = self._attempt_generation(request, budget)
attempt = self._attempt_generation(
request,
budget,
candidate_drug_ids=candidate_drug_ids,
evidence_drug_ids=evidence_drug_ids,
)
if attempt.budget_exhausted:
self._metrics.increment(
@@ -736,6 +837,11 @@ class GroundedAnswerService:
metric_names.GENERATION_REJECTED, reason="malformed_output"
)
return _GenOutcome(reject_reason="malformed_output")
if attempt.unsupported_drug:
self._metrics.increment(
metric_names.GENERATION_REJECTED, reason="unsupported_drug"
)
return _GenOutcome(reject_reason="unsupported_drug")
if attempt.clarification is not None:
return _GenOutcome(
clarification=attempt.clarification,
@@ -791,7 +897,12 @@ class GroundedAnswerService:
),
schema=request.schema,
)
repaired = self._attempt_generation(repair_request, budget)
repaired = self._attempt_generation(
repair_request,
budget,
candidate_drug_ids=candidate_drug_ids,
evidence_drug_ids=evidence_drug_ids,
)
# The repair roughly doubles a turn's model calls, so it is the
# most likely place to run out of wall-clock budget. Observed
# live 2026-08-11 (Isosorbid dinitrat dosage, 40.3s against a 40s
@@ -1051,5 +1162,10 @@ class GroundedAnswerService:
# structured page/bbox fields is enough to render later.
attachment=source.source_crop or source.block_id,
evidence_text=evidence.text,
drug_id=evidence.drug_id,
drug_name=evidence.drug_name,
section_key=evidence.section_key,
section_title=evidence.section_title,
source_document=evidence.source_document,
)))
return citations