Fix live multi-turn: pass the resolved drug, stop did-you-mean garbage

apps/ai-service/rag/answer.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import json
+import re
 from dataclasses import dataclass, replace
 
 from . import grounding, metrics as metric_names
@@ -57,14 +58,24 @@ class GroundedAnswerService:
         query: str,
         subject_scope: SubjectScope,
         intent: QueryIntent,
+        drug_id: str | None = None,
     ) -> GroundedAnswer:
-        result = self._routing.retrieve(query, subject_scope, intent)
+        # When the caller already resolved the drug (e.g. the conversational
+        # layer, incl. an inherited follow-up), retrieve for it directly instead
+        # of re-resolving from the turn text — re-resolution from a rewritten
+        # turn is what abstained good follow-ups as "ambiguous".
+        if drug_id is not None:
+            result = self._routing.retrieve_for_drug(
+                query, drug_id, subject_scope, intent
+            )
+        else:
+            result = self._routing.retrieve(query, subject_scope, intent)
         if result.decision == EvidenceDecision.ABSTAIN:
             self._metrics.increment(metric_names.ABSTENTION, reason=result.reason)
             return GroundedAnswer(result, None)
 
-        citations = self._citations(result)
-        if citations is None:
+        indexed = self._indexed_citations(result)
+        if indexed is None:
             return GroundedAnswer(
                 replace(
                     result,
@@ -74,6 +85,7 @@ class GroundedAnswerService:
                 ),
                 None,
             )
+        all_citations = tuple(citation for _, citation in indexed)
         if result.decision == EvidenceDecision.VERIFY_PDF:
             # Never generated over. A quarantined table or formula is exactly
             # the evidence whose numbers were not reliably reconstructed, so
@@ -82,7 +94,7 @@ class GroundedAnswerService:
                 result,
                 "Nguồn có bảng hoặc công thức cần đối chiếu trực tiếp với ảnh PDF; "
                 "không tự động trích số liệu.",
-                citations,
+                all_citations,
             )
 
         evidence_texts = tuple(item.text for item in result.evidence)
@@ -90,7 +102,12 @@ class GroundedAnswerService:
             f"{text} [{index}]" for index, text in enumerate(evidence_texts, start=1)
         )
 
-        generated = self._generate(query, evidence_texts)
+        generated = self._generate(query, evidence_texts, intro=result.is_drug_overview)
+        answer_text = extractive if generated is None else generated
+        # Show only the sources the answer actually cited, not every chunk that
+        # was retrieved — a paragraph that cites [4] must not drag 13 citation
+        # chips onto the screen. Falls back to all when the text cites nothing.
+        citations = self._cited_only(indexed, answer_text) or all_citations
         if generated is None:
             self._metrics.increment(metric_names.ANSWER_EXTRACTIVE)
             return GroundedAnswer(result, extractive, citations)
@@ -98,12 +115,14 @@ class GroundedAnswerService:
         self._metrics.increment(metric_names.GENERATION_SERVED)
         return GroundedAnswer(result, generated, citations, generated=True)
 
-    def _generate(self, query: str, evidence_texts: tuple[str, ...]) -> str | None:
+    def _generate(
+        self, query: str, evidence_texts: tuple[str, ...], intro: bool = False
+    ) -> str | None:
         """A verified generation, or None to fall back to the source text."""
         if self._generator is None or not evidence_texts:
             return None
 
-        request = build_request(query, evidence_texts)
+        request = build_request(query, evidence_texts, intro=intro)
         try:
             raw = self._generator.generate(request.system, request.user, request.schema)
         except AnswerGenerationUnavailable:
@@ -144,9 +163,21 @@ class GroundedAnswerService:
         return answer
 
     @staticmethod
-    def _citations(result: RetrievalResult) -> tuple[Citation, ...] | None:
-        citations = []
-        for evidence in result.evidence:
+    def _cited_only(
+        indexed: list[tuple[int, Citation]], answer_text: str
+    ) -> tuple[Citation, ...]:
+        """Keep citations whose 1-based evidence marker [n] appears in the text."""
+        used = {int(m) for m in re.findall(r"\[(\d+)\]", answer_text)}
+        return tuple(citation for index, citation in indexed if index in used)
+
+    @staticmethod
+    def _indexed_citations(
+        result: RetrievalResult,
+    ) -> list[tuple[int, Citation]] | None:
+        """Citations tagged with the 1-based evidence index the prompt gives them,
+        so the response can show only the ones the answer cited."""
+        citations: list[tuple[int, Citation]] = []
+        for index, evidence in enumerate(result.evidence, start=1):
             if not evidence.source_refs:
                 return None
             for source in evidence.source_refs:
@@ -157,7 +188,7 @@ class GroundedAnswerService:
                     start = end = source.printed_page
                 else:
                     return None
-                citations.append(Citation(
+                citations.append((index, Citation(
                     chunk_id=evidence.matched_doc_id,
                     printed_page_start=int(start),
                     printed_page_end=int(end),
@@ -169,5 +200,5 @@ class GroundedAnswerService:
                     # real crop path wins; otherwise the block id plus the
                     # structured page/bbox fields is enough to render later.
                     attachment=source.source_crop or source.block_id,
-                ))
-        return tuple(citations)
+                )))
+        return citations

apps/ai-service/rag/conversational.py

@@ -46,8 +46,22 @@ from .reasoning import (
     clarify_for,
     run_turn,
 )
-from .routing import CatalogDrugResolver, DrugResolutionStatus
-from .sections import SECTION_PHRASES, SectionResolver
+from .routing import CatalogDrugResolver, DrugResolutionStatus, normalize_name
+from .sections import SectionResolver
+
+# Turns that only confirm a prior suggestion. They resolve no drug and must not
+# be fuzzy-matched against the catalog (which returns garbage like terbinafin).
+# Stored normalised (normalize_name strips diacritics: "đúng" -> "dung"), or the
+# lookup below never matches.
+_CONFIRMATION_WORDS = (
+    "đúng", "đúng rồi", "đúng vậy", "phải", "phải rồi", "chuẩn", "chuẩn rồi",
+    "chính xác", "ừ", "uh", "ok", "oke", "yes", "vâng",
+)
+_CONFIRMATIONS = frozenset(normalize_name(word) for word in _CONFIRMATION_WORDS)
+
+
+def _is_confirmation(text: str) -> bool:
+    return normalize_name(text) in _CONFIRMATIONS
 
 SUMMARY_EVERY = 4  # regenerate the summary at most every S turns, per ADR 0007 §2
 
@@ -272,11 +286,30 @@ class ConversationalLoopService:
         # a near-miss for real drug names, offer them ("did you mean") rather
         # than a bare "which drug?" — a typo should not dead-end.
         if resolved.drug_id is None:
-            # Only genuinely-close names are offered. A far match (Arginin for
-            # "metfomin") is noise, not a suggestion — so the bar is high, and
-            # when nothing clears it the honest answer is "not in the formulary",
-            # never a padded list of unrelated drugs.
-            suggestions = self._resolver.suggest(query, k=3, min_score=0.72)
+            # A bare confirmation ("đúng") with no drug in context is not a drug
+            # lookup — never fuzzy-match it (that returned terbinafin/tretinoin).
+            if _is_confirmation(query):
+                reason = "confirm_without_context"
+                clarification = Clarification(
+                    reason=reason,
+                    question="Bạn muốn xác nhận thuốc nào? Vui lòng gõ tên thuốc để mình tra cứu.",
+                    options=(),
+                )
+                self._metrics.increment(metric_names.CLARIFY_ASKED, reason=reason)
+                self._persist(state, resolved, None)
+                return ConversationTurnResult(None, clarification, None, False, None, reason)
+            # Only offer "did you mean" for a SHORT, drug-name-shaped miss (a
+            # typo). Fuzzy-matching a whole sentence ("EPO điều trị thiếu máu…")
+            # or a confirmation ("đúng") against 684 aliases returns confident
+            # garbage — that is the did-you-mean loop the reviewer hit. A long or
+            # confirming turn that resolves no drug is answered honestly, not
+            # with a list of unrelated drugs.
+            looks_like_name = len(normalize_name(query).split()) <= 4
+            suggestions = (
+                self._resolver.suggest(query, k=3, min_score=0.72)
+                if looks_like_name and not _is_confirmation(query)
+                else []
+            )
             if suggestions:
                 names = [self._drug_name(drug_id) for drug_id, _ in suggestions]
                 reason = "did_you_mean"
@@ -301,16 +334,13 @@ class ConversationalLoopService:
         if resolved.inherited_drug:
             self._metrics.increment(metric_names.FOLLOWUP_INHERITED)
 
-        # One call to the safe engine with the self-contained (rewritten) query.
-        # A multi-round retrieval-refine loop was tried and removed: refining an
-        # already-answerable whole-section result cannot fetch more (the section
-        # is complete) and, worse, the refined query drops the inherited drug and
-        # abstains — discarding a good answer. Refinement belongs to the
-        # similarity path, not here. Clarify + inheritance are the loop's value,
-        # and both happen above this line.
-        effective = self._rewrite(query, resolved)
+        # One call to the safe engine. The drug is passed already-resolved (incl.
+        # an inherited follow-up drug), so the engine does NOT re-resolve it from
+        # the turn text — that double-resolution is what abstained follow-ups as
+        # "ambiguous". The turn's own text drives section routing; when it names
+        # no attribute the drug-overview + rerank path finds the relevant part.
         grounded: GroundedAnswer | None = self._answers.answer(
-            effective, subject_scope, intent
+            query, subject_scope, intent, drug_id=resolved.drug_id
         )
 
         answer = grounded.answer if grounded else None
@@ -339,18 +369,6 @@ class ConversationalLoopService:
         """A readable display name from a drug id ('paracetamol_acetaminophen')."""
         return drug_id.replace("_", " ").title()
 
-    @staticmethod
-    def _rewrite(query: str, resolved) -> str:
-        parts: list[str] = []
-        if resolved.inherited_drug and resolved.drug_id:
-            parts.append(resolved.drug_id)
-        if resolved.inherited_section and resolved.section_key:
-            phrases = SECTION_PHRASES.get(resolved.section_key)
-            if phrases:
-                parts.append(phrases[0])
-        parts.append(query)
-        return " ".join(parts)
-
     def _append_user(self, state, text, drug_id, section_key) -> None:
         state = state.append(Turn("user", text, _now(), drug_id, section_key))
         self._store.save(state)

apps/ai-service/rag/models.py

@@ -81,3 +81,7 @@ class RetrievalResult:
     evidence: tuple[Evidence, ...] = field(default_factory=tuple)
     resolved_drug_id: str | None = None
     drug_resolution_status: str = "not_attempted"
+    # True when the user typed only a drug name (no attribute): the answer layer
+    # should introduce the drug (what it is + what it treats), not restate a
+    # section verbatim.
+    is_drug_overview: bool = False

apps/ai-service/rag/ports.py

@@ -26,6 +26,27 @@ class AnswerGenerationUnavailable(RuntimeError):
     """
 
 
+class RerankUnavailable(RuntimeError):
+    """The reranker could not be reached.
+
+    Same fail-open contract as the other provider errors, but softer: losing the
+    reranker only means the candidates keep their original order, so the caller
+    catches this and proceeds rather than abstaining. Rerank is an ordering
+    improvement on the fallback, never a precondition for an answer.
+    """
+
+
+class Reranker(Protocol):
+    """Reorders candidate texts by joint relevance to the query.
+
+    Returns indices into `documents`, most relevant first. A cross-encoder pass
+    that recovers precision the bi-encoder embedding cannot; applied only to the
+    similarity/overview fallback, never to the deterministic section route.
+    """
+
+    def rerank(self, query: str, documents: list[str], top_n: int | None = None) -> list[int]: ...
+
+
 class AnswerGenerator(Protocol):
     """Rewrites retrieved evidence into prose. Never a source of facts.
 

apps/ai-service/rag/prompt.py

@@ -24,10 +24,17 @@ Quy tắc bắt buộc:
 2. Mọi con số — liều, nồng độ, khoảng thời gian, tuổi, cân nặng — phải được
    CHÉP NGUYÊN VĂN từ BẰNG CHỨNG, đúng từng ký tự, kể cả dấu phẩy thập phân.
    Không làm tròn, không đổi đơn vị, không quy đổi.
-3. Mỗi ý phải gắn số nguồn dạng [n], với n là số thứ tự đoạn bằng chứng.
-4. Nếu BẰNG CHỨNG không đủ để trả lời, nói rõ là không đủ. Đó là câu trả lời
-   hợp lệ, không phải thất bại.
-5. Giữ nguyên thuật ngữ chuyên môn của sách. Không diễn giải cho người
+3. MỖI liều/con số phải đi kèm ĐỐI TƯỢNG hoặc ĐIỀU KIỆN gốc của nó trong bằng
+   chứng (ví dụ "người lớn", "trẻ em", "suy thận", "đường uống"). TUYỆT ĐỐI
+   không gán liều của đối tượng này cho đối tượng khác, và không gộp các liều
+   khác đối tượng thành một.
+4. Gắn số nguồn [n] cho từng ý, với n là đoạn bằng chứng THỰC SỰ chứa ý đó.
+   Chỉ trích [n] nếu đọc đoạn n thấy đúng ý đang nói. Không lặp lại cùng một
+   [n] ở mọi câu — gắn một lần cho một cụm cùng nguồn là đủ. Không bịa số [n].
+5. Nếu BẰNG CHỨNG không đủ (thiếu đối tượng được hỏi, thiếu con số, hoặc chỉ nói
+   chung chung), nói rõ là không đủ và đặt evidence_sufficient=false. Đó là câu
+   trả lời hợp lệ. Không suy diễn để lấp chỗ trống.
+6. Giữ nguyên thuật ngữ chuyên môn của sách. Không diễn giải cho người
    không chuyên.
 
 Viết gọn. Trả lời đúng điều được hỏi, không mở rộng phạm vi."""
@@ -60,12 +67,18 @@ class GenerationRequest:
     schema: dict
 
 
-def build_request(question: str, evidence_texts: tuple[str, ...]) -> GenerationRequest:
+def build_request(
+    question: str, evidence_texts: tuple[str, ...], intro: bool = False
+) -> GenerationRequest:
     """The prompt for one question over one ordered evidence list.
 
     Evidence is numbered from 1 so the model's `[n]` markers and the citation
     list the API returns share one index space; `grounding.verify` rejects any
     marker outside it.
+
+    `intro=True` is the "user typed only a drug name" case: instead of restating
+    a section, write a short introduction — what the drug is, its class and its
+    main indication — then invite a specific follow-up. Still evidence-only.
     """
     if not evidence_texts:
         raise ValueError("cannot build a grounded prompt with no evidence")
@@ -73,5 +86,15 @@ def build_request(question: str, evidence_texts: tuple[str, ...]) -> GenerationR
     blocks = "\n\n".join(
         f"[{index}] {text}" for index, text in enumerate(evidence_texts, start=1)
     )
-    user = f"BẰNG CHỨNG:\n\n{blocks}\n\nCÂU HỎI: {question}"
+    if intro:
+        task = (
+            f"Người dùng mới gõ tên thuốc: {question}. Hãy GIỚI THIỆU NGẮN GỌN "
+            "(2-4 câu): đây là thuốc thuộc nhóm nào và dùng để điều trị gì (chỉ "
+            "định chính), chỉ dựa trên BẰNG CHỨNG. KHÔNG liệt kê dạng bào chế/hàm "
+            "lượng. Kết thúc bằng một câu mời hỏi tiếp về thuộc tính cụ thể (liều "
+            "dùng, chống chỉ định, thận trọng, tương tác…)."
+        )
+    else:
+        task = f"CÂU HỎI: {question}"
+    user = f"BẰNG CHỨNG:\n\n{blocks}\n\n{task}"
     return GenerationRequest(system=SYSTEM_PROMPT, user=user, schema=ANSWER_SCHEMA)

apps/ai-service/rag/routing.py

@@ -181,12 +181,10 @@ class QueryRoutingService:
         self._retrieval = retrieval
         self._resolver = resolver
 
-    def retrieve(
-        self,
-        query: str,
-        subject_scope: SubjectScope = SubjectScope.UNKNOWN,
-        intent: QueryIntent = QueryIntent.UNKNOWN,
-    ) -> RetrievalResult:
+    @staticmethod
+    def _scope_gate(
+        subject_scope: SubjectScope, intent: QueryIntent
+    ) -> RetrievalResult | None:
         # Scope comes from the API/policy layer. Unknown is deliberately
         # fail-closed; retrieval must not infer clinical scope from keywords.
         if subject_scope == SubjectScope.NON_HUMAN:
@@ -197,6 +195,41 @@ class QueryRoutingService:
             return RetrievalResult(EvidenceDecision.ABSTAIN, "recommendation_out_of_scope")
         if intent == QueryIntent.UNKNOWN:
             return RetrievalResult(EvidenceDecision.ABSTAIN, "query_intent_unknown")
+        return None
+
+    def retrieve_for_drug(
+        self,
+        query: str,
+        drug_id: str,
+        subject_scope: SubjectScope = SubjectScope.UNKNOWN,
+        intent: QueryIntent = QueryIntent.UNKNOWN,
+    ) -> RetrievalResult:
+        """Retrieve for an ALREADY-resolved drug, skipping name resolution.
+
+        The conversational layer has already resolved (and possibly inherited)
+        the drug; re-resolving from the rewritten turn text is what produced the
+        `drug_resolution_ambiguous` empty answers on follow-ups. The query text
+        still drives section routing and the intro/overview decision.
+        """
+        gate = self._scope_gate(subject_scope, intent)
+        if gate is not None:
+            return gate
+        result = self._retrieval.retrieve(query, drug_id)
+        return replace(
+            result,
+            resolved_drug_id=drug_id,
+            drug_resolution_status=DrugResolutionStatus.RESOLVED,
+        )
+
+    def retrieve(
+        self,
+        query: str,
+        subject_scope: SubjectScope = SubjectScope.UNKNOWN,
+        intent: QueryIntent = QueryIntent.UNKNOWN,
+    ) -> RetrievalResult:
+        gate = self._scope_gate(subject_scope, intent)
+        if gate is not None:
+            return gate
         resolution = self._resolver.resolve(query)
         if resolution.status == DrugResolutionStatus.NOT_FOUND:
             return RetrievalResult(

apps/ai-service/rag/service.py

@@ -3,15 +3,35 @@ from __future__ import annotations
 from dataclasses import dataclass
 
 from .models import Evidence, EvidenceDecision, RetrievalResult, SearchHit
-from .ports import ParentStore, QueryEmbeddingUnavailable, Retriever
+from .ports import (
+    ParentStore,
+    QueryEmbeddingUnavailable,
+    Reranker,
+    RerankUnavailable,
+    Retriever,
+)
 from .sections import SectionResolver
 
 
+# The sections that introduce a drug: what it is, its class, its main use, its
+# mechanism — in book order. A bare drug name is answered from these, not from
+# the dosage-forms table that happens to sit near the top of the monograph.
+INTRO_SECTIONS = (
+    "ten_chung_quoc_te",
+    "loai_thuoc",
+    "chi_dinh",
+    "duoc_ly_va_co_che_tac_dung",
+)
+
+
 @dataclass(frozen=True)
 class EvidencePolicy:
     minimum_score: float = 0.12
     candidate_limit: int = 5
     evidence_limit: int = 3
+    # A free-form question about a resolved drug otherwise hands the LLM the
+    # entire monograph; rerank trims it to the sections that actually answer.
+    rerank_top_k: int = 6
 
 
 class RetrievalService:
@@ -30,11 +50,13 @@ class RetrievalService:
         parent_store: ParentStore,
         policy: EvidencePolicy | None = None,
         section_resolver: SectionResolver | None = None,
+        reranker: Reranker | None = None,
     ) -> None:
         self._retriever = retriever
         self._parent_store = parent_store
         self._policy = policy or EvidencePolicy()
         self._section_resolver = section_resolver
+        self._reranker = reranker
 
     def retrieve(self, query: str, drug_id: str) -> RetrievalResult:
         if not query.strip() or not drug_id.strip():
@@ -46,12 +68,23 @@ class RetrievalService:
             # truncated list of contraindications reads as a complete one.
             return self._decide(self._hydrate(section_hits, limit=None))
 
-        # Drug resolved but no attribute named ("PARACETAMOL"): show the whole
-        # monograph, in book order, rather than dead-ending on "specify an
-        # attribute". A drug reference answers a drug name with the drug.
+        # Drug resolved but no attribute named. A *bare* drug name ("PARACETAMOL")
+        # shows the whole monograph in book order. A *free-form question* about
+        # the drug ("sốt cao uống được không?") would otherwise dump all ~29
+        # sections at the model; rerank keeps only the sections that answer it.
         overview_hits = self._drug_overview(drug_id)
         if overview_hits is not None:
-            return self._decide(self._hydrate(overview_hits, limit=None))
+            if self._is_question(query):
+                overview_hits = self._rerank(query, overview_hits)
+                return self._decide(self._hydrate(overview_hits, limit=None))
+            # A bare drug name is not a question — introduce the drug from its
+            # identity sections (what it is, its class, its main indication),
+            # not the whole monograph starting with the dosage-forms table.
+            intro = [h for h in overview_hits if h.document.section_key in INTRO_SECTIONS]
+            return self._decide(
+                self._hydrate(intro or overview_hits, limit=None),
+                is_drug_overview=True,
+            )
 
         try:
             hits = self._retriever.search(
@@ -69,7 +102,33 @@ class RetrievalService:
             )
         if not hits or hits[0].score < self._policy.minimum_score:
             return RetrievalResult(EvidenceDecision.ABSTAIN, "insufficient_retrieval_score")
-        return self._decide(self._hydrate(hits))
+        return self._decide(self._hydrate(self._rerank(query, hits)))
+
+    @staticmethod
+    def _is_question(query: str) -> bool:
+        """A bare drug name (one or two tokens) wants the whole monograph; more
+        than that is a question whose overview should be reranked to the point."""
+        return len(query.split()) > 2
+
+    def _rerank(self, query: str, hits: list[SearchHit]) -> list[SearchHit]:
+        """Reorder hits by cross-encoder relevance, keep the top-k.
+
+        Fail-open: no reranker configured, or the provider is unreachable, and
+        the original order is returned unchanged — an ordering aid must never be
+        able to lose an answer. The section route never reaches this.
+        """
+        if self._reranker is None or len(hits) <= 1:
+            return hits
+        try:
+            order = self._reranker.rerank(
+                query,
+                [hit.document.text for hit in hits],
+                top_n=self._policy.rerank_top_k,
+            )
+        except RerankUnavailable:
+            return hits
+        reranked = [hits[index] for index in order if 0 <= index < len(hits)]
+        return reranked[: self._policy.rerank_top_k] or hits
 
     def _drug_overview(self, drug_id: str) -> list[SearchHit] | None:
         """Every prose section of the drug, or None if the store cannot scroll."""
@@ -96,14 +155,21 @@ class RetrievalService:
         hits = find_by_section(drug_id, match.section_key)
         return hits or None
 
-    def _decide(self, evidence: tuple[Evidence, ...]) -> RetrievalResult:
+    def _decide(
+        self, evidence: tuple[Evidence, ...], is_drug_overview: bool = False
+    ) -> RetrievalResult:
         if not evidence:
             return RetrievalResult(EvidenceDecision.ABSTAIN, "parent_hydration_failed")
         if any(not item.source_refs for item in evidence):
             return RetrievalResult(EvidenceDecision.ABSTAIN, "missing_provenance")
         if any(item.requires_visual_check for item in evidence):
             return RetrievalResult(EvidenceDecision.VERIFY_PDF, "visual_verification_required", evidence)
-        return RetrievalResult(EvidenceDecision.ANSWERABLE, "grounded_evidence_available", evidence)
+        return RetrievalResult(
+            EvidenceDecision.ANSWERABLE,
+            "grounded_evidence_available",
+            evidence,
+            is_drug_overview=is_drug_overview,
+        )
 
     def _hydrate(
         self, hits: list[SearchHit], limit: int | None = -1