diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..4657ae8 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,10 @@ +node_modules +**/node_modules +.next +**/.next +.git +**/__pycache__ +**/*.pyc +.venv +venv +Golden Dataset diff --git a/apps/ai-service/Dockerfile b/apps/ai-service/Dockerfile new file mode 100644 index 0000000..34cc11a --- /dev/null +++ b/apps/ai-service/Dockerfile @@ -0,0 +1,34 @@ +FROM python:3.12-slim + +WORKDIR /app + +RUN apt-get update \ + && apt-get install -y --no-install-recommends gcc \ + && rm -rf /var/lib/apt/lists/* + +COPY apps/ai-service/ ./ +# config.py's `entities_path` default assumes a full monorepo checkout +# (parents[2] from apps/ai-service/config.py = repo root); this image only +# has apps/ai-service flattened into /app, so the file is baked in here and +# ENTITIES_PATH in .env.prod points at it instead. +COPY ingestion/data/verified/drug_entities.json ./ingestion_data/drug_entities.json + +# Flat module layout (rag/, adapters/, routers/...), not an installable +# package — setuptools' auto-discovery rejects "multiple top-level packages" +# for `pip install .`, so the runtime deps are listed directly instead, +# mirroring pyproject.toml's [project.dependencies] + the metrics/generation +# extras + boto3 (used for Bedrock, not declared in pyproject.toml). +RUN pip install --no-cache-dir \ + "fastapi>=0.115,<1" \ + "httpx>=0.27,<1" \ + "psycopg[binary]>=3.2,<4" \ + "pydantic-settings>=2.6,<3" \ + "qdrant-client>=1.7,<2" \ + "uvicorn[standard]>=0.30,<1" \ + "prometheus-client>=0.20,<1" \ + "anthropic>=0.112,<1" \ + "boto3" + +EXPOSE 8000 + +CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/apps/ai-service/adapters/bedrock_claude.py b/apps/ai-service/adapters/bedrock_claude.py index e8552f5..2c21014 100644 --- a/apps/ai-service/adapters/bedrock_claude.py +++ b/apps/ai-service/adapters/bedrock_claude.py @@ -72,7 +72,12 @@ class BedrockClaudeAnswerGenerator: if self._client is None: from anthropic import AnthropicBedrockMantle - self._client = AnthropicBedrockMantle(aws_region=self._region) + # max_retries: the SDK's own default (2) is lower than the retry + # budget given the other Bedrock adapters (bedrock_converse.py, + # embedding.py) after a real throttling burst measured live + # 2026-08-07 — matched here for consistency, in case this + # provider is ever selected instead of bedrock-converse. + self._client = AnthropicBedrockMantle(aws_region=self._region, max_retries=4) return self._client def generate(self, system: str, user: str, schema: dict) -> str: diff --git a/apps/ai-service/adapters/bedrock_converse.py b/apps/ai-service/adapters/bedrock_converse.py index c8baed4..41bd517 100644 --- a/apps/ai-service/adapters/bedrock_converse.py +++ b/apps/ai-service/adapters/bedrock_converse.py @@ -94,7 +94,15 @@ class BedrockConverseAnswerGenerator: config=Config( connect_timeout=10, read_timeout=60, - retries={"max_attempts": 3, "mode": "standard"}, + # "adaptive" was tried 2026-08-07 and reverted same day: its + # client-side rate limiter remembers "throttled" across + # requests and paces even unrelated, otherwise-healthy calls + # down after a burst — turned a single answerable turn's + # baseline ~9s into 1-5 MINUTES following this session's own + # heavy adversarial test traffic. "standard" retries each + # call independently, no shared state to get stuck in a bad + # regime. max_attempts alone (3->4) is kept. + retries={"max_attempts": 4, "mode": "standard"}, ), ) return self._client @@ -168,7 +176,15 @@ class BedrockCohereReranker: config=Config( connect_timeout=10, read_timeout=30, - retries={"max_attempts": 3, "mode": "standard"}, + # "adaptive" was tried 2026-08-07 and reverted same day: its + # client-side rate limiter remembers "throttled" across + # requests and paces even unrelated, otherwise-healthy calls + # down after a burst — turned a single answerable turn's + # baseline ~9s into 1-5 MINUTES following this session's own + # heavy adversarial test traffic. "standard" retries each + # call independently, no shared state to get stuck in a bad + # regime. max_attempts alone (3->4) is kept. + retries={"max_attempts": 4, "mode": "standard"}, ), ) return self._client diff --git a/apps/ai-service/adapters/embedding.py b/apps/ai-service/adapters/embedding.py index 30a7331..b8e50ff 100644 --- a/apps/ai-service/adapters/embedding.py +++ b/apps/ai-service/adapters/embedding.py @@ -74,7 +74,10 @@ class BedrockCohereQueryEmbedder: config=Config( connect_timeout=10, read_timeout=30, - retries={"max_attempts": 3, "mode": "standard"}, + # "adaptive" was tried 2026-08-07 and reverted same day — see + # the matching comment in adapters/bedrock_converse.py for the + # measured 1-5-minute regression it caused. "standard" kept. + retries={"max_attempts": 4, "mode": "standard"}, ), ) return self._client diff --git a/apps/ai-service/adapters/postgres.py b/apps/ai-service/adapters/postgres.py index 6189c1f..e927fee 100644 --- a/apps/ai-service/adapters/postgres.py +++ b/apps/ai-service/adapters/postgres.py @@ -93,3 +93,59 @@ class PostgresTraceRepository: decision=row[4], reason=row[5], resolved_drug_id=row[6], citations=tuple(row[7]), created_at=row[8], ) + + +class PostgresConversationStore: + """Durable, cross-worker replacement for `RagAgent`'s in-process + `dict[str, list[str]]` history — the F-08 gap named in ADR 0008: history + was lost on restart and not shared across workers. + + Append-only, one connection per call — the same tradeoffs as + `PostgresTraceRepository` (F-09: real pooling is a further improvement + not made here) and the same `connect_timeout=5` for the same reason + (an unreachable-but-not-refusing host hangs on the OS TCP timeout + otherwise, defeating a caller's fail-open try/except just as completely + as no try/except at all). + + Raises on any error rather than swallowing it — `RagAgent` is the + caller that decides fail-open (lose this turn's memory, not the + response), matching how `routers/rag.py` already wraps + `PostgresTraceRepository.save()` rather than the repository hiding its + own failures. + """ + + def __init__(self, dsn: str) -> None: + self._dsn = dsn + + def migrate(self, migration_path: Path) -> None: + import psycopg + + statement = migration_path.read_text(encoding="utf-8") + with psycopg.connect(self._dsn, connect_timeout=5) as connection: + connection.execute(statement) + + def recent(self, conversation_id: str, limit: int) -> list[str]: + """The last `limit` lines, oldest first — matches the ordering the + understanding LLM prompt already expects ("LỊCH SỬ HỘI THOẠI (cũ -> + mới)").""" + import psycopg + + with psycopg.connect(self._dsn, connect_timeout=5) as connection: + rows = connection.execute( + """ + SELECT line FROM rag_conversation_turn + WHERE conversation_id = %s ORDER BY id DESC LIMIT %s + """, + (conversation_id, limit), + ).fetchall() + return [row[0] for row in reversed(rows)] + + def append(self, conversation_id: str, line: str) -> None: + import psycopg + + with psycopg.connect(self._dsn, connect_timeout=5) as connection: + connection.execute( + "INSERT INTO rag_conversation_turn (conversation_id, line) " + "VALUES (%s, %s)", + (conversation_id, line), + ) diff --git a/apps/ai-service/adapters/qdrant.py b/apps/ai-service/adapters/qdrant.py index e60a022..91c2353 100644 --- a/apps/ai-service/adapters/qdrant.py +++ b/apps/ai-service/adapters/qdrant.py @@ -233,6 +233,118 @@ class QdrantRetriever: hits.append(SearchHit(_document(payload), 1.0)) return hits + def find_by_indication(self, indication_text: str, limit: int) -> list[SearchHit]: + """Reverse lookup: every drug whose `chi_dinh` text mentions the given + symptom/indication, keyword-matched. Deterministic, no fabrication + risk — the same "exact match wins, no-match-means-None" philosophy + `find_by_section` already uses, applied across drugs instead of + within one. `limit` caps how many DRUGS are returned (one hit per + drug, first match wins), not how many chunks are scanned — a common + symptom can match far more drugs than is useful to show. + + Prose only: a `block_descriptor` chunk carries no real `chi_dinh` + text (its text is built only from metadata per the quarantine + contract), so keyword-matching it would be meaningless. + + `indication_text`, normalized, must appear as a CONTIGUOUS, + word-boundary-anchored phrase in the chunk's text — not a plain + substring (risks a false positive inside an unrelated longer word + after diacritic-stripping) and not a scattered bag-of-words match + either. Found live 2026-08-07: a token-SUBSET match (every word + present *somewhere*, any order) let a long nonsense phrase built + from common filler words ("bệnh chưa từng ghi nhận trong sách…") + false-positive against real chi_dinh text, since words that common + appear scattered through nearly everything — it reached generation + before being caught, instead of failing here where it's cheap. A + genuine paraphrase that doesn't share the book's exact wording is + `search_indication`'s job (semantic), not this one's (lexical). + """ + import re + + from qdrant_client.models import FieldCondition, Filter, MatchValue + + from rag.text import normalize_name + + needle = normalize_name(indication_text) + if not needle: + return [] + needle_pattern = re.compile(rf"(?:^| ){re.escape(needle)}(?:$| )") + + scroll_filter = Filter( + must=[ + FieldCondition(key="section_key", match=MatchValue(value="chi_dinh")), + FieldCondition(key="chunk_kind", match=MatchValue(value="prose")), + ] + ) + hits: list[SearchHit] = [] + seen_drugs: set[str] = set() + offset = None + while True: + points, offset = self._client.scroll( + collection_name=self._collection_name, + scroll_filter=scroll_filter, + limit=256, + offset=offset, + with_payload=True, + ) + for point in points: + payload = dict(point.payload or {}) + drug_id = payload.get("drug_id") + if drug_id in seen_drugs: + continue + text = normalize_name(payload.get("text", "")) + if not needle_pattern.search(f" {text} "): + continue + seen_drugs.add(drug_id) + hits.append(SearchHit(_document(payload), 1.0)) + if len(hits) >= limit: + return hits + if offset is None: + break + return hits + + def search_indication(self, query: str, limit: int) -> list[SearchHit]: + """Dense-vector fallback for `find_by_indication` when no exact + keyword phrase match exists — catches paraphrases ("sốt cao" vs + "thân nhiệt tăng") a literal phrase match cannot. Deliberately narrow + (`section_key=chi_dinh` only, never the whole corpus) so this stays + a targeted fallback for one specific gap, not a return to unranked + similarity search — see ADR 0008 on why the live path otherwise + avoids `search()`. One hit per drug, highest-scoring chunk kept + (Qdrant returns points pre-sorted by score).""" + from qdrant_client.models import FieldCondition, Filter, MatchValue + + vector = list(self._embedder.embed_query(query)) + if len(vector) != self._embedder.dimensions: + raise ValueError( + f"query vector has {len(vector)} dimensions; " + f"expected {self._embedder.dimensions}" + ) + points = self._client.search( + collection_name=self._collection_name, + query_vector=vector, + query_filter=Filter( + must=[ + FieldCondition(key="section_key", match=MatchValue(value="chi_dinh")), + FieldCondition(key="chunk_kind", match=MatchValue(value="prose")), + ] + ), + limit=limit * 4, + with_payload=True, + ) + hits: list[SearchHit] = [] + seen_drugs: set[str] = set() + for point in points: + payload = dict(point.payload or {}) + drug_id = payload.get("drug_id") + if drug_id in seen_drugs: + continue + seen_drugs.add(drug_id) + hits.append(SearchHit(_document(payload), float(point.score))) + if len(hits) >= limit: + break + return hits + class QdrantParentStore: def __init__(self, client: Any, collection_name: str) -> None: diff --git a/apps/ai-service/bootstrap.py b/apps/ai-service/bootstrap.py index 21b176e..1a1d8e1 100644 --- a/apps/ai-service/bootstrap.py +++ b/apps/ai-service/bootstrap.py @@ -3,7 +3,7 @@ from __future__ import annotations from qdrant_client import QdrantClient from adapters.embedding import BedrockCohereQueryEmbedder -from adapters.postgres import PostgresTraceRepository +from adapters.postgres import PostgresConversationStore, PostgresTraceRepository from adapters.qdrant import QdrantParentStore, QdrantRetriever from config import Settings from rag.agent import RagAgent @@ -180,5 +180,8 @@ def build_runtime(settings: Settings): retrieval=retrieval, answers=answers, autocomplete=resolver, + max_wall_clock_ms=settings.max_wall_clock_ms, + max_llm_calls_per_turn=settings.max_llm_calls_per_turn, + store=PostgresConversationStore(settings.postgres_dsn), ) return answers, agent, trace_writer, metrics diff --git a/apps/ai-service/config.py b/apps/ai-service/config.py index 3b58e05..a6edde8 100644 --- a/apps/ai-service/config.py +++ b/apps/ai-service/config.py @@ -40,6 +40,13 @@ class Settings(BaseSettings): Path(__file__).resolve().parents[2] / "ingestion/data/verified/drug_entities.json" ) + # F-08: a per-turn budget across RagAgent's sequential Bedrock calls + # (understand, sufficiency, generate, up to 2 entailment retries). + # Defaults sized with headroom above what a normal turn measures live + # (~8-9s, 4-5 calls) — see rag/agent.py's MAX_WALL_CLOCK_MS/ + # MAX_LLM_CALLS_PER_TURN for the full rationale. + max_wall_clock_ms: int = 20_000 + max_llm_calls_per_turn: int = 8 @lru_cache diff --git a/apps/ai-service/migrate.py b/apps/ai-service/migrate.py index ecc0114..72ab81d 100644 --- a/apps/ai-service/migrate.py +++ b/apps/ai-service/migrate.py @@ -1,13 +1,20 @@ from pathlib import Path -from adapters.postgres import PostgresTraceRepository +from adapters.postgres import PostgresConversationStore, PostgresTraceRepository from config import get_settings def main() -> None: - migration = Path(__file__).parent / "migrations/001_rag_retrieval_trace.sql" - PostgresTraceRepository(get_settings().postgres_dsn).migrate(migration) - print(f"Applied {migration.name}") + dsn = get_settings().postgres_dsn + migrations_dir = Path(__file__).parent / "migrations" + + trace_migration = migrations_dir / "001_rag_retrieval_trace.sql" + PostgresTraceRepository(dsn).migrate(trace_migration) + print(f"Applied {trace_migration.name}") + + conversation_migration = migrations_dir / "002_rag_conversation_turn.sql" + PostgresConversationStore(dsn).migrate(conversation_migration) + print(f"Applied {conversation_migration.name}") if __name__ == "__main__": diff --git a/apps/ai-service/migrations/002_rag_conversation_turn.sql b/apps/ai-service/migrations/002_rag_conversation_turn.sql new file mode 100644 index 0000000..bd5f744 --- /dev/null +++ b/apps/ai-service/migrations/002_rag_conversation_turn.sql @@ -0,0 +1,13 @@ +-- F-08 durable conversation history: replaces RagAgent's in-process dict +-- (lost on restart, not shared across workers). Append-only; `id`'s +-- insertion order is the "oldest -> newest" ordering the understanding LLM +-- prompt already expects, no separate turn-index column needed. +CREATE TABLE IF NOT EXISTS rag_conversation_turn ( + id bigserial PRIMARY KEY, + conversation_id text NOT NULL, + line text NOT NULL, + created_at timestamptz NOT NULL DEFAULT now() +); + +CREATE INDEX IF NOT EXISTS rag_conversation_turn_conv_idx + ON rag_conversation_turn (conversation_id, id); diff --git a/apps/ai-service/rag/agent.py b/apps/ai-service/rag/agent.py index 5471495..10fb417 100644 --- a/apps/ai-service/rag/agent.py +++ b/apps/ai-service/rag/agent.py @@ -17,17 +17,50 @@ This module owns routing only; it states no medical fact of its own. """ from __future__ import annotations +import logging +import time from dataclasses import dataclass, field from typing import Protocol from .answer import Citation, GroundedAnswerService +from .budget import RequestBudget from .models import EvidenceDecision, RetrievalResult from .policy import looks_non_human from .service import RetrievalService from .understanding import QueryFrame, QueryUnderstander +logger = logging.getLogger(__name__) + TUONG_TAC = "tuong_tac_thuoc" HISTORY_TURNS = 6 +# F-08: measured live 2026-08-07, a normal answerable turn makes 4 sequential +# Bedrock calls (understand, sufficiency, generate, entailment) and costs +# ~8-9s; a noisy entailment retry adds a 5th. Defaults sized with headroom +# above that measured normal case, not at its exact edge, so ordinary +# traffic never trips the budget — it exists for the pathological case +# (a stuck/slow call, or an unexpectedly long retry chain), not to shave +# time off the common path. +MAX_WALL_CLOCK_MS = 20_000 +MAX_LLM_CALLS_PER_TURN = 8 +# Found live 2026-08-07 (50-question hand-typed browser audit): the +# understanding LLM can get stuck re-asking the same (or a near-identical) +# clarify question forever — reproduced 3 times independently, one case +# never converged after 5 real answered turns. `understanding.py`'s prior- +# frame merge (F-11) fixes most of the underlying cause, but a code-level +# backstop is still needed: nothing bounds a model that just keeps deciding +# needs_clarify=true no matter what. After this many CONSECUTIVE clarify +# turns on the same conversation, force a hard stop instead of asking again. +MAX_CONSECUTIVE_CLARIFY = 4 + + +class ConversationStore(Protocol): + """Durable, cross-worker alternative to the in-process history dict — + the gap ADR 0008 names as still open. Satisfied by + `adapters.postgres.PostgresConversationStore`; entirely optional — with + none configured, `RagAgent` behaves exactly as before this existed.""" + + def recent(self, conversation_id: str, limit: int) -> list[str]: ... + def append(self, conversation_id: str, line: str) -> None: ... class AutocompleteSource(Protocol): @@ -47,6 +80,7 @@ class AgentReply: drugs: tuple[str, ...] = () turn_type: str = "" generated: bool = False + quick_replies: tuple[str, ...] = () class RagAgent: @@ -57,13 +91,23 @@ class RagAgent: answers: GroundedAnswerService, autocomplete: AutocompleteSource | None = None, history_turns: int = HISTORY_TURNS, + max_wall_clock_ms: int = MAX_WALL_CLOCK_MS, + max_llm_calls_per_turn: int = MAX_LLM_CALLS_PER_TURN, + store: ConversationStore | None = None, ) -> None: self._understander = understander self._retrieval = retrieval self._answers = answers self._autocomplete = autocomplete self._history_turns = history_turns + self._max_wall_clock_ms = max_wall_clock_ms + self._max_llm_calls_per_turn = max_llm_calls_per_turn + self._store = store self._history: dict[str, list[str]] = {} + # In-process only, same durability caveat as `_history` (ADR 0008's + # named gap) — lost on restart, not shared across workers. + self._last_frame: dict[str, QueryFrame] = {} + self._clarify_streak: dict[str, int] = {} def complete(self, prefix: str, k: int = 8) -> list[str]: """Display names matching a typed prefix, for input autocomplete.""" @@ -72,20 +116,108 @@ class RagAgent: return [_display_name(drug_id) for drug_id in self._autocomplete.complete(prefix, k)] def handle(self, turn: str, conversation_id: str | None = None) -> AgentReply: - history = self._history.get(conversation_id, []) if conversation_id else [] - frame = self._understander.understand(turn, tuple(history)) - reply = self._route(turn, frame) + # F-08: one budget per turn, threaded through every LLM call this + # turn makes (understand, then whatever `_route` reaches). + t0 = time.monotonic() + budget = RequestBudget.start(self._max_wall_clock_ms, self._max_llm_calls_per_turn) + history = self._get_history(conversation_id) + t1 = time.monotonic() + prior_frame = self._last_frame.get(conversation_id) if conversation_id else None + frame = self._understander.understand( + turn, tuple(history), budget=budget, prior_frame=prior_frame + ) + t2 = time.monotonic() + reply = self._route(turn, frame, budget) + reply = self._enforce_clarify_circuit_breaker(conversation_id, reply) + t3 = time.monotonic() if conversation_id is not None: self._remember(conversation_id, turn, reply) + self._last_frame[conversation_id] = frame + t4 = time.monotonic() + # Temporary instrumentation (2026-08-07): added specifically to + # pinpoint a live, reproduced-in-browser case of the FIRST LLM call + # already reporting the F-08 budget exhausted — i.e. >20s elapsed + # before even one Bedrock call was attempted, with no code between + # `RequestBudget.start()` and that first `budget.require()` that + # should plausibly take anywhere near that long. Logs unconditionally + # (not just on the slow path) so a normal turn's timing is on record + # too, for comparison. + # .warning, not .info: uvicorn's default logging config only wires + # handlers onto its own "uvicorn"/"uvicorn.access" loggers, not the + # root logger, so a plain .info() here would silently go nowhere — + # confirmed by `understanding.py`'s existing warning-level log + # already showing up in the same server output this session. + logger.warning( + "handle() timing: history=%.2fs understand=%.2fs route=%.2fs " + "remember=%.2fs total=%.2fs", + t1 - t0, t2 - t1, t3 - t2, t4 - t3, t4 - t0, + ) return reply - def _route(self, turn: str, frame: QueryFrame) -> AgentReply: + def _enforce_clarify_circuit_breaker( + self, conversation_id: str | None, reply: AgentReply + ) -> AgentReply: + """Hard stop after `MAX_CONSECUTIVE_CLARIFY` clarify turns in a row. + + Every other failure mode in this file degrades to a bounded, honest + abstain — this is the one path that previously had no bound at all: + a model that keeps deciding needs_clarify=true has no natural exit, + and the user has no way out except abandoning the conversation. Any + non-clarify decision (answered, or a different abstain reason) resets + the streak — this only fires on genuinely consecutive clarifies. + """ + if conversation_id is None: + return reply + if reply.decision != "clarify": + self._clarify_streak.pop(conversation_id, None) + return reply + streak = self._clarify_streak.get(conversation_id, 0) + 1 + if streak >= MAX_CONSECUTIVE_CLARIFY: + self._clarify_streak.pop(conversation_id, None) + return AgentReply( + "abstain", "clarify_loop_exhausted", + answer=( + "Hệ thống chưa xác định đủ thông tin sau nhiều lần hỏi lại. " + "Anh/chị vui lòng gõ lại TOÀN BỘ câu hỏi trong một tin nhắn " + "đầy đủ (tên thuốc, đối tượng, cân nặng/tuổi nếu có, đường " + "dùng), hoặc bấm \"Tạo phiên tra cứu mới\" để bắt đầu lại." + ), + turn_type=reply.turn_type, + ) + self._clarify_streak[conversation_id] = streak + return reply + + def _get_history(self, conversation_id: str | None) -> list[str]: + if conversation_id is None: + return [] + if self._store is not None: + try: + return self._store.recent(conversation_id, self._history_turns * 2) + except Exception: + # Fail-open (F-09's precedent): a store outage means this + # turn is understood fresh, with no memory of earlier ones — + # worse UX, not a 500. `routers/rag.py` wraps + # `PostgresTraceRepository.save()` the same bare way for the + # same reason. + return [] + return self._history.get(conversation_id, []) + + def _route(self, turn: str, frame: QueryFrame, budget: RequestBudget) -> AgentReply: tt = frame.turn_type if frame.needs_clarify and frame.clarify_reason: - return AgentReply("clarify", "needs_more_info", + # `system_error` set means this isn't a real clarify at all — the + # understanding call itself failed (provider outage, malformed + # output) and failed closed to this same shape. Surface the real + # reason instead of the generic "needs_more_info" so a technical + # failure is distinguishable from an ordinary question back, both + # in the API response and in `/metrics`/traces (found live + # 2026-08-07: these were indistinguishable, which is why a real + # outage looked identical to normal clarify traffic). + return AgentReply("clarify", frame.system_error or "needs_more_info", clarification=frame.clarify_reason, - drugs=frame.drugs, turn_type=tt) + drugs=frame.drugs, turn_type=tt, + quick_replies=frame.quick_replies) if tt == "smalltalk": return AgentReply( @@ -111,40 +243,54 @@ class RagAgent: answer=f"Không tìm thấy \"{names}\" trong Dược thư Quốc gia Việt Nam.", turn_type=tt) if tt == "symptom_to_drug": - # Reverse lookup (indication/adverse-effect -> drugs) is a distinct - # retrieval mode, not yet wired. Be honest rather than abstain blank. + if frame.indication: + return self._symptom_to_drug(turn, frame, budget) return AgentReply( - "clarify", "reverse_lookup_not_ready", - clarification="Tra ngược theo triệu chứng/chỉ định đang được bổ " - "sung. Anh/chị cho biết tên thuốc cụ thể để tôi tra giúp?", + "clarify", "no_indication", + clarification="Anh/chị mô tả triệu chứng hoặc chỉ định cần tra giúp " + "em với?", turn_type=tt) return AgentReply( "clarify", "no_drug", clarification="Anh/chị muốn tra thuốc nào?", turn_type=tt) if tt == "interaction" and len(frame.drugs) >= 2: - return self._interaction(turn, frame) + return self._interaction(turn, frame, budget) # drug_attribute / drug_overview / dosing_calc / fallback: one drug + section - return self._single_drug(turn, frame) + return self._single_drug(turn, frame, budget) - def _single_drug(self, turn: str, frame: QueryFrame) -> AgentReply: + def _single_drug(self, turn: str, frame: QueryFrame, budget: RequestBudget) -> AgentReply: + query = _synthesize_query(turn, frame) result = self._retrieval.retrieve_framed( - frame.drugs[0], frame.attribute, turn, + frame.drugs[0], frame.attribute, query, is_overview=frame.turn_type == "drug_overview", ) - return self._grounded(turn, result, frame) + return self._grounded(query, result, frame, budget=budget) - def _interaction(self, turn: str, frame: QueryFrame) -> AgentReply: + def _interaction(self, turn: str, frame: QueryFrame, budget: RequestBudget) -> AgentReply: """Gather the interaction section of each named drug and synthesise. Absence of a match is stated as "not found in each drug's interaction section", never as "safe" — the answer layer's grounding still applies. + + A quarantined table/formula in EITHER drug's interaction section + forces the whole combined answer to VERIFY_PDF via + `RetrievalService.decide` — the same policy the single-drug path + already applies to a section with quarantined content. Previously + this only kept `part.decision == ANSWERABLE` parts, so a quarantined + drug's evidence (and the "table exists, verify PDF" notice it must + produce per the quarantine contract) was silently dropped instead of + surfaced; a confident interaction answer could omit exactly the + unverified contraindication table it should have flagged. Generating + a synthesis claim from one verified and one unverified source is not + safer than generating from either alone, so both now block + generation the same way. """ evidences = [] for drug_id in frame.drugs: part = self._retrieval.retrieve_framed(drug_id, TUONG_TAC, turn) - if part.decision == EvidenceDecision.ANSWERABLE: + if part.decision in (EvidenceDecision.ANSWERABLE, EvidenceDecision.VERIFY_PDF): evidences.extend(part.evidence) if not evidences: listed = " và ".join(frame.drugs) @@ -153,16 +299,47 @@ class RagAgent: answer=f"Không tìm thấy mục tương tác thuốc cho {listed} trong Dược " "thư. Điều này KHÔNG có nghĩa là an toàn khi phối hợp.", drugs=frame.drugs, turn_type=frame.turn_type) - combined = RetrievalResult( - EvidenceDecision.ANSWERABLE, "interaction_evidence", - tuple(evidences), + combined = self._retrieval.decide(tuple(evidences)) + return self._grounded(turn, combined, frame, budget=budget) + + def _symptom_to_drug( + self, turn: str, frame: QueryFrame, budget: RequestBudget + ) -> AgentReply: + """Reverse lookup: a symptom/indication -> which drugs' `chi_dinh` + actually names it. A factual list from the formulary, not a + treatment ranking or recommendation — no drug is preferred over + another here, only cited as indicated ([[feedback_no_recommendation_gate]]: + this audience is doctors/pharmacists, a lookup like this is normal use). + Absence is stated plainly, never as "no such drug exists" — the + formulary may simply not name this indication under any monograph. + """ + result = self._retrieval.retrieve_by_indication(frame.indication) + if result.decision == EvidenceDecision.ABSTAIN: + return AgentReply( + "abstain", result.reason, + answer=f"Không tìm thấy thuốc nào trong Dược thư Quốc gia Việt Nam ghi " + f"nhận chỉ định cho \"{frame.indication}\". Điều này KHÔNG có " + "nghĩa là không có thuốc điều trị — vui lòng tra theo tên thuốc " + "cụ thể nếu đã biết.", + turn_type=frame.turn_type) + # `matched_doc_id` is always `{drug_id}__chi_dinh__{part_index}` — the + # drugs actually found, not `frame.drugs` (empty by construction for + # this turn_type; the router only reaches here with no named drug). + matched_drugs = tuple(dict.fromkeys( + evidence.matched_doc_id.split("__")[0] for evidence in result.evidence + )) + return self._grounded( + turn, result, frame, drugs=matched_drugs, list_mode=True, budget=budget ) - return self._grounded(turn, combined, frame) def _grounded( - self, turn: str, result: RetrievalResult, frame: QueryFrame + self, turn: str, result: RetrievalResult, frame: QueryFrame, + drugs: tuple[str, ...] | None = None, list_mode: bool = False, + budget: RequestBudget | None = None, ) -> AgentReply: - ga = self._answers.answer_from_result(turn, result) + ga = self._answers.answer_from_result( + turn, result, list_mode=list_mode, budget=budget + ) decision = ga.result.decision.value if ga.clarification is not None: decision = "clarify" @@ -172,18 +349,34 @@ class RagAgent: answer=ga.answer, clarification=ga.clarification, citations=ga.citations, - drugs=frame.drugs, + drugs=drugs if drugs is not None else frame.drugs, turn_type=frame.turn_type, generated=ga.generated, + quick_replies=ga.quick_replies, ) def _remember(self, conversation_id: str, turn: str, reply: AgentReply) -> None: - history = self._history.setdefault(conversation_id, []) - history.append(f"Người dùng: {turn}") + lines = [f"Người dùng: {turn}"] spoken = reply.answer or reply.clarification if spoken: - history.append(f"Trợ lý: {spoken[:300]}") - # Keep only the recent window; the LLM re-reads it every turn. + lines.append(f"Trợ lý: {spoken[:300]}") + + if self._store is not None: + try: + for line in lines: + self._store.append(conversation_id, line) + except Exception: + # Fail-open: this turn's memory is lost, not the response + # already computed and about to be returned to the caller. + pass + return + + history = self._history.setdefault(conversation_id, []) + history.extend(lines) + # Keep only the recent window; the LLM re-reads it every turn. Only + # needed for the in-process dict — the store path windows at READ + # time instead (`recent(..., limit)`), so old rows just sit unused + # rather than needing a delete on every turn. excess = len(history) - self._history_turns * 2 if excess > 0: del history[:excess] @@ -192,3 +385,60 @@ class RagAgent: def _display_name(drug_id: str) -> str: """A readable display name from a drug id ('paracetamol_acetaminophen').""" return drug_id.replace("_", " ").title() + + +_POPULATION_LABELS = { + "tre_em": "trẻ em", + "tre_so_sinh": "trẻ sơ sinh", + "nguoi_lon": "người lớn", + "nguoi_cao_tuoi": "người cao tuổi", + "phu_nu_co_thai": "phụ nữ có thai", + "phu_nu_cho_con_bu": "phụ nữ cho con bú", + "suy_than": "suy thận", + "suy_gan": "suy gan", +} + +_ROUTE_LABELS = { + "uong": "uống", + "tiem_tinh_mach": "tiêm tĩnh mạch", + "tiem_bap": "tiêm bắp", + "tiem_duoi_da": "tiêm dưới da", + "dat_truc_trang": "đặt trực tràng", + "boi_ngoai_da": "bôi ngoài da", + "nho_mat": "nhỏ mắt", + "nho_mui": "nhỏ mũi", + "khac": "khác", +} + + +def _synthesize_query(turn: str, frame: QueryFrame) -> str: + """Fold the structured context `understanding.py` resolved — possibly + across several turns — into one self-contained question. + + `GroundedAnswerService.answer_from_result` has no conversation history of + its own; the `query` string it receives IS the entire context its + sufficiency-check and generation LLM calls see. Passing the bare current + turn loses everything resolved earlier: a reply like "Uống" answering a + route question three turns into a dose conversation would reach + generation as just "Uống", indistinguishable from a user who typed + nothing else — the two P0s the 2026-08-06 audit named (population/ + weight/age/route extracted but discarded downstream) are exactly this + gap. Redundant when the turn is already self-contained (a fresh + single-shot question re-states its own population/route, so this just + repeats it) — harmless, since omission is the failure mode, not + repetition. + """ + parts = [turn] + if frame.population: + parts.append(f"Đối tượng: {_POPULATION_LABELS.get(frame.population, frame.population)}") + if frame.age_text: + parts.append(f"Tuổi: {frame.age_text}") + if frame.weight_kg is not None: + parts.append(f"Cân nặng: {frame.weight_kg:g} kg") + if frame.route: + parts.append(f"Đường dùng: {_ROUTE_LABELS.get(frame.route, frame.route)}") + if frame.indication: + parts.append(f"Chỉ định/triệu chứng: {frame.indication}") + if len(parts) == 1: + return turn + return ". ".join(parts) + "." diff --git a/apps/ai-service/rag/answer.py b/apps/ai-service/rag/answer.py index e44a076..7707634 100644 --- a/apps/ai-service/rag/answer.py +++ b/apps/ai-service/rag/answer.py @@ -5,12 +5,17 @@ import re from dataclasses import dataclass, replace from . import grounding, metrics as metric_names +from .budget import RequestBudget, RequestBudgetExhausted from .metrics import Metrics, NullMetrics from .models import EvidenceDecision, QueryIntent, RetrievalResult, SubjectScope from .ports import AnswerGenerationUnavailable, AnswerGenerator from .prompt import build_entailment_request, build_request, build_sufficiency_request from .routing import QueryRoutingService +# See `_verify_entailment`'s docstring for the measured trade-off behind +# widening this from 2 to 3. +_ENTAILMENT_MAX_ATTEMPTS = 3 + @dataclass(frozen=True) class Citation: @@ -22,6 +27,10 @@ class Citation: bbox: tuple[float, float, float, float] | None = None source_crop: str | None = None attachment: str | None = None + # The exact retrieved text this citation stands for — the same string + # handed to the generator/entailment checks, so the UI can show precisely + # what was retrieved rather than a fabricated summary of it. + evidence_text: str = "" @dataclass(frozen=True) @@ -34,12 +43,35 @@ class GroundedAnswer: # (e.g. a dose question with no age/weight). The answer field carries the # question; the caller renders it as a clarification, not a final answer. clarification: str | None = None + # Short suggested replies for `clarification`, e.g. ("Người lớn", "Trẻ + # em") — only populated when the sufficiency check judged the question + # to have a few natural discrete answers, never invented client-side. + quick_replies: tuple[str, ...] = () @dataclass(frozen=True) class _GenOutcome: answer: str | None = None clarification: str | None = None + # The specific reason a rejection happened — the exact string already + # used for the GENERATION_REJECTED metric, propagated here so + # `answer_from_result` can put it in the API response's `reason` field + # instead of a generic catch-all. `None` when `answer`/`clarification` + # is set (nothing was rejected). + reject_reason: str | None = None + + +@dataclass(frozen=True) +class _RawAttempt: + """One raw `_attempt_generation` call, before any metric is charged — + lets `_generate` retry the noisy `insufficient` case without + double-counting a rejection metric across both attempts.""" + answer: str | None = None + clarification: str | None = None + insufficient: bool = False + outage: bool = False + budget_exhausted: bool = False + malformed: bool = False class GroundedAnswerService: @@ -57,10 +89,13 @@ class GroundedAnswerService: pass confirming each cited claim's *content* — not just its numbers — is actually stated by that block). If a configured generation fails any check, or the provider itself is unreachable, or its output is - malformed, the turn **abstains** (`reason="generation_unavailable"` - or the specific `grounding.verify` reason) rather than silently - degrading to a raw source dump — this product is a real LLM chatbot, - and a citation-stapled paragraph of book text is not an acceptable + malformed, the turn **abstains** with the specific reason that failed + it (`provider_unavailable`, `malformed_output`, `evidence_insufficient`, + a `grounding.verify` reason, or `unsupported_claim`; falls back to + the generic `generation_unavailable` only if none of those was set) + rather than silently degrading to a raw source dump — this product is + a real LLM chatbot, and a citation-stapled paragraph of book text is + not an acceptable stand-in for an answer the model was supposed to produce. """ @@ -94,12 +129,27 @@ class GroundedAnswerService: return self.answer_from_result(query, result) def answer_from_result( - self, query: str, result: RetrievalResult + self, query: str, result: RetrievalResult, list_mode: bool = False, + budget: RequestBudget | None = None, ) -> GroundedAnswer: """Everything after retrieval — grounding, sufficiency, generation, citations. Split out so the new understanding-driven orchestrator (`rag/agent.py`) reuses the safe answer path without going through the - old `QueryRoutingService` text resolution.""" + old `QueryRoutingService` text resolution. + + `list_mode=True`: the evidence is several DIFFERENT drugs' own + sections (symptom_to_drug), not alternative phrasings of one drug's + answer — the sufficiency clarify ("which kind of headache?") that's + right for a single dose question doesn't fit a reverse lookup, whose + whole point is to show what the formulary has and let the clinician + narrow it themselves; skipped here the same way a bare-name intro + already skips it. + + `budget` (F-08): threaded through to every LLM call this method + makes (sufficiency, generate, up to 2 entailment). `None` (the + default) means unbounded, unchanged from before F-08 — only + `RagAgent` constructs a real budget today. + """ if result.decision == EvidenceDecision.ABSTAIN: self._metrics.increment(metric_names.ABSTENTION, reason=result.reason) return GroundedAnswer(result, None) @@ -135,11 +185,22 @@ class GroundedAnswerService: # Reasoning step BEFORE answering: if the turn is under-specified (a dose # with several bands and no age/weight/condition), ask instead of dumping. # A separate focused call is more reliable than folding it into generation. - clarify_q = self._check_sufficiency(query, evidence_texts, result.is_drug_overview) - if clarify_q is not None: - return GroundedAnswer(result, clarify_q, (), clarification=clarify_q) + sufficiency = ( + None if list_mode + else self._check_sufficiency( + query, evidence_texts, result.is_drug_overview, budget=budget + ) + ) + if sufficiency is not None: + clarify_q, quick_replies = sufficiency + return GroundedAnswer( + result, clarify_q, (), clarification=clarify_q, quick_replies=quick_replies + ) - outcome = self._generate(query, evidence_texts, intro=result.is_drug_overview) + outcome = self._generate( + query, evidence_texts, intro=result.is_drug_overview, list_mode=list_mode, + budget=budget, + ) if outcome.clarification is not None: # The model judged the turn under-specified (a dose with no # age/weight/renal-function/indication…) and asked back instead of @@ -163,14 +224,23 @@ class GroundedAnswerService: # source dump is not an acceptable stand-in for a failed # generation, so this abstains instead of silently degrading to # one. - self._metrics.increment( - metric_names.ABSTENTION, reason="generation_unavailable" - ) + # The specific check that failed (provider_unavailable, + # malformed_output, evidence_insufficient, ungrounded_number, + # uncited_claim, unsupported_claim, request_budget_exhausted) — + # found live 2026-08-07: every one of these used to collapse into + # the same generic "generation_unavailable" by the time it + # reached the API response/trace, so a real, diagnosable cause + # (e.g. a genuine provider outage) was indistinguishable from + # ordinary entailment noise without reading server-side metrics + # by hand. `outcome.reject_reason` already carries the granular + # value the metric above uses — just propagate it. + reason = outcome.reject_reason or "generation_unavailable" + self._metrics.increment(metric_names.ABSTENTION, reason=reason) return GroundedAnswer( replace( result, decision=EvidenceDecision.ABSTAIN, - reason="generation_unavailable", + reason=reason, ), None, ) @@ -182,67 +252,107 @@ class GroundedAnswerService: self._metrics.increment(metric_names.GENERATION_SERVED) return GroundedAnswer(result, outcome.answer, citations, generated=True) - def _generate( - self, query: str, evidence_texts: tuple[str, ...], intro: bool = False - ) -> "_GenOutcome": - """A verified generation, a clarifying question, or empty to fall back.""" - if self._generator is None or not evidence_texts: - return _GenOutcome() - - request = build_request(query, evidence_texts, intro=intro) + def _attempt_generation( + self, request: "GenerationRequest", budget: RequestBudget | None + ) -> "_RawAttempt": + """One raw generation call, parsed but not yet metric-counted or + verified — the caller decides whether to retry before charging a + metric to any particular reason.""" try: + if budget is not None: + budget.require() raw = self._generator.generate(request.system, request.user, request.schema) + except RequestBudgetExhausted: + return _RawAttempt(budget_exhausted=True) except AnswerGenerationUnavailable: - self._metrics.increment( - metric_names.GENERATION_REJECTED, reason="provider_unavailable" - ) - return _GenOutcome() + return _RawAttempt(outage=True) try: payload = json.loads(raw) answer = payload["answer"] sufficient = payload["evidence_sufficient"] except (ValueError, TypeError, KeyError): - self._metrics.increment( - metric_names.GENERATION_REJECTED, reason="malformed_output" - ) - return _GenOutcome() + return _RawAttempt(malformed=True) # The model asked for a missing detail (age/weight/renal function/ # indication…) instead of listing every band. A clarify is not a grounded # claim, so it skips the number check — it states no dose. clarify = payload.get("clarifying_question") if isinstance(payload, dict) else None if isinstance(clarify, str) and clarify.strip(): - return _GenOutcome(clarification=clarify.strip()) + return _RawAttempt(clarification=clarify.strip()) if not isinstance(answer, str) or not isinstance(sufficient, bool): + return _RawAttempt(malformed=True) + if not sufficient: + return _RawAttempt(insufficient=True) + return _RawAttempt(answer=answer) + + def _generate( + self, query: str, evidence_texts: tuple[str, ...], intro: bool = False, + list_mode: bool = False, budget: RequestBudget | None = None, + ) -> "_GenOutcome": + """A verified generation, a clarifying question, or empty to fall back.""" + if self._generator is None or not evidence_texts: + return _GenOutcome() + + request = build_request(query, evidence_texts, intro=intro, list_mode=list_mode) + attempt = self._attempt_generation(request, budget) + if attempt.insufficient: + # Empirically noisy (found live 2026-08-07, reproduced 3/3 on a + # fresh retry): the model's own evidence_sufficient=false + # self-assessment sometimes flips to a correct, fully grounded + # answer when asked again with the IDENTICAL evidence — the same + # one-retry pattern `_verify_entailment` already uses below for + # its own noisy judge call. Only the terminal "insufficient AND + # no clarifying question" case retries; a legitimate ask-for- + # more-detail clarify is untouched. + attempt = self._attempt_generation(request, budget) + + if attempt.budget_exhausted: + self._metrics.increment( + metric_names.GENERATION_REJECTED, reason="request_budget_exhausted" + ) + return _GenOutcome(reject_reason="request_budget_exhausted") + if attempt.outage: + self._metrics.increment( + metric_names.GENERATION_REJECTED, reason="provider_unavailable" + ) + return _GenOutcome(reject_reason="provider_unavailable") + if attempt.malformed: self._metrics.increment( metric_names.GENERATION_REJECTED, reason="malformed_output" ) - return _GenOutcome() - if not sufficient: - # The model says the evidence does not answer the question. Showing - # the retrieved section verbatim lets the clinician judge that. + return _GenOutcome(reject_reason="malformed_output") + if attempt.clarification is not None: + return _GenOutcome(clarification=attempt.clarification) + if attempt.insufficient: + # The model says the evidence does not answer the question, on + # both attempts. Showing the retrieved section verbatim lets the + # clinician judge that. self._metrics.increment( metric_names.GENERATION_REJECTED, reason="evidence_insufficient" ) - return _GenOutcome() + return _GenOutcome(reject_reason="evidence_insufficient") + answer = attempt.answer report = grounding.verify(answer, evidence_texts) if not report.grounded: self._metrics.increment( metric_names.GENERATION_REJECTED, reason=report.reason ) - return _GenOutcome() + return _GenOutcome(reject_reason=report.reason) - if not self._verify_entailment(answer, evidence_texts): + if not self._verify_entailment(answer, evidence_texts, budget=budget): self._metrics.increment( metric_names.GENERATION_REJECTED, reason="unsupported_claim" ) - return _GenOutcome() + return _GenOutcome(reject_reason="unsupported_claim") return _GenOutcome(answer=answer) - def _verify_entailment(self, answer: str, evidence_texts: tuple[str, ...]) -> bool: + def _verify_entailment( + self, answer: str, evidence_texts: tuple[str, ...], + budget: RequestBudget | None = None, + ) -> bool: """A second, adversarial LLM pass over an answer that already passed `grounding.verify`. @@ -254,14 +364,22 @@ class GroundedAnswerService: is checked against only the evidence block(s) it names, by a model told to compare wording, not to reason about medicine. - Fails closed on an outage or malformed output. A single rejection is - NOT: live probing (2026-08-06) found the judge call itself is noisy - — the identical claim/evidence pair, called three times, came back - entailed twice and rejected once, discarding a correct, well-cited - interaction answer. So a reject triggers one same-claim retry, and - only a second, agreeing reject discards the generation; a single - provider outage/malformed response still fails closed immediately - (that failure mode is reliable, not noisy — no retry needed there). + Fails closed on an outage or malformed output — that failure mode is + reliable, not noisy, so it stops immediately rather than spending + retries on it. A single rejection is NOT reliable: live probing + (2026-08-06) found the judge call itself is noisy — the identical + claim/evidence pair, called three times, came back entailed twice + and rejected once, discarding a correct, well-cited interaction + answer. Up to `_ENTAILMENT_MAX_ATTEMPTS` same-claim calls run; + accept on the first `True`, discard only if every attempt agrees + reject. Widened from 2 to 3 attempts 2026-08-07 after a live + adversarial sample (50 real questions) measured this specific check + as roughly half of all false abstentions on genuinely answerable + questions. Trade-off, stated plainly: this raises the bar a + genuinely fabricated claim must now clear too (it survives if ANY + one of 3 noisy calls wrongly accepts it, not just 1 of 2) — accepted + because the probed noise is symmetric and the entailment prompt + itself is unchanged, not because the risk is zero. An answer with no claim text at all (nothing between or after its citation markers) is vacuously fine — nothing to verify, no call. """ @@ -274,18 +392,23 @@ class GroundedAnswerService: return True request = build_entailment_request(claims) - first = self._run_entailment_check(request) - if first is None: - return False - if first: - return True - second = self._run_entailment_check(request) - return bool(second) + for _ in range(_ENTAILMENT_MAX_ATTEMPTS): + verdict = self._run_entailment_check(request, budget=budget) + if verdict is None: + return False + if verdict: + return True + return False - def _run_entailment_check(self, request) -> bool | None: - """One entailment call. `None` = outage/malformed (fails closed by the - caller without a retry); `True`/`False` = the judge's verdict.""" + def _run_entailment_check( + self, request, budget: RequestBudget | None = None + ) -> bool | None: + """One entailment call. `None` = outage/malformed/budget-exhausted + (fails closed by the caller without a retry); `True`/`False` = the + judge's verdict.""" try: + if budget is not None: + budget.require() raw = self._generator.generate(request.system, request.user, request.schema) except AnswerGenerationUnavailable: return None @@ -300,17 +423,31 @@ class GroundedAnswerService: return entailed and not unsupported def _check_sufficiency( - self, query: str, evidence_texts: tuple[str, ...], intro: bool = False - ) -> str | None: + self, query: str, evidence_texts: tuple[str, ...], intro: bool = False, + budget: RequestBudget | None = None, + ) -> tuple[str, tuple[str, ...]] | None: """A focused reasoning call: is the turn specific enough to answer, or - must we ask? Returns a clarifying question, or None to proceed. + must we ask? Returns (clarifying_question, quick_replies), or None to + proceed. `quick_replies` is often empty — only populated when the + model judged the missing detail has a few natural discrete answers + (e.g. "Người lớn"/"Trẻ em"), never invented here. Skipped without a model, for a bare-name intro (not a dose), or for a - single evidence block (nothing to disambiguate).""" + single evidence block (nothing to disambiguate). + + Fails OPEN on outage/budget-exhaustion (returns None, proceeds to + generate) — deliberately different from every other call in this + file, which fail closed. This is a reasoning heuristic, not a safety + check; grounding + entailment remain the real gate on whatever gets + generated next, so skipping this one costs UX quality (a dose + question that should have asked for age/weight might not), not + safety.""" if self._generator is None or intro or len(evidence_texts) < 2: return None request = build_sufficiency_request(query, evidence_texts) try: + if budget is not None: + budget.require() raw = self._generator.generate(request.system, request.user, request.schema) except AnswerGenerationUnavailable: return None @@ -321,7 +458,13 @@ class GroundedAnswerService: if isinstance(payload, dict) and payload.get("sufficient") is False: question = payload.get("clarifying_question") if isinstance(question, str) and question.strip(): - return question.strip() + raw_replies = payload.get("quick_replies") + replies = tuple( + reply.strip() + for reply in raw_replies + if isinstance(reply, str) and reply.strip() + ) if isinstance(raw_replies, list) else () + return question.strip(), replies return None @staticmethod @@ -362,5 +505,6 @@ class GroundedAnswerService: # real crop path wins; otherwise the block id plus the # structured page/bbox fields is enough to render later. attachment=source.source_crop or source.block_id, + evidence_text=evidence.text, ))) return citations diff --git a/apps/ai-service/rag/budget.py b/apps/ai-service/rag/budget.py new file mode 100644 index 0000000..9f7ba5a --- /dev/null +++ b/apps/ai-service/rag/budget.py @@ -0,0 +1,65 @@ +"""A per-request LLM-call budget — F-08. + +`RagAgent.handle()` makes up to ~5 sequential Bedrock calls per turn +(understand, sufficiency, generate, up to 2 entailment retries) with no +aggregate deadline before this: each call is bounded only by its own fixed +provider timeout (`read_timeout=60` in `adapters/bedrock_converse.py`, times +up to 3 retries at "standard" backoff — worst case several minutes for one +stuck call, let alone five). Measured live 2026-08-07: a normal answerable +turn costs ~8-9s total; nothing bounds the pathological case. + +Checked before each call, not wrapped around an already-running one — this +bounds how many MORE calls get a chance to start once time/calls run out. It +does not cancel a call already in flight past its own provider timeout; a +hard per-call cancellation would need cooperative cancellation support from +`adapters/bedrock_converse.py`'s boto3 client, a larger change than this +budget object alone. Still a real improvement: five calls each capable of +running to their own 60s+ limit, one after another, is the actual gap this +closes. +""" +from __future__ import annotations + +import time +from dataclasses import dataclass + +from .ports import AnswerGenerationUnavailable + + +class RequestBudgetExhausted(AnswerGenerationUnavailable): + """The per-request budget ran out before a call could be attempted. + + Subclasses `AnswerGenerationUnavailable` deliberately: every existing + `except AnswerGenerationUnavailable:` fail-closed/fail-open handler + already does the right thing for this with no changes — a caller that + wants to log a distinct reason (budget vs. genuine outage) catches this + subclass specifically before the general one. + """ + + +@dataclass +class RequestBudget: + deadline: float + calls_remaining: int + + @classmethod + def start(cls, max_wall_clock_ms: int, max_calls: int) -> "RequestBudget": + return cls( + deadline=time.monotonic() + max_wall_clock_ms / 1000, + calls_remaining=max_calls, + ) + + def has_budget(self) -> bool: + return self.calls_remaining > 0 and time.monotonic() < self.deadline + + def spend(self) -> None: + self.calls_remaining -= 1 + + def require(self) -> None: + """Raise if there's no budget for one more call, else spend it. + The single call site every LLM-call wrapper below should make + immediately before its actual provider call.""" + if not self.has_budget(): + raise RequestBudgetExhausted( + "request budget exhausted (calls or wall-clock deadline)" + ) + self.spend() diff --git a/apps/ai-service/rag/conversation.py b/apps/ai-service/rag/conversation.py deleted file mode 100644 index 83207ce..0000000 --- a/apps/ai-service/rag/conversation.py +++ /dev/null @@ -1,371 +0,0 @@ -"""Conversation state, and the rules for carrying context across turns. - -Pure domain. Everything here works without an LLM, which is deliberate: the -part of "understanding a follow-up" that matters clinically — *which drug is -this still about* — must be deterministic and testable, not inferred. - -Two structures with different jobs: - -`Focus` is structured and drives routing. It is what makes "còn trẻ em thì -sao?" resolvable at all. - -`summary` is prose for the generator. It records **what was discussed**, never -clinical content: a dose restated from a summary carries no citation and could -not be grounding-verified, because that check compares against retrieved -evidence and a summary is not evidence. -""" -from __future__ import annotations - -from dataclasses import dataclass, field, replace -from typing import Literal, Protocol - -# A drug named six turns ago is not context, it is a hazard: conversations -# drift, and inheriting a stale drug produces a confident answer about the -# wrong medicine. -FOCUS_TTL_TURNS = 6 - -# Three exchanges kept verbatim; older turns are folded into the summary. -RECENT_TURNS = 6 - -Role = Literal["user", "assistant"] -Verbosity = Literal["concise", "detailed"] - - -@dataclass(frozen=True) -class Turn: - role: Role - text: str - at: str - drug_id: str | None = None - section_key: str | None = None - # Storing what answered a turn is what lets the planner reuse evidence - # instead of retrieving the same section again. - evidence_ids: tuple[str, ...] = () - - -@dataclass(frozen=True) -class Focus: - """The entities a follow-up may inherit, each with the turn that set it.""" - - drug_id: str | None = None - drug_name: str | None = None - section_key: str | None = None - population: str | None = None - verbosity: Verbosity | None = None - set_at_turn: dict[str, int] = field(default_factory=dict) - - def age_of(self, name: str, turn_count: int) -> int | None: - set_at = self.set_at_turn.get(name) - return None if set_at is None else turn_count - set_at - - def is_fresh(self, name: str, turn_count: int, ttl: int = FOCUS_TTL_TURNS) -> bool: - age = self.age_of(name, turn_count) - return age is not None and age <= ttl - - def with_field(self, name: str, value, turn: int) -> "Focus": - stamps = dict(self.set_at_turn) - stamps[name] = turn - return replace(self, **{name: value}, set_at_turn=stamps) - - def expire(self, turn_count: int, ttl: int = FOCUS_TTL_TURNS) -> "Focus": - """Drops every field older than the TTL, stamps included.""" - kept = { - name: getattr(self, name) - for name in ("drug_id", "drug_name", "section_key", "population", "verbosity") - if self.is_fresh(name, turn_count, ttl) - } - stamps = { - name: at for name, at in self.set_at_turn.items() if name in kept - } - return Focus(**kept, set_at_turn=stamps) - - -@dataclass(frozen=True) -class ConversationState: - conversation_id: str - recent: tuple[Turn, ...] = () - summary: str = "" - focus: Focus = field(default_factory=Focus) - turn_count: int = 0 - # Turns evicted from `recent` since the last time `overflow()` was - # consumed and cleared (by the caller passing `pending_overflow=()` to - # `replace()` after folding them into the summary). NOT derivable from - # `recent` alone — `recent` is already capped at `window`, so comparing - # its length against `window` can never find anything (see the bug note - # on `overflow` below). Plumbing, not conversation content. - pending_overflow: tuple[Turn, ...] = () - - def append(self, turn: Turn, window: int = RECENT_TURNS) -> "ConversationState": - """Adds a turn and evicts the oldest beyond the window. - - Eviction accumulates the dropped turns into `pending_overflow` for - the caller's summariser to fold via `overflow()`, rather than - discarding them here — this type does not decide what a summary - says. Accumulates rather than overwrites because one turn commonly - triggers two `append()` calls in a row (user, then assistant); each - can evict at most one turn, and the second call must not lose the - first's. - """ - combined = (*self.recent, turn) - recent = combined[-window:] - dropped = combined[:-window] if len(combined) > window else () - return replace( - self, - recent=recent, - turn_count=self.turn_count + 1, - pending_overflow=(*self.pending_overflow, *dropped), - ) - - def overflow(self) -> tuple[Turn, ...]: - """Turns evicted from `recent` and not yet folded into the summary. - - Bug fixed 2026-08-06 (Codex review, F-06): this used to check - `len(self.recent) > window`, but `recent` is already truncated to - `window` by every `append()` call, so that comparison could never be - true — dropped turns were silently discarded and the summariser - never received them, no matter how long a conversation ran. The - caller must clear `pending_overflow` (pass `pending_overflow=()` to - `replace()`) after folding, or the same turns fold again next time. - """ - return self.pending_overflow - - def inherited(self, name: str): - """A focus value only if it is still fresh; otherwise None.""" - return getattr(self.focus, name) if self.focus.is_fresh(name, self.turn_count) else None - - -# --- follow-up resolution ----------------------------------------------------- - -# Phrases that mean "same question, different population". Longest-first for the -# same reason `sections.py` sorts that way: "phụ nữ cho con bú" must be tested -# before "phụ nữ", or the more specific reading is never reached. -POPULATION_PHRASES: dict[str, str] = { - "phụ nữ cho con bú": "phu_nu_cho_con_bu", - "người cao tuổi": "nguoi_cao_tuoi", - "phụ nữ có thai": "phu_nu_co_thai", - "người suy thận": "suy_than", - "người suy gan": "suy_gan", - "trẻ sơ sinh": "tre_so_sinh", - "người lớn": "nguoi_lon", - "bà bầu": "phu_nu_co_thai", - "trẻ nhỏ": "tre_em", - "trẻ em": "tre_em", - "người già": "nguoi_cao_tuoi", -} - -VERBOSITY_PHRASES: dict[str, Verbosity] = { - "giải thích kỹ hơn": "detailed", - "nói rõ hơn": "detailed", - "chi tiết hơn": "detailed", - "ngắn gọn": "concise", - "tóm tắt": "concise", -} - -# A turn that is only a qualifier — no drug, no attribute — is a follow-up by -# construction. These are the openers that mark one. -FOLLOWUP_MARKERS = ("còn", "thế còn", "vậy còn", "so với", "thuốc vừa", "cái đó", "nó") - -# Greetings, thanks, farewells and bare acknowledgements. A turn made up only of -# these is social, not a failed drug lookup: answering "Chưa xác định được -# thuốc" to "chào bạn" reads as broken. Longest-first so "cảm ơn nhiều" is -# stripped before "cảm ơn". -SMALLTALK_PHRASES = ( - "xin chào", "chào bạn", "chào ad", "cảm ơn nhiều", "cảm ơn bạn", "cám ơn", - "cảm ơn", "tạm biệt", "hay quá", "tuyệt vời", "hiểu rồi", "được rồi", - "chào", "hello", "hi", "alo", "thanks", "thank", "ok", "oke", "okie", - "ừ", "uh", "haha", "hihi", "bye", -) - - -def is_smalltalk(text: str) -> bool: - """True when a turn carries nothing but social phrases. - - Deliberately conservative: it strips every known social phrase and returns - True only if what remains is empty. "chào bạn, liều paracetamol?" keeps - "liều paracetamol" after stripping, so it is treated as a real question — - a greeting must never swallow the medical part of a turn. - """ - remainder = _normalise(text).strip(" .,!?;:") - for phrase in sorted(SMALLTALK_PHRASES, key=len, reverse=True): - # Space-pad both sides so a short phrase ("hi", "ok") matches a whole - # word only, never a substring of "chi" or "block". - remainder = f" {remainder} ".replace(f" {phrase} ", " ").strip(" .,!?;:") - return not remainder - - -def _normalise(text: str) -> str: - return " ".join(text.casefold().split()) - - -def _longest_first(phrases: dict[str, str]) -> list[tuple[str, str]]: - return sorted(phrases.items(), key=lambda item: -len(item[0])) - - -def detect_population(text: str) -> str | None: - normalised = _normalise(text) - for phrase, tag in _longest_first(POPULATION_PHRASES): - if phrase in normalised: - return tag - return None - - -def detect_verbosity(text: str) -> Verbosity | None: - normalised = _normalise(text) - for phrase, level in _longest_first(VERBOSITY_PHRASES): - if phrase in normalised: - return level - return None - - -def looks_like_followup(text: str) -> bool: - normalised = _normalise(text) - return any(normalised.startswith(marker) for marker in FOLLOWUP_MARKERS) - - -@dataclass(frozen=True) -class ResolvedQuestion: - """What this turn is asking, after the conversation is taken into account.""" - - text: str - drug_id: str | None - section_key: str | None - population: str | None - verbosity: Verbosity | None - inherited_drug: bool - inherited_section: bool - - @property - def needs_carry_over_notice(self) -> bool: - """Whether the answer must name what it inherited. - - An inherited drug that is wrong is a wrong-drug answer, so the answer - has to say which drug it decided this was about. - """ - return self.inherited_drug - - -def resolve_against( - state: ConversationState, - text: str, - drug_id: str | None, - section_key: str | None, -) -> ResolvedQuestion: - """Fills gaps in this turn from conversation focus, freshness permitting. - - `drug_id` and `section_key` are what this turn resolved on its own — the - existing resolvers decide those, unchanged. Only what the turn left blank - is inherited, so an explicit mention always wins over context. - """ - inherited_drug = False - inherited_section = False - - if drug_id is None: - carried = state.inherited("drug_id") - if carried is not None: - drug_id, inherited_drug = carried, True - - if section_key is None: - carried = state.inherited("section_key") - if carried is not None: - section_key, inherited_section = carried, True - - population = detect_population(text) or state.inherited("population") - verbosity = detect_verbosity(text) or state.inherited("verbosity") - - return ResolvedQuestion( - text=text, - drug_id=drug_id, - section_key=section_key, - population=population, - verbosity=verbosity, - inherited_drug=inherited_drug, - inherited_section=inherited_section, - ) - - -def update_focus( - state: ConversationState, - resolved: ResolvedQuestion, -) -> Focus: - """Focus after this turn, stamped with the current turn index.""" - focus = state.focus.expire(state.turn_count) - turn = state.turn_count - for name, value in ( - ("drug_id", resolved.drug_id), - ("section_key", resolved.section_key), - ("population", resolved.population), - ("verbosity", resolved.verbosity), - ): - if value is not None: - focus = focus.with_field(name, value, turn) - return focus - - -# --- persistence and summary -------------------------------------------------- -# -# Protocol + no-LLM default co-located, matching how `reasoning.py` ships -# `SufficiencyAssessor`/`DeterministicAssessor` and `metrics.py` ships -# `Metrics`/`NullMetrics`. The Postgres-backed store lives in `adapters/`. - - -class ConversationStore(Protocol): - """Loads and persists one conversation's state. - - `load` returns a fresh empty state for an unknown id rather than raising: a - first turn has no prior state, and that is not an error. - """ - - def load(self, conversation_id: str) -> "ConversationState": ... - def save(self, state: "ConversationState") -> None: ... - - -class InMemoryConversationStore: - """Reference implementation and the offline/test default.""" - - def __init__(self) -> None: - self._states: dict[str, ConversationState] = {} - - def load(self, conversation_id: str) -> ConversationState: - return self._states.get(conversation_id, ConversationState(conversation_id)) - - def save(self, state: ConversationState) -> None: - self._states[state.conversation_id] = state - - -class Summariser(Protocol): - """Folds turns evicted from the recent window into rolling prose. - - Contract, load-bearing for safety: the summary records *what was discussed*, - never a clinical value. A dose copied into a summary carries no citation and - cannot be grounding-verified — the check compares against retrieved - evidence, and a summary is not evidence. - """ - - def fold(self, prev_summary: str, dropped: tuple["Turn", ...]) -> str: ... - - -class DeterministicSummariser: - """No-LLM default: one topic line per evicted user turn, capped. - - Records only the drug and section a turn was *about* — labels, never cell - values — so the no-clinical-content rule holds by construction rather than - by trusting a generator not to leak a dose. - """ - - MAX_CHARS = 1600 # ~400 tokens, per ADR 0007 §2 - - def fold(self, prev_summary: str, dropped: tuple[Turn, ...]) -> str: - lines = [prev_summary] if prev_summary else [] - for turn in dropped: - if turn.role != "user": - continue - drug = turn.drug_id or "thuốc chưa xác định" - section = turn.section_key or "thông tin chung" - lines.append(f"- đã hỏi {section} của {drug}") - text = "\n".join(lines) - # Keep the most recent topics when over budget: drop oldest lines, not - # mid-line characters, so the summary never ends on a fragment. - while len(text) > self.MAX_CHARS and len(lines) > 1: - lines.pop(0) - text = "\n".join(lines) - return text diff --git a/apps/ai-service/rag/conversational.py b/apps/ai-service/rag/conversational.py deleted file mode 100644 index eb71e69..0000000 --- a/apps/ai-service/rag/conversational.py +++ /dev/null @@ -1,424 +0,0 @@ -"""Orchestration: turns a stateless single-turn engine into a conversation. - -This is the glue ADR 0007 specified and nothing yet called. It owns no rules of -its own — inheritance lives in `conversation.py`, the bounded loop in -`reasoning.py`, grounding in `grounding.py`. Its whole job is the sequence: - - load state - → resolve this turn, then inherit gaps from focus - → derive clarify signals from resolver state (never a model score) - → run the bounded loop (retrieve / generate / verify) - → update focus, append turns, summarise overflow, save - → name any inherited drug in the answer - -Everything here runs with no LLM and no live service: the collaborators are -protocols, so a turn can be exercised end-to-end with fakes. -""" -from __future__ import annotations - -from dataclasses import dataclass, replace -from typing import Protocol - -from . import metrics as metric_names -from .answer import GroundedAnswer, GroundedAnswerService -from .conversation import ( - ConversationState, - ConversationStore, - Summariser, - Turn, - is_smalltalk, - resolve_against, - update_focus, -) -from .metrics import Metrics, NullMetrics -from .models import EvidenceDecision, QueryIntent, SubjectScope -from .reasoning import ( - BudgetExhausted, - Clarification, - ClarifyReason, - DeterministicAssessor, - Generate, - LoopOutcome, - MAX_RETRIEVAL_ROUNDS, - Retrieve, - SufficiencyAssessor, - TurnBudget, - clarify_for, - run_turn, -) -from .routing import CatalogDrugResolver, DrugResolutionStatus, normalize_name -from .sections import SectionResolver - -# Turns that only confirm a prior suggestion. They resolve no drug and must not -# be fuzzy-matched against the catalog (which returns garbage like terbinafin). -# Stored normalised (normalize_name strips diacritics: "đúng" -> "dung"), or the -# lookup below never matches. -_CONFIRMATION_WORDS = ( - "đúng", "đúng rồi", "đúng vậy", "phải", "phải rồi", "chuẩn", "chuẩn rồi", - "chính xác", "ừ", "uh", "ok", "oke", "yes", "vâng", -) -_CONFIRMATIONS = frozenset(normalize_name(word) for word in _CONFIRMATION_WORDS) - - -def _is_confirmation(text: str) -> bool: - return normalize_name(text) in _CONFIRMATIONS - -SUMMARY_EVERY = 4 # regenerate the summary at most every S turns, per ADR 0007 §2 - - -@dataclass(frozen=True) -class TurnResolution: - """What one turn resolved on its own, before conversation is considered. - - `drug_status` is the resolver's verdict — resolved / not_found / ambiguous — - kept distinct from `drug_id` so an ambiguous turn (asks which drug) reads - differently from a bare follow-up (inherits the drug). - """ - - drug_id: str | None - section_key: str | None - drug_status: str - - -class TurnResolverPort(Protocol): - def resolve_turn(self, text: str) -> TurnResolution: ... - - -@dataclass(frozen=True) -class TurnResponse: - answer: str | None - clarification: Clarification | None - evidence_texts: tuple[str, ...] - stopped_because: str - inherited_drug: str | None - generated: bool - - -class ConversationalRagService: - def __init__( - self, - store: ConversationStore, - summariser: Summariser, - resolver: TurnResolverPort, - retrieve: Retrieve, - generate: Generate, - metrics: Metrics | None = None, - summary_every: int = SUMMARY_EVERY, - ) -> None: - self._store = store - self._summariser = summariser - self._resolver = resolver - self._retrieve = retrieve - self._generate = generate - self._metrics = metrics or NullMetrics() - self._summary_every = summary_every - - def answer( - self, conversation_id: str, text: str, budget: TurnBudget | None = None - ) -> TurnResponse: - state = self._store.load(conversation_id) - - turn = self._resolver.resolve_turn(text) - resolved = resolve_against(state, text, turn.drug_id, turn.section_key) - - signals = self._clarify_signals(resolved, turn) - if resolved.inherited_drug: - self._metrics.increment(metric_names.FOLLOWUP_INHERITED) - - outcome = run_turn( - state, - resolved, - self._retrieve, - self._generate, - clarify_signals=signals, - budget=budget or TurnBudget(), - metrics=self._metrics, - ) - - self._persist(state, resolved, outcome) - - answer = outcome.answer - inherited = resolved.drug_id if resolved.needs_carry_over_notice else None - if answer is not None and inherited is not None: - # An inherited drug that is wrong is a wrong-drug answer, so the - # answer has to say which drug it decided this was about. - answer = f"Về {inherited}: {answer}" - - return TurnResponse( - answer=answer, - clarification=outcome.clarification, - evidence_texts=outcome.evidence_texts, - stopped_because=outcome.stopped_because, - inherited_drug=inherited, - generated=outcome.generated, - ) - - @staticmethod - def _clarify_signals(resolved, turn: TurnResolution) -> tuple[str, ...]: - """Resolver states that should ask instead of guess. - - Only fires when the drug is *still* unknown after inheritance: a - follow-up like "còn trẻ em thì sao?" names no drug but inherits one, and - must not be turned into a clarify. - """ - if resolved.drug_id is None: - return (ClarifyReason.AMBIGUOUS_DRUG,) - return () - - def _persist( - self, state: ConversationState, resolved, outcome: LoopOutcome - ) -> None: - focus = update_focus(state, resolved) - state = ConversationState( - conversation_id=state.conversation_id, - recent=state.recent, - summary=state.summary, - focus=focus, - turn_count=state.turn_count, - ) - state = state.append( - Turn("user", resolved.text, _now(), resolved.drug_id, resolved.section_key) - ) - if outcome.answer is not None: - state = state.append( - Turn( - "assistant", - outcome.answer, - _now(), - resolved.drug_id, - resolved.section_key, - evidence_ids=tuple(str(i) for i in range(len(outcome.evidence_texts))), - ) - ) - if state.turn_count % self._summary_every == 0 and state.overflow(): - summary = self._summariser.fold(state.summary, state.overflow()) - state = ConversationState( - conversation_id=state.conversation_id, - recent=state.recent, - summary=summary, - focus=state.focus, - turn_count=state.turn_count, - ) - self._store.save(state) - - -def _now() -> str: - # Timestamps are provenance, not logic; the domain never branches on them, - # so a monotonic placeholder keeps this module free of wall-clock coupling. - return "" - - -# --- live chat core ----------------------------------------------------------- -# -# The deployable multi-turn path. The loop is what *understands and clarifies* -# a turn; retrieval, citation, VERIFY_PDF and grounding stay inside -# GroundedAnswerService, untouched — so clarify + refine are added *around* the -# safe engine, never inside it. - -SMALLTALK_REPLY = ( - "Mình tra cứu Dược thư Quốc gia Việt Nam. Bạn muốn hỏi về thuốc nào, " - "hoặc thuộc tính nào (liều dùng, chống chỉ định, tương tác…)?" -) - - -@dataclass(frozen=True) -class ConversationTurnResult: - answer: str | None - clarification: Clarification | None - grounded: GroundedAnswer | None - smalltalk: bool - inherited_drug: str | None - reason: str - - -class ConversationalLoopService: - def __init__( - self, - answers: GroundedAnswerService, - resolver: CatalogDrugResolver, - section_resolver: SectionResolver, - store: ConversationStore, - assessor: SufficiencyAssessor | None = None, - summariser: Summariser | None = None, - metrics: Metrics | None = None, - ) -> None: - self._answers = answers - self._resolver = resolver - self._section_resolver = section_resolver - self._store = store - self._assessor = assessor or DeterministicAssessor() - self._summariser = summariser - self._metrics = metrics or NullMetrics() - - def answer( - self, - conversation_id: str, - query: str, - subject_scope: SubjectScope, - intent: QueryIntent, - budget: TurnBudget | None = None, - ) -> ConversationTurnResult: - state = self._store.load(conversation_id) - - resolution = self._resolver.resolve(query) - # Only an EXACT name is auto-accepted. A fuzzy match (score < 1.0) is a - # guess, and a formulary must not silently answer about a *different* - # drug than the one meant — a typo is asked about ("did you mean…?"), - # never resolved on a similarity threshold. Autocomplete at input is the - # first line; this is the backstop when a wrong name is still submitted. - is_exact = resolution.status == DrugResolutionStatus.RESOLVED and ( - resolution.score is None or resolution.score >= 0.999 - ) - drug_self = resolution.drug_id if is_exact else None - - # Social turn that names no drug: answer as a person, not a failed lookup. - if drug_self is None and is_smalltalk(query): - self._append_user(state, query, None, None) - return ConversationTurnResult( - SMALLTALK_REPLY, None, None, True, None, "smalltalk" - ) - - section = self._section_resolver.resolve(query) - section_self = section.section_key if section else None - resolved = resolve_against(state, query, drug_self, section_self) - - # Clarify beats guessing: no drug even after inheritance. If the text is - # a near-miss for real drug names, offer them ("did you mean") rather - # than a bare "which drug?" — a typo should not dead-end. - if resolved.drug_id is None: - # A bare confirmation ("đúng") with no drug in context is not a drug - # lookup — never fuzzy-match it (that returned terbinafin/tretinoin). - if _is_confirmation(query): - reason = "confirm_without_context" - clarification = Clarification( - reason=reason, - question="Bạn muốn xác nhận thuốc nào? Vui lòng gõ tên thuốc để mình tra cứu.", - options=(), - ) - self._metrics.increment(metric_names.CLARIFY_ASKED, reason=reason) - self._persist(state, resolved, None) - return ConversationTurnResult(None, clarification, None, False, None, reason) - # Only offer "did you mean" for a SHORT, drug-name-shaped miss (a - # typo). Fuzzy-matching a whole sentence ("EPO điều trị thiếu máu…") - # or a confirmation ("đúng") against 684 aliases returns confident - # garbage — that is the did-you-mean loop the reviewer hit. A long or - # confirming turn that resolves no drug is answered honestly, not - # with a list of unrelated drugs. - looks_like_name = len(normalize_name(query).split()) <= 4 - suggestions = ( - self._resolver.suggest(query, k=3, min_score=0.72) - if looks_like_name and not _is_confirmation(query) - else [] - ) - if suggestions: - names = [self._drug_name(drug_id) for drug_id, _ in suggestions] - reason = "did_you_mean" - clarification = Clarification( - reason=reason, - question=f"Ý bạn là: {', '.join(names)}?", - options=tuple(names), - ) - else: - reason = "drug_not_supported" - clarification = Clarification( - reason=reason, - question=( - "Không có thuốc này trong Dược thư Quốc gia. Vui lòng kiểm " - "tra lại tên, hoặc gõ vài ký tự để chọn từ gợi ý." - ), - options=(), - ) - self._metrics.increment(metric_names.CLARIFY_ASKED, reason=reason) - self._persist(state, resolved, None) - return ConversationTurnResult(None, clarification, None, False, None, reason) - if resolved.inherited_drug: - self._metrics.increment(metric_names.FOLLOWUP_INHERITED) - - # One call to the safe engine. The drug is passed already-resolved (incl. - # an inherited follow-up drug), so the engine does NOT re-resolve it from - # the turn text — that double-resolution is what abstained follow-ups as - # "ambiguous". The turn's own text drives section routing; when it names - # no attribute the drug-overview + rerank path finds the relevant part. - grounded: GroundedAnswer | None = self._answers.answer( - query, subject_scope, intent, drug_id=resolved.drug_id - ) - - # The engine asked for a missing detail (age/weight/renal function/ - # indication…) rather than dumping every dose band — surface it as a - # clarification, not a final answer. - if grounded is not None and grounded.clarification is not None: - self._metrics.increment( - metric_names.CLARIFY_ASKED, reason="needs_more_info" - ) - self._persist(state, resolved, None) - return ConversationTurnResult( - None, - Clarification( - reason="needs_more_info", - question=grounded.clarification, - options=(), - ), - None, - False, - None, - "needs_more_info", - ) - - answer = grounded.answer if grounded else None - inherited = resolved.drug_id if resolved.needs_carry_over_notice else None - if answer is not None and inherited is not None: - answer = f"Về {self._drug_name(inherited)}: {answer}" - if grounded is not None: - grounded = replace(grounded, answer=answer) - - self._persist(state, resolved, grounded) - return ConversationTurnResult( - answer, - None, - grounded, - False, - inherited, - grounded.result.reason if grounded else "no_answer", - ) - - def complete(self, prefix: str, k: int = 8) -> list[str]: - """Display names matching a typed prefix, for input autocomplete.""" - return [self._drug_name(drug_id) for drug_id in self._resolver.complete(prefix, k)] - - @staticmethod - def _drug_name(drug_id: str) -> str: - """A readable display name from a drug id ('paracetamol_acetaminophen').""" - return drug_id.replace("_", " ").title() - - def _append_user(self, state, text, drug_id, section_key) -> None: - state = state.append(Turn("user", text, _now(), drug_id, section_key)) - self._store.save(state) - - def _persist(self, state, resolved, grounded) -> None: - focus = update_focus(state, resolved) - state = replace(state, focus=focus) - state = state.append( - Turn("user", resolved.text, _now(), resolved.drug_id, resolved.section_key) - ) - if grounded is not None and grounded.answer is not None: - state = state.append( - Turn( - "assistant", - grounded.answer, - _now(), - resolved.drug_id, - resolved.section_key, - ) - ) - if ( - self._summariser is not None - and state.turn_count % SUMMARY_EVERY == 0 - and state.overflow() - ): - summary = self._summariser.fold(state.summary, state.overflow()) - # Clear what was just folded — `replace()` keeps every field not - # named here, and `pending_overflow` accumulates across calls - # (see `ConversationState.append`), so leaving it would fold the - # same already-summarised turns again next time. - state = replace(state, summary=summary, pending_overflow=()) - self._store.save(state) diff --git a/apps/ai-service/rag/prompt.py b/apps/ai-service/rag/prompt.py index 30514a0..01c61b9 100644 --- a/apps/ai-service/rag/prompt.py +++ b/apps/ai-service/rag/prompt.py @@ -33,7 +33,14 @@ Quy tắc bắt buộc: [n] ở mọi câu — gắn một lần cho một cụm cùng nguồn là đủ. Không bịa số [n]. 5. Nếu BẰNG CHỨNG không đủ (thiếu đối tượng được hỏi, thiếu con số, hoặc chỉ nói chung chung), nói rõ là không đủ và đặt evidence_sufficient=false. Đó là câu - trả lời hợp lệ. Không suy diễn để lấp chỗ trống. + trả lời hợp lệ. Không suy diễn để lấp chỗ trống. TRƯỜNG HỢP NÀY BẮT BUỘC LUÔN + điền `clarifying_question` giải thích NGẮN GỌN, CỤ THỂ vì sao — dù đó là vì + người dùng chưa nêu đủ dữ kiện (hỏi lại dữ kiện còn thiếu, như quy tắc 7), hay + đơn giản là chuyên luận KHÔNG đề cập nội dung này cho đối tượng/đường dùng + đang hỏi (nói thẳng điều đó, ví dụ "Dược thư không nêu liều dùng đường nhỏ + mắt của thuốc này"). TUYỆT ĐỐI KHÔNG để `clarifying_question`=null khi + evidence_sufficient=false — một câu giải thích cụ thể luôn hữu ích hơn cho + người đọc so với việc để trống. 6. Giữ nguyên thuật ngữ chuyên môn của sách. Không diễn giải cho người không chuyên. 7. HỎI LẠI khi thiếu dữ kiện — ĐÂY LÀ QUY TẮC QUAN TRỌNG NHẤT, ưu tiên hơn việc @@ -70,8 +77,11 @@ ANSWER_SCHEMA = { "clarifying_question": { "type": ["string", "null"], "description": ( - "Câu hỏi lại khi người dùng chưa nêu đủ dữ kiện (vd tuổi/cân nặng " - "cho câu hỏi liều có nhiều mức). null nếu đã đủ dữ kiện để trả lời." + "BẮT BUỘC khi evidence_sufficient=false: câu hỏi lại khi người dùng " + "chưa nêu đủ dữ kiện (vd tuổi/cân nặng), HOẶC — nếu vấn đề là " + "chuyên luận không có nội dung này — một câu nói thẳng điều đó (vd " + "'Dược thư không nêu liều dùng đường nhỏ mắt của thuốc này'). " + "null CHỈ khi evidence_sufficient=true." ), }, }, @@ -94,18 +104,32 @@ Quy tắc: - Câu hỏi KHÔNG về liều (chống chỉ định, tương tác, tác dụng phụ, giới thiệu thuốc…) thì thường ĐỦ. -Trả về DUY NHẤT JSON: {"sufficient": bool, "clarifying_question": string|null}. +Trả về DUY NHẤT JSON: {"sufficient": bool, "clarifying_question": string|null, +"quick_replies": string[]}. Nếu CHƯA đủ: sufficient=false và clarifying_question hỏi NGẮN GỌN tất cả dữ kiện còn thiếu (vd: "Bé mấy tuổi, cân nặng bao nhiêu kg, dùng đường nào và để hạ sốt -hay giảm đau?"). Nếu đủ: sufficient=true, clarifying_question=null.""" +hay giảm đau?"). Nếu đủ: sufficient=true, clarifying_question=null, +quick_replies=[]. + +quick_replies: 2-4 phương án trả lời NGẮN (dưới ~20 ký tự mỗi phương án) cho +CHÍNH câu clarifying_question vừa đặt ra, để người dùng bấm chọn thay vì gõ — +CHỈ khi câu hỏi thực sự có vài lựa chọn rời rạc, tự nhiên (vd đối tượng: "Người +lớn" / "Trẻ em"; đường dùng: "Uống" / "Tiêm"). Để mảng RỖNG nếu câu hỏi cần một +con số cụ thể không có sẵn lựa chọn ngắn (vd hỏi cân nặng chính xác) — không +được bịa ra các phương án number-ish giả.""" SUFFICIENCY_SCHEMA = { "type": "object", "properties": { "sufficient": {"type": "boolean"}, "clarifying_question": {"type": ["string", "null"]}, + "quick_replies": { + "type": "array", + "items": {"type": "string"}, + "maxItems": 4, + }, }, - "required": ["sufficient", "clarifying_question"], + "required": ["sufficient", "clarifying_question", "quick_replies"], "additionalProperties": False, } @@ -180,7 +204,8 @@ def build_entailment_request(claims: list[tuple[str, str]]) -> "GenerationReques def build_request( - question: str, evidence_texts: tuple[str, ...], intro: bool = False + question: str, evidence_texts: tuple[str, ...], intro: bool = False, + list_mode: bool = False, ) -> GenerationRequest: """The prompt for one question over one ordered evidence list. @@ -191,6 +216,14 @@ def build_request( `intro=True` is the "user typed only a drug name" case: instead of restating a section, write a short introduction — what the drug is, its class and its main indication — then invite a specific follow-up. Still evidence-only. + + `list_mode=True` is the symptom_to_drug reverse-lookup case: each evidence + block is a DIFFERENT drug's own `chi_dinh`, not alternative phrasings of + one drug's section — found live 2026-08-07 that without this the model + picked just one drug out of 8 real matches and answered only about that + one, silently dropping the rest. Not a treatment ranking — a factual list + ([[feedback_no_recommendation_gate]] already covers why a "which is best" + framing would be wrong for this audience anyway). """ if not evidence_texts: raise ValueError("cannot build a grounded prompt with no evidence") @@ -206,6 +239,16 @@ def build_request( "lượng. Kết thúc bằng một câu mời hỏi tiếp về thuộc tính cụ thể (liều " "dùng, chống chỉ định, thận trọng, tương tác…)." ) + elif list_mode: + task = ( + f"CÂU HỎI: {question}\n\nMỖI đoạn BẰNG CHỨNG trên là chỉ định của MỘT " + "thuốc KHÁC NHAU. Hãy LIỆT KÊ TẤT CẢ các thuốc mà bằng chứng cho thấy " + "có chỉ định phù hợp với câu hỏi — không chỉ chọn một thuốc. Mỗi thuốc " + "một câu ngắn, gắn đúng số nguồn [n] của thuốc đó. Đây là liệt kê tra " + "cứu, KHÔNG phải khuyến cáo thuốc nào tốt hơn — không xếp hạng, không " + "chọn thuốc \"phù hợp nhất\". Nếu KHÔNG thuốc nào trong bằng chứng thực " + "sự phù hợp với câu hỏi, nói rõ điều đó thay vì liệt kê thuốc không liên quan." + ) else: task = f"CÂU HỎI: {question}" user = f"BẰNG CHỨNG:\n\n{blocks}\n\n{task}" diff --git a/apps/ai-service/rag/reasoning.py b/apps/ai-service/rag/reasoning.py deleted file mode 100644 index 5fe209d..0000000 --- a/apps/ai-service/rag/reasoning.py +++ /dev/null @@ -1,305 +0,0 @@ -"""The bounded reasoning loop. - -Understand → plan → retrieve → assess → refine → generate → verify → repair. -Every edge is bounded, and every budget is decremented **before** the call it -pays for, so exhaustion degrades to the best answer so far rather than to an -error. - -Two rules hold across every path and are the reason this can be added to a -formulary at all: - -- `grounding.verify` still gates every generated answer. Reasoning chooses what - to look up and how to phrase it; it is never a source of facts. -- A clarify signal bypasses the loop entirely. Asking beats guessing, and the - signals are resolver states — ambiguous drug, unresolved attribute — not a - model's confidence score. -""" -from __future__ import annotations - -from dataclasses import dataclass, field, replace -from typing import Protocol - -from . import metrics as metric_names -from .conversation import ConversationState, ResolvedQuestion -from .metrics import Metrics, NullMetrics - -MAX_RETRIEVAL_ROUNDS = 2 -MAX_REPAIRS = 1 -MAX_LLM_CALLS = 4 -MAX_WALL_CLOCK_MS = 20_000 - - -class BudgetExhausted(RuntimeError): - """Raised only inside the loop, never surfaced; the loop catches it.""" - - -@dataclass -class TurnBudget: - """Mutable on purpose: one budget is threaded through one turn.""" - - llm_calls: int = MAX_LLM_CALLS - retrieval_rounds: int = MAX_RETRIEVAL_ROUNDS - repairs: int = MAX_REPAIRS - wall_clock_ms: int = MAX_WALL_CLOCK_MS - elapsed_ms: int = 0 - - def spend_llm(self) -> None: - if self.llm_calls <= 0: - raise BudgetExhausted("llm_calls") - self.llm_calls -= 1 - - def spend_retrieval(self) -> None: - if self.retrieval_rounds <= 0: - raise BudgetExhausted("retrieval_rounds") - self.retrieval_rounds -= 1 - - def spend_repair(self) -> None: - if self.repairs <= 0: - raise BudgetExhausted("repairs") - self.repairs -= 1 - - def out_of_time(self) -> bool: - return self.elapsed_ms >= self.wall_clock_ms - - -class ClarifyReason: - AMBIGUOUS_DRUG = "ambiguous_drug" - NO_ATTRIBUTE = "no_attribute" - MULTI_ATTRIBUTE = "multi_attribute" - STILL_INSUFFICIENT = "still_insufficient" - - -@dataclass(frozen=True) -class Clarification: - reason: str - question: str - options: tuple[str, ...] = () - - -@dataclass(frozen=True) -class Sufficiency: - """The assessor's verdict on retrieved evidence. - - `missing` must name something specific — a section, a population, a second - drug. "Feels incomplete" does not buy a retrieval round; a round is only - spent when there is a concrete thing to go and fetch. - """ - - sufficient: bool - missing: str | None = None - refined_query: str | None = None - - -class SufficiencyAssessor(Protocol): - def assess( - self, resolved: ResolvedQuestion, evidence_texts: tuple[str, ...] - ) -> Sufficiency: ... - - -class DeterministicAssessor: - """The no-LLM default, and the reference for what the port must do. - - Runs offline and is what the loop uses until a provider is enabled. It only - reports insufficiency it can *demonstrate* — a population was asked for and - no retrieved text mentions it — so it can never spin the loop on a feeling. - """ - - POPULATION_TERMS = { - "nguoi_lon": ("người lớn",), - "tre_em": ("trẻ em", "trẻ nhỏ", "trẻ "), - "tre_so_sinh": ("sơ sinh",), - "phu_nu_co_thai": ("thai", "mang thai"), - "phu_nu_cho_con_bu": ("cho con bú", "sữa mẹ"), - "nguoi_cao_tuoi": ("người cao tuổi", "người già"), - "suy_than": ("suy thận", "clcr"), - "suy_gan": ("suy gan",), - } - - def assess( - self, resolved: ResolvedQuestion, evidence_texts: tuple[str, ...] - ) -> Sufficiency: - if not evidence_texts: - return Sufficiency(False, missing="no_evidence") - if resolved.population is None: - return Sufficiency(True) - - terms = self.POPULATION_TERMS.get(resolved.population, ()) - haystack = " ".join(evidence_texts).casefold() - if any(term in haystack for term in terms): - return Sufficiency(True) - return Sufficiency( - False, - missing=f"population:{resolved.population}", - refined_query=f"{resolved.text} {terms[0] if terms else ''}".strip(), - ) - - -@dataclass(frozen=True) -class LoopOutcome: - """What one turn produced, plus what it cost.""" - - answer: str | None - clarification: Clarification | None - evidence_texts: tuple[str, ...] - retrieval_rounds_used: int - repairs_used: int - stopped_because: str - generated: bool = False - - -@dataclass -class LoopTrace: - """Ordered record of stages, for the dashboard and for debugging.""" - - stages: list[str] = field(default_factory=list) - - def enter(self, stage: str) -> None: - self.stages.append(stage) - - -def clarify_for( - reason: str, options: tuple[str, ...] = () -) -> Clarification: - questions = { - ClarifyReason.NO_ATTRIBUTE: ( - "Anh/chị muốn tra thuộc tính nào của thuốc này?" - ), - ClarifyReason.AMBIGUOUS_DRUG: ( - "Câu hỏi có thể ứng với nhiều thuốc. Anh/chị muốn tra thuốc nào?" - ), - ClarifyReason.MULTI_ATTRIBUTE: ( - "Câu hỏi nhắc tới nhiều mục. Anh/chị muốn xem mục nào trước?" - ), - ClarifyReason.STILL_INSUFFICIENT: ( - "Chưa tìm đủ căn cứ trong Dược thư cho ý này. " - "Anh/chị có thể nêu rõ hơn điều cần tra không?" - ), - } - return Clarification(reason, questions[reason], options) - - -class Retrieve(Protocol): - def __call__(self, resolved: ResolvedQuestion) -> tuple[str, ...]: ... - - -class Generate(Protocol): - def __call__( - self, resolved: ResolvedQuestion, evidence: tuple[str, ...], state: ConversationState - ) -> str | None: ... - - -def run_turn( - state: ConversationState, - resolved: ResolvedQuestion, - retrieve: Retrieve, - generate: Generate, - clarify_signals: tuple[str, ...] = (), - assessor: SufficiencyAssessor | None = None, - budget: TurnBudget | None = None, - metrics: Metrics | None = None, - trace: LoopTrace | None = None, -) -> LoopOutcome: - """One conversational turn through the bounded loop. - - `clarify_signals` comes from the existing resolvers — ambiguous drug, - unresolved section, multi-attribute. They short-circuit before any spend, - because a question worth asking is cheaper and safer than a guess. - """ - budget = budget or TurnBudget() - assessor = assessor or DeterministicAssessor() - metrics = metrics or NullMetrics() - trace = trace or LoopTrace() - - trace.enter("understand") - if clarify_signals: - reason = clarify_signals[0] - metrics.increment(metric_names.CLARIFY_ASKED, reason=reason) - trace.enter("clarify") - return LoopOutcome( - answer=None, - clarification=clarify_for(reason), - evidence_texts=(), - retrieval_rounds_used=0, - repairs_used=0, - stopped_because="clarify_signal", - ) - - evidence: tuple[str, ...] = () - rounds_used = 0 - stopped = "sufficient" - - while True: - try: - budget.spend_retrieval() - except BudgetExhausted: - stopped = "retrieval_budget" - break - trace.enter("retrieve") - evidence = retrieve(resolved) - rounds_used += 1 - - trace.enter("assess") - verdict = assessor.assess(resolved, evidence) - if verdict.sufficient: - break - if budget.retrieval_rounds <= 0 or budget.out_of_time(): - stopped = "retrieval_budget" - break - # A round is spent only on a named gap with a genuinely new query. - if not verdict.missing or not verdict.refined_query: - stopped = "no_actionable_gap" - break - if verdict.refined_query == resolved.text: - stopped = "query_unchanged" - break - trace.enter("refine") - metrics.increment(metric_names.LOOP_REFINED, missing=verdict.missing) - resolved = replace(resolved, text=verdict.refined_query) - - metrics.increment(metric_names.LOOP_ROUNDS, rounds=str(rounds_used)) - - if not evidence: - trace.enter("clarify") - metrics.increment( - metric_names.CLARIFY_ASKED, reason=ClarifyReason.STILL_INSUFFICIENT - ) - return LoopOutcome( - answer=None, - clarification=clarify_for(ClarifyReason.STILL_INSUFFICIENT), - evidence_texts=(), - retrieval_rounds_used=rounds_used, - repairs_used=0, - stopped_because="no_evidence", - ) - - repairs_used = 0 - answer: str | None = None - while True: - trace.enter("generate") - try: - budget.spend_llm() - except BudgetExhausted: - stopped = "llm_budget" - break - answer = generate(resolved, evidence, state) - if answer is not None: - break - # `generate` returning None means verification already refused it. - try: - budget.spend_repair() - except BudgetExhausted: - stopped = "repair_budget" - break - repairs_used += 1 - trace.enter("repair") - metrics.increment(metric_names.LOOP_REPAIRED) - - return LoopOutcome( - answer=answer, - clarification=None, - evidence_texts=evidence, - retrieval_rounds_used=rounds_used, - repairs_used=repairs_used, - stopped_because=stopped, - generated=answer is not None, - ) diff --git a/apps/ai-service/rag/routing.py b/apps/ai-service/rag/routing.py index a5cf0a4..213f7e5 100644 --- a/apps/ai-service/rag/routing.py +++ b/apps/ai-service/rag/routing.py @@ -1,5 +1,6 @@ from __future__ import annotations +import functools import re from dataclasses import dataclass, replace from difflib import SequenceMatcher @@ -55,6 +56,20 @@ class CatalogDrugResolver: self._fuzzy_threshold = fuzzy_threshold self._ambiguity_margin = ambiguity_margin + # Measured live 2026-08-07: a single `resolve()` call over the real + # ~10,164-alias catalog costs ~0.65-0.7s, `suggest()` ~0.94-0.97s — both + # O(aliases) regex/SequenceMatcher work, pure functions of their + # arguments (only `self._aliases` et al, fixed at construction, feed + # them). `understanding.py`'s `_candidate_ids` calls both PER HISTORY + # LINE on every single turn — so the SAME already-seen history lines + # were being re-resolved from scratch every turn a conversation grew, + # ~1.6-1.7s of pure CPU per repeated line. A real user's ordinary + # multi-turn conversation was enough to exceed the 20s F-08 budget + # before the first Bedrock call ever ran, surfacing as a false + # "Dịch vụ đang gặp sự cố" — not a provider outage at all. Caching by + # exact input turns all but the newest turn's own text into a dict + # lookup on every subsequent call. + @functools.lru_cache(maxsize=4096) def resolve(self, query: str) -> DrugResolution: normalized_query = normalize_name(query) query_tokens = normalized_query.split() @@ -140,6 +155,9 @@ class CatalogDrugResolver: break return ordered + # See the comment on `resolve` above — same cost, same fix, same + # single-caller read-only usage (safe to hand back a cached list). + @functools.lru_cache(maxsize=4096) def suggest( self, query: str, k: int = 3, min_score: float = 0.5 ) -> list[tuple[str, float]]: diff --git a/apps/ai-service/rag/service.py b/apps/ai-service/rag/service.py index 22a5ff6..cd082be 100644 --- a/apps/ai-service/rag/service.py +++ b/apps/ai-service/rag/service.py @@ -32,6 +32,9 @@ class EvidencePolicy: # A free-form question about a resolved drug otherwise hands the LLM the # entire monograph; rerank trims it to the sections that actually answer. rerank_top_k: int = 6 + # symptom_to_drug: a common symptom can match far more drugs than is + # useful to show in one answer. + indication_candidate_limit: int = 8 class RetrievalService: @@ -157,6 +160,47 @@ class RetrievalService: return RetrievalResult(EvidenceDecision.ABSTAIN, "insufficient_retrieval_score") return self._decide(self._hydrate(self._rerank(query, hits))) + def retrieve_by_indication(self, indication_text: str) -> RetrievalResult: + """Reverse lookup: symptom/indication -> candidate drugs. + + Keyword match first (deterministic, precise — nothing here can + fabricate a drug that doesn't genuinely mention the indication). + Dense-vector search over `chi_dinh` only is the fallback, tried + only when the keyword pass finds nothing, to catch a paraphrase the + book's own wording doesn't share. This is the one place in the live + path dense search is actually used — see ADR 0008. + """ + if not indication_text.strip(): + return RetrievalResult(EvidenceDecision.ABSTAIN, "missing_indication") + + find_by_indication = getattr(self._retriever, "find_by_indication", None) + hits = ( + find_by_indication(indication_text, self._policy.indication_candidate_limit) + if find_by_indication is not None + else [] + ) + if not hits: + search_indication = getattr(self._retriever, "search_indication", None) + if search_indication is not None: + try: + hits = search_indication( + indication_text, self._policy.indication_candidate_limit + ) + except QueryEmbeddingUnavailable: + hits = [] + # Dense search always returns its nearest neighbours, even for + # an indication the corpus has nothing on — verified live: a + # made-up phrase still got 8 unrelated "matches". A weak top + # score means those neighbours aren't really about the + # question, so don't spend a generation call finding that out + # the slow way; abstain here, the same bar `retrieve()`'s own + # dense fallback already applies. + if hits and hits[0].score < self._policy.minimum_score: + hits = [] + if not hits: + return RetrievalResult(EvidenceDecision.ABSTAIN, "no_indication_match") + return self._decide(self._hydrate(hits, limit=None)) + @staticmethod def _is_question(query: str) -> bool: """A bare drug name (one or two tokens) wants the whole monograph; more @@ -206,8 +250,29 @@ class RetrievalService: if match is None: return None hits = find_by_section(drug_id, match.section_key) + if match.section_key == "than_trong": + # A "thận trọng" question about a specific condition sometimes has + # its real answer filed under "chống chỉ định" instead — found live + # 2026-08-10: Aspirin's own "thận trọng" text never says "loét dạ + # dày", the fact only exists in its "chống chỉ định" text ("loét + # dạ dày hoặc tá tràng đang hoạt động"). The two are the closest + # pair of "is this safe for my patient" categories the book has, + # and chống chỉ định text is short — pooling it costs nothing on + # a drug where than_trong already answers, and prevents a false + # "not in this source" clarify/abstain on one where it doesn't. + hits = hits + find_by_section(drug_id, "chong_chi_dinh") return hits or None + def decide(self, evidence: tuple[Evidence, ...]) -> RetrievalResult: + """Public entry point for a caller that assembles its own evidence + pool across several `retrieve_framed` calls — e.g. `RagAgent`'s + 2-drug interaction path — and needs the same quarantine/provenance + policy applied to the combined pool that a single call already gets. + Bypassing this (hand-rolling `RetrievalResult(ANSWERABLE, ...)`) is + exactly how the interaction path silently dropped a quarantined + drug's evidence instead of surfacing VERIFY_PDF for it.""" + return self._decide(evidence) + def _decide( self, evidence: tuple[Evidence, ...], is_drug_overview: bool = False ) -> RetrievalResult: diff --git a/apps/ai-service/rag/understanding.py b/apps/ai-service/rag/understanding.py index ed34a6a..7af9b97 100644 --- a/apps/ai-service/rag/understanding.py +++ b/apps/ai-service/rag/understanding.py @@ -35,9 +35,15 @@ stub runs the whole path offline in tests. from __future__ import annotations import json -from dataclasses import dataclass, field +import logging +from dataclasses import dataclass, field, replace from typing import Protocol, Sequence +from .budget import RequestBudget +from .ports import AnswerGenerationUnavailable + +logger = logging.getLogger(__name__) + # The 19 monograph section keys, kept here as the closed vocabulary the model may # use for `attribute`. Adding a new section is one entry, not a code change. SECTION_KEYS = ( @@ -126,8 +132,22 @@ class QueryFrame: weight_kg: float | None = None age_text: str | None = None indication: str | None = None # symptom/disease, for symptom_to_drug + route: str | None = None # e.g. "uong", "tiem_tinh_mach", "dat_truc_trang" needs_clarify: bool = False clarify_reason: str | None = None + # Short suggested replies for `clarify_reason` (e.g. ("Người lớn", "Trẻ + # em")) — only when the model judged the missing detail has a few + # natural discrete answers; often empty (e.g. a question needing a + # specific weight has no clean short options). + quick_replies: tuple[str, ...] = () + # Set only when `needs_clarify` fired because of a real technical + # failure (provider outage, malformed model output) rather than the + # model genuinely judging the turn under-specified. Found live + # 2026-08-07: both cases produced the exact same `reason="needs_more_info"` + # downstream, making a real, diagnosable outage indistinguishable from an + # ordinary clarifying question in the API response and trace — this lets + # `RagAgent._route` surface the real cause instead. + system_error: str | None = None raw: dict = field(default_factory=dict, compare=False) @@ -147,8 +167,24 @@ FRAME_SCHEMA = { ), "age_text": "the age exactly as stated (e.g. '3 tuổi', '5 tháng'), else null", "indication": "the symptom or disease if turn_type is symptom_to_drug, else null", + "route": ( + "route of administration if stated or implied, normalized to one of: " + "uong | tiem_tinh_mach | tiem_bap | tiem_duoi_da | dat_truc_trang | " + "boi_ngoai_da | nho_mat | nho_mui | khac, else null. A bare reply like " + "'uống' or 'tiêm' to your own prior clarify question about route IS " + "this field — read it here, do not leave it null and re-ask." + ), "needs_clarify": "true only if the turn cannot be acted on without more info", "clarify_reason": "short Vietnamese question to ask, or null", + "quick_replies": ( + "2-4 short suggested replies (each under ~20 chars) to your OWN " + "clarify_reason, for the user to tap instead of typing — ONLY when " + "clarify_reason genuinely has a few natural discrete answers (e.g. " + "['Người lớn', 'Trẻ em'] or ['Uống', 'Tiêm']). Empty list [] if the " + "missing detail needs a specific free-form value (e.g. an exact " + "weight) with no clean short options — never invent numeric-ish " + "options." + ), } _SYSTEM = """Bạn là bộ HIỂU CÂU HỎI cho một chatbot tra cứu Dược thư Quốc gia Việt Nam. @@ -169,7 +205,39 @@ Quy tắc bắt buộc: không kèm đơn vị khác) NGHĨA LÀ 30 kg -> điền weight_kg=30, không bỏ trống. - Lượt nối tiếp ("còn liều thì sao", "nó dùng cho trẻ em?") -> dùng LỊCH SỬ để biết thuốc đang nói tới và điền vào "drugs". -- Chào hỏi/vu vơ -> "smalltalk". Ngoài phạm vi chuyên luận thuốc -> "out_of_scope".""" +- QUAN TRỌNG — lượt hiện tại trả lời câu hỏi bạn VỪA hỏi: nếu dòng "Trợ lý:" cuối + cùng trong LỊCH SỬ là một câu hỏi làm rõ (vd "Người lớn hay trẻ em?", "Uống hay + tiêm?", "Cân nặng bao nhiêu kg?"), và CÂU HỎI HIỆN TẠI là một câu trả lời ngắn + hợp lý cho đúng câu đó (vd "Uống", "Người lớn", "30kg") — hãy đọc nó là câu trả + lời, điền vào field tương ứng (route/population/weight_kg/age_text), giữ lại các + field đã biết từ các lượt trước đó trong LỊCH SỬ (đừng bỏ trống lại), và CHỈ đặt + needs_clarify=true với PHẦN THÔNG TIN CÒN THIẾU KHÁC (nếu có) — TUYỆT ĐỐI KHÔNG + lặp lại nguyên văn clarify_reason đã được trả lời. Nếu sau khi điền, đã đủ dữ + kiện (đối tượng + đường dùng, và tuổi/cân nặng nếu là trẻ em) thì needs_clarify=false. + Nếu có khối "THÔNG TIN ĐÃ XÁC ĐỊNH TỪ CÁC LƯỢT TRƯỚC" bên dưới, các mục đó ĐÃ + ĐƯỢC XÁC NHẬN — không hỏi lại, không đặt needs_clarify=true vì thiếu đúng mục + đã liệt kê ở đó. +- NGƯỢC LẠI — nếu CÂU HỎI HIỆN TẠI là một câu hỏi y khoa MỚI, KHÔNG phải một câu + trả lời ngắn cho câu hỏi làm rõ gần nhất (không khớp loại thông tin vừa hỏi) và + KHÔNG nhắc lại thuốc/triệu chứng nào đã có trong LỊCH SỬ hay khối "THÔNG TIN ĐÃ + XÁC ĐỊNH": đây là LƯỢT MỚI HOÀN TOÀN — TUYỆT ĐỐI KHÔNG mang "drugs"/ + "population"/"weight_kg"/"age_text"/"route"/"indication" của lượt trước sang lượt + này, chỉ điền những gì thực sự có trong CÂU HỎI HIỆN TẠI. Ví dụ: lượt trước đang + hỏi về Omeprazol nhưng câu hiện tại là "tôi bị đau đầu nên uống thuốc gì" (không + nhắc Omeprazol) -> chủ đề mới, "drugs" phải để trống trừ khi có thuốc thực sự + được nhắc trong câu này. +- Nếu CÂU HỎI HIỆN TẠI là một lời PHỦ ĐỊNH/SỬA LẠI câu trả lời vừa rồi (vd "tôi + có hỏi X đâu", "tôi không hỏi vậy", "đâu phải thế", "ý tôi không phải vậy", + "sai rồi") — đây là dấu hiệu bạn vừa hiểu SAI ý người dùng ở lượt trước. + TUYỆT ĐỐI KHÔNG lặp lại đúng route/population/thuộc tính vừa trả lời (đã bị + từ chối): đặt needs_clarify=true và hỏi lại thật ngắn gọn, cụ thể người dùng + thực sự muốn hỏi điều gì (vd "Anh/chị muốn hỏi đường dùng nào ạ?"), không tự + suy đoán lại giá trị cũ. +- Chào hỏi/vu vơ -> "smalltalk". Ngoài phạm vi chuyên luận thuốc -> "out_of_scope". +- Khi needs_clarify=true, kèm "quick_replies": 2-4 phương án NGẮN cho câu hỏi lại + đó, CHỈ khi nó thực sự có vài lựa chọn rời rạc tự nhiên (vd đối tượng: "Người + lớn"/"Trẻ em"). Để mảng rỗng nếu cần một giá trị cụ thể không có lựa chọn ngắn + (vd hỏi cân nặng chính xác) — không bịa phương án dạng số.""" class JsonLlm(Protocol): @@ -181,7 +249,11 @@ class JsonLlm(Protocol): class QueryUnderstander(Protocol): def understand( - self, turn: str, history: Sequence[str] = () + self, + turn: str, + history: Sequence[str] = (), + budget: RequestBudget | None = None, + prior_frame: QueryFrame | None = None, ) -> QueryFrame: ... @@ -240,7 +312,13 @@ class LlmQueryUnderstander: ids.add(drug_id) return ids - def understand(self, turn: str, history: Sequence[str] = ()) -> QueryFrame: + def understand( + self, + turn: str, + history: Sequence[str] = (), + budget: RequestBudget | None = None, + prior_frame: QueryFrame | None = None, + ) -> QueryFrame: shown = { drug_id: self._catalog[drug_id] for drug_id in self._candidate_ids(turn, history) @@ -255,6 +333,7 @@ class LlmQueryUnderstander: "LỊCH SỬ HỘI THOẠI (cũ -> mới):\n" + "\n".join(history) if history else "LỊCH SỬ HỘI THOẠI: (chưa có)" ) + known_block = _known_facts_block(prior_frame) user = ( f"DANH SÁCH THUỐC ỨNG VIÊN cho lượt này (drug_id\\ttên) — CHỈ được chọn " f"drug_id từ đây, đây KHÔNG phải toàn bộ Dược thư, chỉ là các thuốc khớp " @@ -263,11 +342,39 @@ class LlmQueryUnderstander: "CÁC SECTION KEY hợp lệ cho 'attribute' (key: ý nghĩa):\n" + "\n".join(f"{key}: {SECTION_KEY_HINTS[key]}" for key in SECTION_KEYS) + "\n\n" + f"{known_block}" f"{history_block}\n\n" f"CÂU HỎI HIỆN TẠI: {turn}" ) - raw_text = self._llm.generate(_SYSTEM, user, FRAME_SCHEMA) - return self._parse(raw_text, shown) + # Found live 2026-08-07 (F-10 adversarial pass): unlike every other + # LLM call site in this product (`answer.py`'s sufficiency/generate/ + # entailment all catch this), this one call had no error handling at + # all — a provider outage here propagated straight through + # `RagAgent.handle()` and `routers/rag.py` (which only wraps the + # trace-save call, not `agent.handle()`) into an unhandled 500, + # rather than the graceful abstain every other failure mode gets. + try: + if budget is not None: + budget.require() + raw_text = self._llm.generate(_SYSTEM, user, FRAME_SCHEMA) + except AnswerGenerationUnavailable as exc: + # Found live 2026-08-07: this except block silently swallowed + # the real exception entirely — no log line anywhere — so a + # genuine provider outage (throttling, timeout, IAM, whatever) + # left zero trace to diagnose from. Now logged with the actual + # exception, and tagged with a `system_error` code distinct from + # an ordinary clarify (see `QueryFrame.system_error`). + logger.warning( + "understanding call failed (%s): %s", type(exc).__name__, exc + ) + return QueryFrame( + turn_type="out_of_scope", + needs_clarify=True, + clarify_reason="Dịch vụ đang gặp sự cố tạm thời, vui lòng thử lại " + "sau ít phút.", + system_error="understanding_provider_unavailable", + ) + return _merge_with_prior_frame(self._parse(raw_text, shown), prior_frame) @staticmethod def _resolve_id(value: str, shown: dict[str, str]) -> str | None: @@ -291,11 +398,15 @@ class LlmQueryUnderstander: try: data = json.loads(raw_text) except (json.JSONDecodeError, TypeError): - # Fail closed to a clarify rather than to a wrong reading. + # Fail closed to a clarify rather than to a wrong reading. Not a + # provider outage (the call succeeded) — the model's own output + # didn't parse, a distinct, separately diagnosable cause. + logger.warning("understanding call returned unparseable JSON: %r", raw_text) return QueryFrame( turn_type="out_of_scope", needs_clarify=True, clarify_reason="Xin lỗi, tôi chưa hiểu rõ câu hỏi. Anh/chị hỏi lại giúp nhé?", + system_error="understanding_malformed_output", ) resolved = [ (d, self._resolve_id(d, shown)) for d in _as_list(data.get("drugs")) @@ -322,12 +433,86 @@ class LlmQueryUnderstander: weight_kg=_clean_float(data.get("weight_kg")), age_text=_clean_str(data.get("age_text")), indication=_clean_str(data.get("indication")), + route=_clean_str(data.get("route")), needs_clarify=bool(data.get("needs_clarify")), clarify_reason=_clean_str(data.get("clarify_reason")), + quick_replies=tuple(_as_list(data.get("quick_replies"))), raw=data if isinstance(data, dict) else {}, ) +_KNOWN_FACT_LABELS: tuple[tuple[str, str], ...] = ( + ("population", "Đối tượng"), + ("age_text", "Tuổi"), + ("route", "Đường dùng"), + ("indication", "Chỉ định/triệu chứng"), + ("attribute", "Thuộc tính đang tra"), +) + + +def _known_facts_block(prior_frame: QueryFrame | None) -> str: + """The structured "already established" summary shown to the model on a + clarify-continuation turn. + + Found live 2026-08-07 (50-question hand-typed browser audit): relying on + the model to re-derive the WHOLE frame from raw text history every turn + is fragile — reproduced 3 times independently (Insulin storage, weight- + based Azithromycin dosing, a headache question mislabeled OMEPRAZOL) as + either a non-terminating re-ask of an already-answered clarify question, + or a stale drug bleeding into an unrelated new topic. Stating the known + fields explicitly, as data rather than asking the model to infer them + from a growing text transcript, removes most of the guesswork; `_merge_ + with_prior_frame` below is the code-level backstop for whatever the + model still drops. + """ + if prior_frame is None or not prior_frame.needs_clarify: + return "" + parts = [] + if prior_frame.drugs: + parts.append(f"Thuốc: {', '.join(prior_frame.drugs)}") + if prior_frame.weight_kg is not None: + parts.append(f"Cân nặng: {prior_frame.weight_kg:g} kg") + for field_name, label in _KNOWN_FACT_LABELS: + value = getattr(prior_frame, field_name) + if value: + parts.append(f"{label}: {value}") + if not parts: + return "" + return ( + "THÔNG TIN ĐÃ XÁC ĐỊNH TỪ CÁC LƯỢT TRƯỚC (dữ liệu CÓ THẬT, đã xác nhận " + "— KHÔNG hỏi lại các mục này; nếu câu hỏi hiện tại là một chủ đề mới " + "không liên quan, hãy bỏ qua khối này thay vì gán nhầm vào lượt mới):\n" + + "\n".join(parts) + "\n\n" + ) + + +def _merge_with_prior_frame(frame: QueryFrame, prior_frame: QueryFrame | None) -> QueryFrame: + """Code-level backstop for the model dropping an already-known field. + + Only fires when the prior turn was itself a clarify (there is something + to continue) and this turn's own `drugs` extraction agrees with it (empty, + meaning a short reply like "20kg"/"Uống" that names no drug of its own, or + an exact match) — a turn that resolves a DIFFERENT drug is a genuine topic + change and must not inherit stale population/weight/route/indication from + the old one (the headache/OMEPRAZOL bleed this guards against runs the + other way: don't let old fields survive into an unrelated new drug either). + """ + if prior_frame is None or not prior_frame.needs_clarify: + return frame + if frame.drugs and frame.drugs != prior_frame.drugs: + return frame + return replace( + frame, + drugs=frame.drugs or prior_frame.drugs, + population=frame.population or prior_frame.population, + age_text=frame.age_text or prior_frame.age_text, + weight_kg=frame.weight_kg if frame.weight_kg is not None else prior_frame.weight_kg, + route=frame.route or prior_frame.route, + indication=frame.indication or prior_frame.indication, + attribute=frame.attribute or prior_frame.attribute, + ) + + def _as_list(value) -> list[str]: if isinstance(value, str): return [value] if value.strip() else [] diff --git a/apps/ai-service/routers/rag.py b/apps/ai-service/routers/rag.py index ad8f07d..bb06e52 100644 --- a/apps/ai-service/routers/rag.py +++ b/apps/ai-service/routers/rag.py @@ -35,6 +35,9 @@ class CitationResponse(BaseModel): bbox: tuple[float, float, float, float] | None = None source_crop: str | None = None attachment: str | None = None + # The exact retrieved chunk text this citation stands for — lets the UI + # show precisely what was retrieved, not a client-side guess at it. + evidence_text: str = "" class RagQueryResponse(BaseModel): @@ -44,6 +47,16 @@ class RagQueryResponse(BaseModel): answer: str | None resolved_drug_id: str | None citations: list[CitationResponse] + # Whether `answer` is an LLM paraphrase (verified by grounding + + # entailment) or a verbatim extractive quote of the retrieved source — + # the UI shows these differently so a clinician knows which they're + # reading. + generated: bool = False + # Short suggested replies for a `decision == "clarify"` turn (e.g. + # ["Người lớn", "Trẻ em"]) — only populated by the sufficiency-check + # clarify path today; other clarify sources (no_drug, dosing_calc's + # needs_clarify) leave this empty rather than fabricate options. + quick_replies: list[str] = [] def _answer_service(request: Request) -> GroundedAnswerService: @@ -91,6 +104,7 @@ def _map_citations(items) -> list[CitationResponse]: bbox=item.bbox, source_crop=item.source_crop, attachment=item.attachment, + evidence_text=item.evidence_text, ) for item in items ] @@ -128,6 +142,8 @@ def query_rag( answer = reply.clarification if reply.clarification is not None else reply.answer resolved_drug_id = ", ".join(reply.drugs) if reply.drugs else None citations = _map_citations(reply.citations) + generated = reply.generated + quick_replies = list(reply.quick_replies) else: # No generator configured (ANSWER_PROVIDER=disabled): there is no LLM # to understand a turn with, so this is retrieval-only, single-turn, @@ -138,12 +154,16 @@ def query_rag( answer = grounded.clarification resolved_drug_id = grounded.result.resolved_drug_id citations = [] + generated = False + quick_replies = list(grounded.quick_replies) else: decision = grounded.result.decision.value reason = grounded.result.reason answer = grounded.answer resolved_drug_id = grounded.result.resolved_drug_id citations = _map_citations(grounded.citations) + generated = grounded.generated + quick_replies = [] # Trace persistence is fail-open (F-09): an already-computed, safe answer # must reach the caller even if Postgres is unreachable. `save()` opens a @@ -175,4 +195,6 @@ def query_rag( answer=answer, resolved_drug_id=resolved_drug_id, citations=citations, + generated=generated, + quick_replies=quick_replies, ) diff --git a/apps/ai-service/tests/test_agent.py b/apps/ai-service/tests/test_agent.py index 726ca8a..65ce196 100644 --- a/apps/ai-service/tests/test_agent.py +++ b/apps/ai-service/tests/test_agent.py @@ -8,18 +8,20 @@ Qdrant/Bedrock fixtures). """ from __future__ import annotations -from rag.agent import RagAgent +from rag.agent import MAX_CONSECUTIVE_CLARIFY, RagAgent from rag.answer import GroundedAnswerService +from rag.metrics import GENERATION_REJECTED, InMemoryMetrics from rag.models import Evidence, EvidenceDecision, RetrievalResult, SourceRef from rag.understanding import QueryFrame SOURCE = SourceRef(physical_page=100, precision="region", printed_page=100) -def _evidence(text: str) -> Evidence: +def _evidence(text: str, requires_visual_check: bool = False) -> Evidence: return Evidence( evidence_id="e0", matched_doc_id="e0", kind="prose", text=text, score=1.0, - source_refs=(SOURCE,), hydrated_from_parent=False, requires_visual_check=False, + source_refs=(SOURCE,), hydrated_from_parent=False, + requires_visual_check=requires_visual_check, ) @@ -27,7 +29,7 @@ class _FixedUnderstander: def __init__(self, frame: QueryFrame) -> None: self._frame = frame - def understand(self, turn, history=()): + def understand(self, turn, history=(), budget=None, prior_frame=None): return self._frame @@ -36,20 +38,50 @@ class _FixedRetrieval: drug_id regardless of section/query, so these tests assert routing, not retrieval (that's `test_retrieval_service.py`'s job).""" - def __init__(self, results: dict[str, RetrievalResult]) -> None: + def __init__( + self, + results: dict[str, RetrievalResult], + indication_results: dict[str, RetrievalResult] | None = None, + ) -> None: self._results = results + self._indication_results = indication_results or {} + self.calls: list[tuple[str, str | None, str]] = [] def retrieve_framed(self, drug_id, section_key, query, is_overview=False): + self.calls.append((drug_id, section_key, query)) return self._results.get( drug_id, RetrievalResult(EvidenceDecision.ABSTAIN, "not_configured") ) + def retrieve_by_indication(self, indication_text): + return self._indication_results.get( + indication_text, RetrievalResult(EvidenceDecision.ABSTAIN, "no_indication_match") + ) -def _agent(frame: QueryFrame, results: dict[str, RetrievalResult] | None = None) -> RagAgent: + def decide(self, evidence): + # Mirrors `RetrievalService.decide`'s real policy (not a stub that + # always says ANSWERABLE) so `_interaction`'s use of it is actually + # under test, not just its own call site. + if not evidence: + return RetrievalResult(EvidenceDecision.ABSTAIN, "parent_hydration_failed") + if any(item.requires_visual_check for item in evidence): + return RetrievalResult(EvidenceDecision.VERIFY_PDF, "visual_verification_required", evidence) + return RetrievalResult(EvidenceDecision.ANSWERABLE, "grounded_evidence_available", evidence) + + +def _agent( + frame: QueryFrame, + results: dict[str, RetrievalResult] | None = None, + indication_results: dict[str, RetrievalResult] | None = None, +) -> RagAgent: # `routing=None`: `answer_from_result` (the only method this path calls) # never touches it — see `rag/answer.py`. answers = GroundedAnswerService(routing=None) - return RagAgent(_FixedUnderstander(frame), _FixedRetrieval(results or {}), answers) + return RagAgent( + _FixedUnderstander(frame), + _FixedRetrieval(results or {}, indication_results), + answers, + ) def test_smalltalk_does_not_touch_retrieval(): @@ -100,6 +132,25 @@ def test_needs_clarify_frame_is_surfaced_directly(): reply = agent.handle("liều paracetamol cho trẻ em") assert reply.decision == "clarify" assert reply.clarification == "Bé mấy tuổi, cân nặng bao nhiêu kg?" + # No quick_replies on the frame -> none surfaced (a specific weight has + # no clean short options; must not be fabricated downstream). + assert reply.quick_replies == () + + +def test_needs_clarify_frame_carries_quick_replies_through(): + """This is the path real traffic actually hits (checked live): the + understanding LLM call itself sets needs_clarify/clarify_reason before + retrieval ever runs, short-circuiting `_route` — quick_replies must + survive that same short-circuit, not just the sufficiency-check path + inside `GroundedAnswerService`.""" + agent = _agent(QueryFrame( + turn_type="drug_attribute", drugs=("paracetamol",), + needs_clarify=True, clarify_reason="Người lớn hay trẻ em?", + quick_replies=("Người lớn", "Trẻ em"), + )) + reply = agent.handle("liều paracetamol") + assert reply.decision == "clarify" + assert reply.quick_replies == ("Người lớn", "Trẻ em") def test_single_drug_attribute_retrieves_and_answers(): @@ -118,6 +169,58 @@ def test_single_drug_attribute_retrieves_and_answers(): assert "500 mg" in reply.answer +def test_context_resolved_across_turns_is_folded_into_the_query(): + """The 2026-08-07 P0 (named in the 2026-08-06 audit): population/weight/ + age/route are extracted by understanding.py but were never passed into + retrieval or generation — so a reply like "Uống" three turns into a dose + conversation reached `GroundedAnswerService` as literally just "Uống", + with no notion that population=adult was already established. Fixed via + `_synthesize_query`; this asserts the synthesized text — not the bare + turn — is what retrieval and generation actually see.""" + result = RetrievalResult( + EvidenceDecision.ANSWERABLE, "grounded_evidence_available", + (_evidence("Liều uống người lớn: 500 mg."), + _evidence("Liều tiêm người lớn: 1 g.")), + resolved_drug_id="paracetamol_acetaminophen", + ) + retrieval = _FixedRetrieval({"paracetamol_acetaminophen": result}) + agent = RagAgent( + _FixedUnderstander(QueryFrame( + turn_type="dosing_calc", drugs=("paracetamol_acetaminophen",), + attribute="lieu_luong_va_cach_dung", population="nguoi_lon", + route="uong", needs_clarify=False, + )), + retrieval, + GroundedAnswerService(routing=None), + ) + agent.handle("Uống") + assert len(retrieval.calls) == 1 + _, _, query = retrieval.calls[0] + assert query.startswith("Uống") + assert "người lớn" in query + assert "uống" in query.lower() + + +def test_context_synthesis_is_a_no_op_when_the_frame_has_no_resolved_fields(): + """A fresh, fully-specified single-shot question already states its own + context — synthesis must not alter it or introduce redundant noise.""" + result = RetrievalResult( + EvidenceDecision.ANSWERABLE, "grounded_evidence_available", + (_evidence("x"), _evidence("y")), resolved_drug_id="metformin", + ) + retrieval = _FixedRetrieval({"metformin": result}) + agent = RagAgent( + _FixedUnderstander(QueryFrame( + turn_type="drug_attribute", drugs=("metformin",), + attribute="chong_chi_dinh", + )), + retrieval, + GroundedAnswerService(routing=None), + ) + agent.handle("Chống chỉ định của metformin là gì?") + assert retrieval.calls[0][2] == "Chống chỉ định của metformin là gì?" + + def test_interaction_combines_both_drugs_evidence(): warfarin = RetrievalResult( EvidenceDecision.ANSWERABLE, "grounded_evidence_available", @@ -137,6 +240,34 @@ def test_interaction_combines_both_drugs_evidence(): assert "chảy máu" in reply.answer +def test_interaction_with_one_drug_quarantined_never_answers_confidently(): + """The P0 the 2026-08-06 audit found: `_interaction` used to keep only + `ANSWERABLE` parts, so a quarantined drug's evidence (and the "table + exists, verify PDF" notice the quarantine contract requires) was + silently dropped — a confident interaction answer could omit a real + unverified contraindication table for one of the two drugs. Fixed via + `RetrievalService.decide` applied to the combined pool, the same policy + the single-drug path already uses.""" + warfarin = RetrievalResult( + EvidenceDecision.ANSWERABLE, "grounded_evidence_available", + (_evidence("Tương tác với aspirin làm tăng nguy cơ chảy máu."),), + ) + aspirin = RetrievalResult( + EvidenceDecision.VERIFY_PDF, "visual_verification_required", + (_evidence("Bảng tương tác cần đối chiếu PDF.", requires_visual_check=True),), + ) + agent = _agent( + QueryFrame(turn_type="interaction", drugs=("warfarin", "aspirin")), + {"warfarin": warfarin, "aspirin": aspirin}, + ) + reply = agent.handle("warfarin với aspirin có dùng chung được không") + # Must NOT be a confident "answerable" that silently omits aspirin's + # quarantined table — must ask for PDF verification instead. + assert reply.decision == "verify_pdf" + # Both drugs' evidence must still be present (as citations), not dropped. + assert len(reply.citations) == 2 + + def test_interaction_with_no_evidence_abstains_and_does_not_imply_safety(): agent = _agent( QueryFrame(turn_type="interaction", drugs=("drug_a", "drug_b")), {}, @@ -147,18 +278,65 @@ def test_interaction_with_no_evidence_abstains_and_does_not_imply_safety(): assert "KHÔNG có nghĩa là an toàn" in reply.answer -def test_symptom_to_drug_without_a_drug_name_asks_honestly_not_wired_yet(): - agent = _agent(QueryFrame(turn_type="symptom_to_drug", indication="sốt cao")) - reply = agent.handle("sốt cao uống thuốc gì") +def test_symptom_to_drug_with_no_indication_extracted_asks_for_one(): + agent = _agent(QueryFrame(turn_type="symptom_to_drug")) + reply = agent.handle("có thuốc gì không") assert reply.decision == "clarify" - assert reply.reason == "reverse_lookup_not_ready" + assert reply.reason == "no_indication" + + +def test_symptom_to_drug_with_no_match_abstains_not_silently_safe(): + agent = _agent( + QueryFrame(turn_type="symptom_to_drug", indication="bệnh hiếm gặp x"), + indication_results={ + "bệnh hiếm gặp x": RetrievalResult(EvidenceDecision.ABSTAIN, "no_indication_match"), + }, + ) + reply = agent.handle("thuốc gì trị bệnh hiếm gặp x") + assert reply.decision == "abstain" + assert "bệnh hiếm gặp x" in reply.answer + assert "KHÔNG" in reply.answer + + +def test_symptom_to_drug_returns_the_matched_drugs_not_frame_drugs(): + """`frame.drugs` is empty by construction for this turn_type (the router + only reaches `_symptom_to_drug` with no named drug) — the reply's + `drugs` must come from what retrieval actually found.""" + result = RetrievalResult( + EvidenceDecision.ANSWERABLE, "grounded_evidence_available", + (_evidence("Paracetamol chỉ định hạ sốt."), + _evidence("Ibuprofen chỉ định hạ sốt, giảm đau.")), + ) + # Overwrite matched_doc_id per evidence to simulate two different drugs + # (the shared `_evidence` helper always uses "e0" — construct directly). + ev_a = Evidence( + evidence_id="paracetamol_acetaminophen__chi_dinh__0", + matched_doc_id="paracetamol_acetaminophen__chi_dinh__0", + kind="prose", text="Paracetamol chỉ định hạ sốt.", score=1.0, + source_refs=(SOURCE,), hydrated_from_parent=False, requires_visual_check=False, + ) + ev_b = Evidence( + evidence_id="ibuprofen__chi_dinh__0", matched_doc_id="ibuprofen__chi_dinh__0", + kind="prose", text="Ibuprofen chỉ định hạ sốt, giảm đau.", score=1.0, + source_refs=(SOURCE,), hydrated_from_parent=False, requires_visual_check=False, + ) + result = RetrievalResult( + EvidenceDecision.ANSWERABLE, "grounded_evidence_available", (ev_a, ev_b), + ) + agent = _agent( + QueryFrame(turn_type="symptom_to_drug", indication="sốt"), + indication_results={"sốt": result}, + ) + reply = agent.handle("sốt thì uống thuốc gì") + assert reply.decision == "answerable" + assert reply.drugs == ("paracetamol_acetaminophen", "ibuprofen") def test_history_is_passed_to_the_understander_on_the_next_turn(): received_history: list[tuple[str, ...]] = [] class _RecordingUnderstander: - def understand(self, turn, history=()): + def understand(self, turn, history=(), budget=None, prior_frame=None): received_history.append(tuple(history)) return QueryFrame(turn_type="smalltalk") @@ -176,7 +354,7 @@ def test_history_is_isolated_per_conversation_id(): received_history: list[tuple[str, ...]] = [] class _RecordingUnderstander: - def understand(self, turn, history=()): + def understand(self, turn, history=(), budget=None, prior_frame=None): received_history.append(tuple(history)) return QueryFrame(turn_type="smalltalk") @@ -190,6 +368,26 @@ def test_history_is_isolated_per_conversation_id(): assert received_history[1] == () +# --- F-10: `conversation_id` presence/absence must not change the safety +# decision on a fresh (first) turn — only whether the turn is remembered +# afterward ------------------------------------------------------------- + + +def test_conversation_id_presence_or_absence_reaches_the_same_decision(): + """A single-turn call (no `conversation_id`) and the first turn of a + fresh multi-turn conversation must resolve identically — both see empty + history, so nothing about `conversation_id` itself may become a second, + undocumented safety signal.""" + agent = _agent(QueryFrame( + turn_type="drug_attribute", unknown_drugs=("aspirinol",), + )) + without_id = agent.handle("liều aspirinol") + with_id = agent.handle("liều aspirinol", conversation_id="fresh-conv") + + assert without_id.decision == with_id.decision == "abstain" + assert without_id.reason == with_id.reason == "drug_not_in_formulary" + + def test_autocomplete_delegates_to_the_configured_source(): class _Source: def complete(self, prefix, k): @@ -210,3 +408,291 @@ def test_autocomplete_with_no_source_configured_returns_empty(): _FixedRetrieval({}), answers, ) assert agent.complete("met") == [] + + +# --- F-08: a per-turn budget actually stops real provider calls once spent, +# it isn't just bookkeeping --------------------------------------------- + + +def test_a_budget_exhausted_during_understand_blocks_every_later_call(): + """`max_llm_calls_per_turn=1` means the (simulated) `understand()` call + spends the entire turn's budget — sufficiency and generate must never + reach the generator at all, not just receive an error from it. Proves + the budget is threaded end to end through `RagAgent`, not only present + at the one call site each unit test exercises in isolation.""" + class _BudgetSpendingUnderstander: + """Stands in for the real `LlmQueryUnderstander`, which calls + `budget.require()` once before its own LLM call — simulated here so + this test doesn't need a live-shaped LLM fake for the understand + step, only for the reply's calls to fail budget.""" + + def understand(self, turn, history=(), budget=None, prior_frame=None): + if budget is not None: + budget.require() + return QueryFrame( + turn_type="drug_attribute", drugs=("metformin",), + attribute="chi_dinh", + ) + + result = RetrievalResult( + EvidenceDecision.ANSWERABLE, "grounded_evidence_available", + (_evidence("đoạn 0"), _evidence("đoạn 1")), + ) + + generator_calls: list[dict] = [] + + class _CountingGenerator: + def generate(self, system, user, schema): + generator_calls.append(schema) + return '{"answer": "unused", "evidence_sufficient": true, "clarifying_question": null}' + + metrics = InMemoryMetrics() + answers = GroundedAnswerService( + routing=None, generator=_CountingGenerator(), metrics=metrics + ) + agent = RagAgent( + _BudgetSpendingUnderstander(), _FixedRetrieval({"metformin": result}), answers, + max_llm_calls_per_turn=1, + ) + + reply = agent.handle("liều metformin") + + # The real point of F-08: no more actual provider calls happen once the + # budget is spent — not "the generator returned an error", the generator + # is never invoked at all. + assert generator_calls == [] + assert reply.decision == "abstain" + assert reply.answer is None + assert metrics.total(GENERATION_REJECTED, reason="request_budget_exhausted") >= 1 + + +def test_a_generous_budget_does_not_change_normal_behaviour(): + """Control for the test above: with the default (generous) budget, the + same setup answers normally — proves the previous test's tiny budget is + what caused the block, not some other change to the fixtures.""" + class _BudgetSpendingUnderstander: + def understand(self, turn, history=(), budget=None, prior_frame=None): + if budget is not None: + budget.require() + return QueryFrame( + turn_type="drug_attribute", drugs=("metformin",), + attribute="chi_dinh", + ) + + result = RetrievalResult( + EvidenceDecision.ANSWERABLE, "grounded_evidence_available", + (_evidence("Liều 500 mg mỗi ngày."), _evidence("đoạn 1")), + ) + + class _Generator: + def generate(self, system, user, schema): + if "sufficient" in schema.get("properties", {}): + return '{"sufficient": true, "clarifying_question": null, "quick_replies": []}' + if "entailed" in schema.get("properties", {}): + return '{"entailed": true, "unsupported": []}' + return '{"answer": "Liều 500 mg [1].", "evidence_sufficient": true, "clarifying_question": null}' + + answers = GroundedAnswerService(routing=None, generator=_Generator()) + agent = RagAgent( + _BudgetSpendingUnderstander(), _FixedRetrieval({"metformin": result}), answers, + ) + + reply = agent.handle("liều metformin") + + assert reply.decision == "answerable" + assert reply.answer == "Liều 500 mg [1]." + + +# --- durable conversation history (ADR 0008's named gap): an optional +# `ConversationStore` replaces the in-process dict when configured -------- + + +class _FakeStore: + def __init__(self) -> None: + self.lines: dict[str, list[str]] = {} + self.fail_reads = False + self.fail_writes = False + + def recent(self, conversation_id, limit): + if self.fail_reads: + raise ConnectionError("store unreachable") + return self.lines.get(conversation_id, [])[-limit:] + + def append(self, conversation_id, line): + if self.fail_writes: + raise ConnectionError("store unreachable") + self.lines.setdefault(conversation_id, []).append(line) + + +def test_history_round_trips_through_a_configured_store(): + received_history: list[tuple[str, ...]] = [] + + class _RecordingUnderstander: + def understand(self, turn, history=(), budget=None, prior_frame=None): + received_history.append(tuple(history)) + return QueryFrame(turn_type="smalltalk") + + store = _FakeStore() + answers = GroundedAnswerService(routing=None) + agent = RagAgent(_RecordingUnderstander(), _FixedRetrieval({}), answers, store=store) + + agent.handle("chào bạn", conversation_id="c1") + agent.handle("còn liều thì sao?", conversation_id="c1") + + assert received_history[0] == () + assert any("chào bạn" in line for line in received_history[1]) + assert store.lines["c1"] # actually persisted, not just read back in-process + + +def test_store_read_failure_fails_open_to_fresh_history_not_a_crash(): + store = _FakeStore() + store.lines["c1"] = ["Người dùng: câu cũ", "Trợ lý: trả lời cũ"] + store.fail_reads = True + + answers = GroundedAnswerService(routing=None) + agent = RagAgent( + _FixedUnderstander(QueryFrame(turn_type="smalltalk")), + _FixedRetrieval({}), answers, store=store, + ) + + # Must not raise -- degrades to no history for this turn. + reply = agent.handle("chào bạn", conversation_id="c1") + assert reply.decision == "answerable" + + +def test_store_write_failure_fails_open_the_response_still_returns(): + store = _FakeStore() + store.fail_writes = True + + answers = GroundedAnswerService(routing=None) + agent = RagAgent( + _FixedUnderstander(QueryFrame(turn_type="smalltalk")), + _FixedRetrieval({}), answers, store=store, + ) + + # Must not raise, even though persisting this turn silently fails. + reply = agent.handle("chào bạn", conversation_id="c1") + assert reply.decision == "answerable" + assert reply.answer is not None + + +# --- F-11: the clarify-loop circuit breaker. Found live 2026-08-07 +# (50-question hand-typed browser audit): the understanding LLM can keep +# deciding needs_clarify=true forever with no natural exit — reproduced 3 +# times independently, one case never converged after 5 real answered turns. +# `understanding.py`'s prior-frame merge fixes most of the underlying cause, +# but this is the code-level bound that guarantees a user is never stuck. -- + + +class _AlwaysClarifyUnderstander: + """Simulates a model stuck re-asking regardless of what the user + answers — exactly the observed live failure, isolated from any real + LLM's actual (variable) behaviour so this test is deterministic.""" + + def understand(self, turn, history=(), budget=None, prior_frame=None): + return QueryFrame( + turn_type="dosing_calc", drugs=("paracetamol_acetaminophen",), + needs_clarify=True, clarify_reason="Bé nặng bao nhiêu kg?", + ) + + +def test_clarify_loop_is_hard_stopped_after_max_consecutive_turns(): + answers = GroundedAnswerService(routing=None) + agent = RagAgent(_AlwaysClarifyUnderstander(), _FixedRetrieval({}), answers) + + replies = [ + agent.handle(f"turn {i}", conversation_id="stuck") + for i in range(MAX_CONSECUTIVE_CLARIFY) + ] + + # Every turn up to the last stays a genuine clarify... + for reply in replies[:-1]: + assert reply.decision == "clarify" + # ...the Nth forces a hard stop instead of asking again. + assert replies[-1].decision == "abstain" + assert replies[-1].reason == "clarify_loop_exhausted" + assert replies[-1].answer is not None + + +def test_clarify_streak_resets_after_the_hard_stop_so_a_new_attempt_can_proceed(): + """Confirms the breaker is a bounded pause, not a permanent lockout of + the conversation id — the very next turn gets a fresh streak.""" + answers = GroundedAnswerService(routing=None) + agent = RagAgent(_AlwaysClarifyUnderstander(), _FixedRetrieval({}), answers) + + for i in range(MAX_CONSECUTIVE_CLARIFY): + agent.handle(f"turn {i}", conversation_id="stuck") + reply = agent.handle("one more try", conversation_id="stuck") + + assert reply.decision == "clarify" + + +def test_a_resolved_turn_resets_the_clarify_streak(): + """An answerable turn in between two clarify runs must not let their + streaks combine — only genuinely consecutive clarifies count.""" + result = RetrievalResult( + EvidenceDecision.ANSWERABLE, "grounded_evidence_available", + (_evidence("Liều 500 mg."),), + ) + clarify_frame = QueryFrame( + turn_type="dosing_calc", drugs=("paracetamol_acetaminophen",), + needs_clarify=True, clarify_reason="Bé nặng bao nhiêu kg?", + ) + resolved_frame = QueryFrame( + turn_type="drug_attribute", drugs=("metformin",), + attribute="lieu_luong_va_cach_dung", + ) + # (MAX-1) clarifies, one resolved turn, then (MAX-1) clarifies again — + # scripted explicitly so the test asserts the streak reset, not an + # incidental side effect of some other call-counting scheme. + script = ( + [clarify_frame] * (MAX_CONSECUTIVE_CLARIFY - 1) + + [resolved_frame] + + [clarify_frame] * (MAX_CONSECUTIVE_CLARIFY - 1) + ) + + class _ScriptedUnderstander: + def __init__(self, frames): + self._frames = iter(frames) + + def understand(self, turn, history=(), budget=None, prior_frame=None): + return next(self._frames) + + answers = GroundedAnswerService(routing=None) + agent = RagAgent( + _ScriptedUnderstander(script), _FixedRetrieval({"metformin": result}), answers, + ) + + for i in range(MAX_CONSECUTIVE_CLARIFY - 1): + reply = agent.handle(f"clarify {i}", conversation_id="c1") + assert reply.decision == "clarify" + resolved = agent.handle("answerable turn", conversation_id="c1") + assert resolved.decision == "answerable" + + # Streak was reset by the resolved turn -- this run of clarifies must + # not be treated as a continuation of the earlier (pre-reset) run. + for i in range(MAX_CONSECUTIVE_CLARIFY - 1): + reply = agent.handle(f"clarify again {i}", conversation_id="c1") + assert reply.decision == "clarify" + + +def test_prior_frame_is_threaded_from_the_last_turn_to_the_understander(): + received: list[QueryFrame | None] = [] + + class _RecordingUnderstander: + def understand(self, turn, history=(), budget=None, prior_frame=None): + received.append(prior_frame) + return QueryFrame( + turn_type="dosing_calc", drugs=("paracetamol_acetaminophen",), + needs_clarify=True, clarify_reason="Bé nặng bao nhiêu kg?", + ) + + answers = GroundedAnswerService(routing=None) + agent = RagAgent(_RecordingUnderstander(), _FixedRetrieval({}), answers) + + agent.handle("liều paracetamol cho bé", conversation_id="c1") + agent.handle("20kg", conversation_id="c1") + + assert received[0] is None + assert received[1] is not None + assert received[1].clarify_reason == "Bé nặng bao nhiêu kg?" diff --git a/apps/ai-service/tests/test_budget.py b/apps/ai-service/tests/test_budget.py new file mode 100644 index 0000000..7255784 --- /dev/null +++ b/apps/ai-service/tests/test_budget.py @@ -0,0 +1,59 @@ +"""`rag/budget.py::RequestBudget` — F-08.""" +from __future__ import annotations + +import time + +import pytest + +from rag.budget import RequestBudget, RequestBudgetExhausted +from rag.ports import AnswerGenerationUnavailable + + +def test_budget_exhausted_is_a_subclass_of_answer_generation_unavailable(): + """Deliberate: every existing `except AnswerGenerationUnavailable:` + fail-open/fail-closed handler in the codebase must catch this with zero + changes, since it predates F-08 and already encodes the right behaviour + for "the provider is unavailable to us right now".""" + assert issubclass(RequestBudgetExhausted, AnswerGenerationUnavailable) + + +def test_fresh_budget_has_budget(): + budget = RequestBudget.start(max_wall_clock_ms=20_000, max_calls=5) + assert budget.has_budget() is True + + +def test_require_spends_one_call(): + budget = RequestBudget.start(max_wall_clock_ms=20_000, max_calls=2) + budget.require() + assert budget.calls_remaining == 1 + budget.require() + assert budget.calls_remaining == 0 + assert budget.has_budget() is False + + +def test_require_raises_once_calls_are_exhausted(): + budget = RequestBudget.start(max_wall_clock_ms=20_000, max_calls=1) + budget.require() + with pytest.raises(RequestBudgetExhausted): + budget.require() + + +def test_require_raises_once_the_deadline_has_passed(): + budget = RequestBudget.start(max_wall_clock_ms=0, max_calls=100) + time.sleep(0.01) + assert budget.has_budget() is False + with pytest.raises(RequestBudgetExhausted): + budget.require() + + +def test_a_failed_require_does_not_spend_a_call(): + """`require()` raises before decrementing when there's no budget left — + `calls_remaining` must not go negative, which would otherwise make a + budget that's already exhausted look like it has "negative debt" instead + of cleanly `0`.""" + budget = RequestBudget.start(max_wall_clock_ms=20_000, max_calls=1) + budget.require() + for _ in range(3): + with pytest.raises(RequestBudgetExhausted): + budget.require() + assert budget.calls_remaining == 0 diff --git a/apps/ai-service/tests/test_citation_and_intro.py b/apps/ai-service/tests/test_citation_and_intro.py index d8dfd0b..cb66093 100644 --- a/apps/ai-service/tests/test_citation_and_intro.py +++ b/apps/ai-service/tests/test_citation_and_intro.py @@ -16,6 +16,7 @@ from rag.models import ( SourceRef, SubjectScope, ) +from rag.ports import AnswerGenerationUnavailable from rag.prompt import build_request @@ -41,22 +42,31 @@ class _Routing: class _Generator: - def __init__(self, payload: dict, entailment_payload: dict | None = None) -> None: + def __init__( + self, payload: dict, entailment_payload: dict | None = None, + sufficiency_payload: dict | None = None, + ) -> None: self._payload = payload self._entailment_payload = entailment_payload or { "entailed": True, "unsupported": [], } + self._sufficiency_payload = sufficiency_payload def generate(self, system: str, user: str, schema: dict) -> str: # noqa: ARG002 - # `_generate` also runs a post-generation entailment check; tell the - # two request shapes apart by schema so callers here only need to - # fake the main answer, not both. - payload = ( - self._entailment_payload - if "entailed" in schema.get("properties", {}) - else self._payload - ) + # `_generate` also runs a post-generation entailment check, and + # `_check_sufficiency` runs its own separate call before that — tell + # the three request shapes apart by schema so a test asserting on one + # doesn't have to also shape a payload for the others. + props = schema.get("properties", {}) + if "entailed" in props: + payload = self._entailment_payload + elif "sufficient" in props and self._sufficiency_payload is not None: + payload = self._sufficiency_payload + else: + payload = self._payload + if isinstance(payload, BaseException): + raise payload return json.dumps(payload, ensure_ascii=False) @@ -122,6 +132,42 @@ def test_underspecified_dose_asks_instead_of_dumping(): assert "tuổi" in g.clarification assert g.answer == g.clarification assert g.generated is False + assert g.quick_replies == () + + +def test_underspecified_dose_carries_quick_replies_when_the_model_offers_them(): + result = _answerable(_evidence(0, 100), _evidence(1, 200)) + gen = _Generator( + {"sufficient": False, + "clarifying_question": "Người lớn hay trẻ em?", + "quick_replies": ["Người lớn", "Trẻ em"]} + ) + service = GroundedAnswerService(_Routing(result), gen) + + g = service.answer("paracetamol", SubjectScope.HUMAN, QueryIntent.FACT_LOOKUP) + + assert g.quick_replies == ("Người lớn", "Trẻ em") + + +def test_sufficiency_check_outage_fails_open_to_generation_not_abstain(): + """F-10: `_check_sufficiency` documents (and this pins) a deliberate + fail-OPEN on provider outage — unlike every other failure mode in this + service, a sufficiency-check outage does not abstain, it just skips the + clarify heuristic and lets grounding/entailment (tested elsewhere) be + the real safety net on whatever gets generated next.""" + result = _answerable(_evidence(0, 100), _evidence(1, 200)) + gen = _Generator( + {"answer": "Đoạn bằng chứng 0 [1].", "evidence_sufficient": True, + "clarifying_question": None}, + sufficiency_payload=AnswerGenerationUnavailable("Bedrock unreachable"), + ) + service = GroundedAnswerService(_Routing(result), gen) + + g = service.answer("liều người lớn", SubjectScope.HUMAN, QueryIntent.FACT_LOOKUP) + + assert g.clarification is None + assert g.result.decision == EvidenceDecision.ANSWERABLE + assert g.generated is True def test_sufficient_query_is_not_turned_into_a_clarification(): @@ -144,3 +190,52 @@ def test_bare_name_builds_an_intro_prompt(): normal = build_request("Liều?", ("đoạn A",), intro=False) assert "CÂU HỎI:" in normal.user assert "GIỚI THIỆU" not in normal.user + + +def test_list_mode_prompt_instructs_enumerating_every_drug_not_ranking(): + """2026-08-07 finding: without an explicit instruction, the model picked + one drug out of 8 real symptom_to_drug matches and silently dropped the + rest — verified live. `list_mode` closes that.""" + request = build_request("thuốc gì trị sốt", ("chỉ định A", "chỉ định B"), list_mode=True) + assert "LIỆT KÊ TẤT CẢ" in request.user + assert "không xếp hạng" in request.user or "KHÔNG" in request.user + assert "CÂU HỎI:" in request.user + + +def test_list_mode_skips_the_sufficiency_clarify(): + """If the sufficiency call were NOT actually skipped, it would read + `sufficiency_payload` (`sufficient=False`) and clarify. `list_mode=True` + must never call it at all, so only the real answer payload is ever read.""" + result = _answerable(_evidence(0, 100), _evidence(1, 200)) + gen = _Generator( + {"answer": "Đoạn bằng chứng 0 [1]. Đoạn bằng chứng 1 [2].", + "evidence_sufficient": True, "clarifying_question": None}, + sufficiency_payload={ + "sufficient": False, "clarifying_question": "Loại nào?", "quick_replies": [], + }, + ) + service = GroundedAnswerService(_Routing(result), gen) + + g = service.answer_from_result("thuốc gì trị sốt", result, list_mode=True) + + assert g.clarification is None + assert g.answer is not None + assert g.generated is True + + +def test_without_list_mode_the_same_evidence_does_ask_for_clarification(): + """Control for the test above: the same sufficiency payload, without + `list_mode`, must actually clarify — proving the previous test's "not + skipped" branch is reachable and would have failed loudly.""" + result = _answerable(_evidence(0, 100), _evidence(1, 200)) + gen = _Generator( + {"answer": "unused", "evidence_sufficient": True, "clarifying_question": None}, + sufficiency_payload={ + "sufficient": False, "clarifying_question": "Loại nào?", "quick_replies": [], + }, + ) + service = GroundedAnswerService(_Routing(result), gen) + + g = service.answer_from_result("thuốc gì trị sốt", result, list_mode=False) + + assert g.clarification == "Loại nào?" diff --git a/apps/ai-service/tests/test_conversation.py b/apps/ai-service/tests/test_conversation.py deleted file mode 100644 index 9a90adf..0000000 --- a/apps/ai-service/tests/test_conversation.py +++ /dev/null @@ -1,189 +0,0 @@ -"""Follow-ups must inherit context, and must never inherit it silently. - -The cases here are the ones the owner named on 2026-08-05: "còn trẻ em thì -sao?", "giải thích kỹ hơn", and not making the user repeat themselves. The -adversarial cases are the ones that make inheritance dangerous in a formulary -— a stale drug, and an explicit mention being overridden by context. -""" -from __future__ import annotations - -from rag.conversation import ( - FOCUS_TTL_TURNS, - ConversationState, - Focus, - Turn, - detect_population, - detect_verbosity, - looks_like_followup, - resolve_against, - update_focus, -) - - -def _state(turn_count: int = 1, **focus_fields) -> ConversationState: - focus = Focus() - for name, value in focus_fields.items(): - focus = focus.with_field(name, value, turn_count - 1) - return ConversationState("c1", focus=focus, turn_count=turn_count) - - -# --- the follow-ups the owner asked for -------------------------------------- - - -def test_con_tre_em_thi_sao_inherits_drug_and_section(): - state = _state(drug_id="metformin", section_key="lieu_luong_va_cach_dung") - - resolved = resolve_against(state, "còn trẻ em thì sao?", None, None) - - assert resolved.drug_id == "metformin" - assert resolved.section_key == "lieu_luong_va_cach_dung" - assert resolved.population == "tre_em" - assert resolved.inherited_drug is True - - -def test_giai_thich_ky_hon_sets_verbosity_and_keeps_the_topic(): - state = _state(drug_id="warfarin", section_key="tuong_tac_thuoc") - - resolved = resolve_against(state, "giải thích kỹ hơn", None, None) - - assert resolved.drug_id == "warfarin" - assert resolved.verbosity == "detailed" - - -def test_the_user_is_not_made_to_repeat_the_drug(): - state = _state(drug_id="metformin") - - resolved = resolve_against(state, "chống chỉ định", None, "chong_chi_dinh") - - assert resolved.drug_id == "metformin" - assert resolved.section_key == "chong_chi_dinh" - - -# --- what makes inheritance safe --------------------------------------------- - - -def test_an_explicit_drug_always_beats_context(): - """Naming a drug must override whatever the conversation was about, or a - deliberate topic change silently answers about the previous medicine.""" - state = _state(drug_id="metformin", section_key="lieu_luong_va_cach_dung") - - resolved = resolve_against(state, "liều dùng warfarin", "warfarin", None) - - assert resolved.drug_id == "warfarin" - assert resolved.inherited_drug is False - - -def test_a_stale_drug_is_dropped_rather_than_inherited(): - """Beyond the TTL the drug is not context, it is a hazard.""" - state = _state(turn_count=FOCUS_TTL_TURNS + 3, drug_id="metformin") - # `_state` stamps at turn_count - 1, so age is 1; age it past the TTL. - aged = ConversationState( - "c1", - focus=Focus(drug_id="metformin", set_at_turn={"drug_id": 0}), - turn_count=FOCUS_TTL_TURNS + 2, - ) - - assert state.inherited("drug_id") == "metformin" - assert aged.inherited("drug_id") is None - - resolved = resolve_against(aged, "còn trẻ em thì sao?", None, None) - assert resolved.drug_id is None - - -def test_an_inherited_drug_must_be_named_in_the_answer(): - state = _state(drug_id="metformin") - - inherited = resolve_against(state, "còn trẻ em thì sao?", None, None) - explicit = resolve_against(state, "liều warfarin", "warfarin", None) - - assert inherited.needs_carry_over_notice is True - assert explicit.needs_carry_over_notice is False - - -# --- phrase detection --------------------------------------------------------- - - -def test_longest_population_phrase_wins(): - """`phụ nữ cho con bú` must not be read as `phụ nữ`, and `trẻ sơ sinh` - must not be read as `trẻ em` — the same rule `sections.py` relies on.""" - assert detect_population("phụ nữ cho con bú") == "phu_nu_cho_con_bu" - assert detect_population("trẻ sơ sinh dùng sao") == "tre_so_sinh" - assert detect_population("bà bầu uống được không") == "phu_nu_co_thai" - assert detect_population("liều cho người lớn") == "nguoi_lon" - - -def test_no_population_named_is_none_not_a_guess(): - assert detect_population("liều dùng paracetamol") is None - assert detect_verbosity("liều dùng paracetamol") is None - - -def test_followup_markers(): - assert looks_like_followup("còn trẻ em thì sao?") is True - assert looks_like_followup("so với metformin thì sao") is True - assert looks_like_followup("liều dùng paracetamol") is False - - -# --- window and focus update -------------------------------------------------- - - -def test_recent_window_evicts_oldest(): - state = ConversationState("c1") - for index in range(8): - state = state.append(Turn("user", f"q{index}", "2026-08-05"), window=6) - - assert len(state.recent) == 6 - assert state.recent[0].text == "q2" - assert state.turn_count == 8 - - -def test_evicted_turns_reach_overflow_not_silently_dropped(): - """Bug fixed 2026-08-06 (Codex review, F-06): `overflow()` used to check - `len(self.recent) > window`, but `append()` already truncates `recent` - to `window`, so that comparison could never be true — evicted turns - never reached the summariser no matter how long a conversation ran. - Exact repro from the review: 8 turns into a window of 6.""" - state = ConversationState("c1") - for index in range(8): - state = state.append(Turn("user", f"q{index}", "2026-08-05"), window=6) - - overflow = state.overflow() - - assert [turn.text for turn in overflow] == ["q0", "q1"] - - -def test_overflow_accumulates_across_the_two_appends_one_turn_makes(): - """A live turn typically calls `append()` twice in a row (user, then - assistant). Each can evict at most one turn; the second call's overflow - must not overwrite, and so lose, the first's.""" - state = ConversationState("c1") - for index in range(6): - state = state.append(Turn("user", f"q{index}", "2026-08-05"), window=6) - assert state.overflow() == () # window exactly full, nothing evicted yet - - state = state.append(Turn("user", "q6", "2026-08-05"), window=6) - state = state.append(Turn("assistant", "a6", "2026-08-05"), window=6) - - assert [turn.text for turn in state.overflow()] == ["q0", "q1"] - - -def test_overflow_is_empty_again_after_the_caller_clears_it(): - from dataclasses import replace - - state = ConversationState("c1") - for index in range(8): - state = state.append(Turn("user", f"q{index}", "2026-08-05"), window=6) - assert state.overflow() != () - - state = replace(state, pending_overflow=()) - - assert state.overflow() == () - - -def test_focus_update_stamps_the_current_turn(): - state = _state(turn_count=3) - - resolved = resolve_against(state, "liều dùng metformin", "metformin", "lieu_luong_va_cach_dung") - focus = update_focus(state, resolved) - - assert focus.drug_id == "metformin" - assert focus.set_at_turn["drug_id"] == 3 diff --git a/apps/ai-service/tests/test_conversation_summary.py b/apps/ai-service/tests/test_conversation_summary.py deleted file mode 100644 index 5e23a2f..0000000 --- a/apps/ai-service/tests/test_conversation_summary.py +++ /dev/null @@ -1,50 +0,0 @@ -from rag.conversation import ( - ConversationState, - DeterministicSummariser, - InMemoryConversationStore, - Turn, -) - - -def test_store_returns_fresh_state_for_unknown_id(): - store = InMemoryConversationStore() - state = store.load("conv-new") - assert state.conversation_id == "conv-new" - assert state.turn_count == 0 - assert state.recent == () - - -def test_store_round_trips_saved_state(): - store = InMemoryConversationStore() - state = ConversationState("conv-1", summary="s", turn_count=3) - store.save(state) - assert store.load("conv-1") is state - - -def test_summariser_records_topic_labels_only(): - s = DeterministicSummariser() - dropped = ( - Turn("user", "Chống chỉ định của metformin?", "t0", - drug_id="metformin", section_key="chong_chi_dinh"), - Turn("assistant", "Quá mẫn với metformin, suy thận Clcr < 60...", "t1", - drug_id="metformin", section_key="chong_chi_dinh"), - ) - out = s.fold("", dropped) - # The label line is present... - assert "chong_chi_dinh của metformin" in out - # ...and no clinical value leaked from the assistant turn. - assert "Clcr" not in out - assert "60" not in out - - -def test_summariser_stays_within_budget_dropping_oldest(): - s = DeterministicSummariser() - dropped = tuple( - Turn("user", f"q{i}", f"t{i}", drug_id=f"drug{i}", section_key="lieu_luong") - for i in range(400) - ) - out = s.fold("", dropped) - assert len(out) <= DeterministicSummariser.MAX_CHARS - # Most-recent topic survives, oldest is dropped. - assert "drug399" in out - assert "drug0 " not in out diff --git a/apps/ai-service/tests/test_conversational_loop.py b/apps/ai-service/tests/test_conversational_loop.py deleted file mode 100644 index c96950d..0000000 --- a/apps/ai-service/tests/test_conversational_loop.py +++ /dev/null @@ -1,133 +0,0 @@ -from rag.answer import GroundedAnswer -from rag.conversation import DeterministicSummariser, InMemoryConversationStore -from rag.conversational import ( - SMALLTALK_REPLY, - ConversationalLoopService, -) -from rag.models import ( - Evidence, - EvidenceDecision, - QueryIntent, - RetrievalResult, - SubjectScope, -) -from rag.reasoning import ClarifyReason -from rag.routing import CatalogDrugResolver -from rag.sections import SectionResolver - - -def _grounded(answer, evidence_text): - ev = Evidence("e1", "e1", "prose", evidence_text, 1.0, (), False, False) - result = RetrievalResult( - EvidenceDecision.ANSWERABLE, "grounded_evidence_available", - (ev,), "metformin", "resolved", - ) - return GroundedAnswer(result, answer, (), False) - - -class FakeAnswers: - def __init__(self, answer_text, evidence_text): - self._a = answer_text - self._e = evidence_text - self.calls = [] - - def answer(self, query, subject_scope, intent, drug_id=None): - self.calls.append((query, drug_id)) - return _grounded(self._a, self._e) - - -def _service(answers): - return ConversationalLoopService( - answers=answers, - resolver=CatalogDrugResolver({"metformin": {"metformin"}}), - section_resolver=SectionResolver(), - store=InMemoryConversationStore(), - summariser=DeterministicSummariser(), - ) - - -def test_smalltalk_answers_socially_without_calling_engine(): - answers = FakeAnswers("x", "x") - svc = _service(answers) - out = svc.answer("c1", "chào bạn", SubjectScope.HUMAN, QueryIntent.FACT_LOOKUP) - assert out.smalltalk is True - assert out.answer == SMALLTALK_REPLY - assert answers.calls == [] # a greeting is not a drug lookup - - -def test_medical_turn_returns_grounded_answer(): - answers = FakeAnswers("Quá mẫn với metformin.", "Quá mẫn với metformin.") - svc = _service(answers) - out = svc.answer( - "c2", "chống chỉ định metformin", SubjectScope.HUMAN, QueryIntent.FACT_LOOKUP - ) - assert out.smalltalk is False - assert out.answer == "Quá mẫn với metformin." - assert out.grounded is not None - - -def test_followup_inherits_drug_and_passes_it_resolved(): - answers = FakeAnswers( - "Ở trẻ em điều chỉnh theo cân nặng.", - "Ở trẻ em, liều metformin điều chỉnh theo cân nặng.", - ) - svc = _service(answers) - svc.answer("c3", "chống chỉ định metformin", SubjectScope.HUMAN, QueryIntent.FACT_LOOKUP) - out = svc.answer("c3", "còn trẻ em thì sao?", SubjectScope.HUMAN, QueryIntent.FACT_LOOKUP) - assert out.inherited_drug == "metformin" - assert out.answer.startswith("Về Metformin:") - # The inherited drug is passed already-resolved (not re-resolved from the - # rewritten turn text), and the raw follow-up drives section routing. - last_query, last_drug_id = answers.calls[-1] - assert last_drug_id == "metformin" - assert last_query == "còn trẻ em thì sao?" - # State carried the drug forward. - assert svc._store.load("c3").focus.drug_id == "metformin" - - -def test_confirmation_is_not_fuzzy_matched_to_a_drug(): - """'đúng' must not be fuzzy-matched to terbinafin/tretinoin (the did-you-mean - loop the reviewer hit); it asks which drug instead.""" - answers = FakeAnswers("x", "x") - svc = _service(answers) - out = svc.answer("cc", "đúng", SubjectScope.HUMAN, QueryIntent.FACT_LOOKUP) - assert out.clarification is not None - assert out.clarification.reason == "confirm_without_context" - assert answers.calls == [] - - -def test_a_long_sentence_that_names_no_drug_is_not_offered_did_you_mean(): - """A full question ('EPO điều trị thiếu máu...') that resolves no drug is - answered honestly, not with garbage suggestions from fuzzing the sentence.""" - answers = FakeAnswers("x", "x") - svc = _service(answers) # catalog holds only metformin - out = svc.answer( - "cl", "EPO điều trị thiếu máu do hóa trị ung thư liều khởi đầu bao nhiêu", - SubjectScope.HUMAN, QueryIntent.FACT_LOOKUP, - ) - assert out.clarification is not None - assert out.clarification.reason == "drug_not_supported" - assert answers.calls == [] - - -def test_no_close_drug_reports_not_supported(): - answers = FakeAnswers("x", "x") - svc = _service(answers) # catalog holds only metformin - out = svc.answer("c4", "cái này thế nào?", SubjectScope.HUMAN, QueryIntent.FACT_LOOKUP) - assert out.answer is None - assert out.clarification is not None - # Nothing close to a real drug: honest "not in the formulary", not a guess. - assert out.clarification.reason == "drug_not_supported" - assert answers.calls == [] - - -def test_typo_offers_did_you_mean_not_silent_resolution(): - answers = FakeAnswers("x", "x") - svc = _service(answers) # catalog holds only metformin - out = svc.answer("c5", "metformim", SubjectScope.HUMAN, QueryIntent.FACT_LOOKUP) - # A near-miss is asked about, never auto-resolved on a similarity threshold. - assert out.answer is None - assert out.clarification is not None - assert out.clarification.reason == "did_you_mean" - assert "Metformin" in out.clarification.options - assert answers.calls == [] diff --git a/apps/ai-service/tests/test_conversational_service.py b/apps/ai-service/tests/test_conversational_service.py deleted file mode 100644 index 7b39cd8..0000000 --- a/apps/ai-service/tests/test_conversational_service.py +++ /dev/null @@ -1,81 +0,0 @@ -from rag.conversation import DeterministicSummariser, InMemoryConversationStore -from rag.conversational import ConversationalRagService, TurnResolution -from rag.reasoning import ClarifyReason, MAX_LLM_CALLS, MAX_RETRIEVAL_ROUNDS, TurnBudget - - -class FakeResolver: - """Maps a turn's text to what it resolves on its own (no context).""" - - def __init__(self, table): - self._table = table - - def resolve_turn(self, text): - for needle, resolution in self._table: - if needle in text: - return resolution - return TurnResolution(drug_id=None, section_key=None, drug_status="not_found") - - -def _service(resolver, retrieve, generate): - return ConversationalRagService( - store=InMemoryConversationStore(), - summariser=DeterministicSummariser(), - resolver=resolver, - retrieve=retrieve, - generate=generate, - ) - - -def test_followup_inherits_drug_and_answer_names_it(): - resolver = FakeResolver([ - ("metformin", TurnResolution("metformin", "chong_chi_dinh", "resolved")), - # "còn trẻ em" names no drug on its own — must inherit. - ("trẻ em", TurnResolution(None, None, "not_found")), - ]) - # Evidence mentions "trẻ em" so the population assessor is satisfied. - retrieve = lambda q: ("Ở trẻ em, liều metformin điều chỉnh theo cân nặng.",) - generate = lambda q, ev, st: "liều theo cân nặng" - svc = _service(resolver, retrieve, generate) - - first = svc.answer("c1", "Chống chỉ định của metformin?") - assert first.inherited_drug is None - - second = svc.answer("c1", "còn trẻ em thì sao?") - assert second.inherited_drug == "metformin" - assert second.answer.startswith("Về metformin:") - - -def test_no_drug_and_no_context_asks_without_spending_budget(): - resolver = FakeResolver([]) # nothing resolves - calls = {"retrieve": 0, "generate": 0} - - def retrieve(q): - calls["retrieve"] += 1 - return ("x",) - - def generate(q, ev, st): - calls["generate"] += 1 - return "x" - - svc = _service(resolver, retrieve, generate) - budget = TurnBudget() - out = svc.answer("c2", "cái này thế nào?", budget=budget) - - assert out.answer is None - assert out.clarification is not None - assert out.clarification.reason == ClarifyReason.AMBIGUOUS_DRUG - # Asking short-circuits before any spend. - assert calls == {"retrieve": 0, "generate": 0} - assert budget.retrieval_rounds == MAX_RETRIEVAL_ROUNDS - assert budget.llm_calls == MAX_LLM_CALLS - - -def test_state_persists_across_turns(): - resolver = FakeResolver([ - ("metformin", TurnResolution("metformin", "chi_dinh", "resolved")), - ]) - svc = _service(resolver, lambda q: ("Chỉ định của metformin.",), lambda q, ev, st: "ok") - svc.answer("c3", "chỉ định metformin?") - state = svc._store.load("c3") - assert state.turn_count == 2 # user + assistant - assert state.focus.drug_id == "metformin" diff --git a/apps/ai-service/tests/test_grounded_generation.py b/apps/ai-service/tests/test_grounded_generation.py index a2470f1..49ac186 100644 --- a/apps/ai-service/tests/test_grounded_generation.py +++ b/apps/ai-service/tests/test_grounded_generation.py @@ -67,23 +67,28 @@ class _FixedRouting: class _Generator: """Returns whatever payload the test wants the model to have produced. - `_generate` now makes up to four calls through this port: the main - answer, a sufficiency check (skipped here — one evidence block), and up - to two entailment calls (a reject retries once — live probing found the - judge noisy on an identical claim/evidence pair). They're told apart by - schema, so a test that only cares about the main answer doesn't also - have to fake an entailment response by hand; `entailment_payload` - overrides it when a test wants the entailment pass to reject. Pass a - list of payloads to get a different answer on each successive - entailment call (e.g. `[reject, accept]` for the retry-recovers case). + `_generate` now makes up to five calls through this port: the main + answer (a lone `evidence_sufficient: false` retries once — the same + noisy-judge finding as entailment, live-confirmed 2026-08-07), a + sufficiency check (skipped here — one evidence block), and up to three + entailment calls (widened from two 2026-08-07: live probing found the + judge noisy on an identical claim/evidence pair, and a real adversarial + sample showed a single retry still discarding correct answers on the + unlucky reject-reject draw). They're told apart by schema, so a test + that only cares about one call doesn't have to fake the others; `payload` + and `entailment_payload` each take either a fixed value or a list for a + different answer on each successive call to that schema (e.g. + `[reject, reject, accept]` for the third-attempt-recovers case). """ def __init__(self, payload, entailment_payload=None) -> None: - self._payload = payload + payloads = payload + self._payloads = list(payloads) if isinstance(payloads, list) else [payloads] + self._call = 0 default = {"entailed": True, "unsupported": []} - payloads = entailment_payload if entailment_payload is not None else default + e_payloads = entailment_payload if entailment_payload is not None else default self._entailment_payloads = ( - list(payloads) if isinstance(payloads, list) else [payloads] + list(e_payloads) if isinstance(e_payloads, list) else [e_payloads] ) self._entailment_call = 0 @@ -93,7 +98,9 @@ class _Generator: payload = self._entailment_payloads[index] self._entailment_call += 1 else: - payload = self._payload + index = min(self._call, len(self._payloads) - 1) + payload = self._payloads[index] + self._call += 1 if isinstance(payload, BaseException): raise payload if isinstance(payload, str): @@ -127,7 +134,12 @@ def test_invented_dose_is_refused_and_never_reaches_the_answer(): # this is a real LLM chatbot, not the retired offline-extractive build). assert grounded.answer is None assert grounded.result.decision == EvidenceDecision.ABSTAIN - assert grounded.result.reason == "generation_unavailable" + # The specific check that rejected it, not a generic catch-all — found + # live 2026-08-07: every rejection reason used to collapse into + # "generation_unavailable" by the time it reached the API response, + # making a real provider outage indistinguishable from ordinary + # entailment noise without reading server metrics by hand. + assert grounded.result.reason == "ungrounded_number" assert metrics.total(GENERATION_REJECTED, reason="ungrounded_number") == 1 @@ -217,17 +229,42 @@ def test_entailment_retries_once_after_a_reject_and_a_later_accept_serves(): assert metrics.total(GENERATION_SERVED) == 1 -def test_entailment_two_agreeing_rejects_still_discard(): +def test_entailment_recovers_on_third_attempt_after_two_rejects(): + """The improvement 2026-08-07 widened the retry from 2 to 3 attempts + after a live 50-question adversarial sample found the 2-attempt policy's + own math (~11% false-discard rate on a genuinely valid claim, from the + noise probed in the docstring above) matched the observed real + abstention rate almost exactly. Two rejects followed by a real accept + must now be served, not discarded.""" + grounded, metrics = _answer( + {"answer": "Metformin dùng điều trị đái tháo đường [1].", + "evidence_sufficient": True}, + entailment_payload=[ + {"entailed": False, "unsupported": [1]}, + {"entailed": False, "unsupported": [1]}, + {"entailed": True, "unsupported": []}, + ], + ) + + assert grounded.generated is True + assert metrics.total(GENERATION_SERVED) == 1 + assert metrics.total(GENERATION_REJECTED) == 0 + + +def test_entailment_three_agreeing_rejects_still_discard(): grounded, metrics = _answer( {"answer": "Metformin chữa ung thư [1].", "evidence_sufficient": True}, entailment_payload=[ {"entailed": False, "unsupported": [1]}, {"entailed": False, "unsupported": [1]}, + {"entailed": False, "unsupported": [1]}, ], ) assert grounded.generated is False assert grounded.answer is None + # All 3 attempts are noisy-judge calls against the SAME claim/evidence — + # a real, reliable rejection must still discard exactly once, not 3x. assert metrics.total(GENERATION_REJECTED, reason="unsupported_claim") == 1 @@ -287,10 +324,46 @@ def test_every_generation_failure_abstains_instead_of_a_raw_source_dump(payload, assert grounded.generated is False assert grounded.answer is None assert grounded.result.decision == EvidenceDecision.ABSTAIN - assert grounded.result.reason == "generation_unavailable" + # The API/trace-visible reason must match the specific check that + # failed, not a generic "generation_unavailable" for every cause — + # otherwise a real outage and ordinary model noise are indistinguishable + # from the outside (the exact gap a live report 2026-08-07 named). + assert grounded.result.reason == reason assert metrics.total(GENERATION_REJECTED, reason=reason) == 1 +def test_evidence_insufficient_retries_once_and_recovers(): + """Found live 2026-08-07 via a 50-question adversarial sample: a real + section that plainly contains the answer (confirmed by re-asking the + identical question 3/3 times successfully right after) still drew an + `evidence_sufficient: false` self-judgment once — the same noisy-judge + pattern already known for entailment, just on a different field of the + same call. A lone insufficient verdict must not be final.""" + grounded, metrics = _answer([ + {"answer": "...", "evidence_sufficient": False}, + {"answer": "Metformin dùng điều trị đái tháo đường [1].", + "evidence_sufficient": True}, + ]) + + assert grounded.generated is True + assert grounded.answer == "Metformin dùng điều trị đái tháo đường [1]." + assert metrics.total(GENERATION_SERVED) == 1 + assert metrics.total(GENERATION_REJECTED) == 0 + + +def test_evidence_insufficient_twice_still_abstains(): + grounded, metrics = _answer([ + {"answer": "...", "evidence_sufficient": False}, + {"answer": "...", "evidence_sufficient": False}, + ]) + + assert grounded.generated is False + assert grounded.answer is None + # Both attempts are the same noisy self-judgment on the same evidence — + # a real, reliable "insufficient" must still discard exactly once. + assert metrics.total(GENERATION_REJECTED, reason="evidence_insufficient") == 1 + + def test_no_generator_configured_still_answers(): service = GroundedAnswerService(_FixedRouting(_result())) diff --git a/apps/ai-service/tests/test_live_datastores.py b/apps/ai-service/tests/test_live_datastores.py index 4f6f598..80abade 100644 --- a/apps/ai-service/tests/test_live_datastores.py +++ b/apps/ai-service/tests/test_live_datastores.py @@ -18,6 +18,9 @@ ROOT = Path(__file__).resolve().parents[3] CHUNKS = ROOT / "ingestion/data/processed/chunks.jsonl" PDF = ROOT / "ingestion/data/raw/duoc-thu-quoc-gia-viet-nam-2018.pdf" MIGRATION = Path(__file__).resolve().parents[1] / "migrations/001_rag_retrieval_trace.sql" +CONVERSATION_MIGRATION = ( + Path(__file__).resolve().parents[1] / "migrations/002_rag_conversation_turn.sql" +) class _PlumbingEmbedder: @@ -131,6 +134,38 @@ def test_real_postgres_migration_insert_and_read_back(): assert stored.citations[0]["printed_page_start"] == 101 +def test_real_postgres_conversation_store_round_trip(): + """F-08's durable conversation history against a real Postgres, not a + fake — proves `append`/`recent` actually persist and window correctly, + the concrete capability this whole feature exists to add over the + in-process dict it replaces.""" + from adapters.postgres import PostgresConversationStore + + store = PostgresConversationStore( + "postgresql://duoc_thu:duoc_thu@localhost:5432/duoc_thu" + ) + store.migrate(CONVERSATION_MIGRATION) + conversation_id = f"test-{uuid.uuid4()}" + + store.append(conversation_id, "Người dùng: liều paracetamol") + store.append(conversation_id, "Trợ lý: cần biết đối tượng") + store.append(conversation_id, "Người dùng: người lớn") + + assert store.recent(conversation_id, limit=10) == [ + "Người dùng: liều paracetamol", + "Trợ lý: cần biết đối tượng", + "Người dùng: người lớn", + ] + # Windowing at read time: the oldest line falls outside a limit=2 read. + assert store.recent(conversation_id, limit=2) == [ + "Trợ lý: cần biết đối tượng", + "Người dùng: người lớn", + ] + # A conversation_id that was never written to reads back empty, not an + # error — the same "no history yet" case a fresh conversation hits live. + assert store.recent(f"never-seen-{uuid.uuid4()}", limit=10) == [] + + class _FakeJsonLlm: """Deterministic stand-in for the Bedrock Converse generator. Satisfies both `JsonLlm` (query understanding) and `AnswerGenerator` (answer + diff --git a/apps/ai-service/tests/test_qdrant_adapter.py b/apps/ai-service/tests/test_qdrant_adapter.py index 351ef49..e08c368 100644 --- a/apps/ai-service/tests/test_qdrant_adapter.py +++ b/apps/ai-service/tests/test_qdrant_adapter.py @@ -1,4 +1,36 @@ -from adapters.qdrant import _source_refs +from adapters.qdrant import QdrantRetriever, _source_refs + + +class _FakePoint: + def __init__(self, payload: dict) -> None: + self.payload = payload + self.score = 1.0 + + +class _FakeScrollClient: + """Mimics qdrant-client's `.scroll()` shape closely enough to exercise + `find_by_indication`'s keyword-matching logic directly — a fake filter + (not a real one), so it returns every payload handed to it regardless + of `scroll_filter`; the payloads given in each test already represent + what a real `section_key=chi_dinh, chunk_kind=prose` filter would have + returned, which is the part `find_by_indication` cannot get wrong on + its own (the filter construction itself is a one-line, inspectable + `Filter(must=[...])` — not worth a second fake layer to prove).""" + + def __init__(self, payloads: list[dict]) -> None: + self._payloads = payloads + + def scroll(self, collection_name, scroll_filter, limit, offset, with_payload): # noqa: ARG002 + return [_FakePoint(p) for p in self._payloads], None + + +def _chi_dinh_payload(drug_id: str, text: str) -> dict: + return { + "chunk_id": f"{drug_id}__chi_dinh__0", "drug_id": drug_id, + "drug_name": drug_id.upper(), "section_key": "chi_dinh", + "chunk_kind": "prose", "text": text, + "heading_physical_page": 100, "printed_page_range": [101, 101], + } def test_descriptor_source_ref_comes_from_attachment_not_heading_page(): @@ -45,3 +77,85 @@ def test_prose_ref_uses_exact_chunk_range_and_keeps_attachment_region(): assert refs[1].block_id == "p105_t0" assert refs[1].physical_page == 105 assert refs[1].printed_page == 106 + + +def test_find_by_indication_matches_a_drug_that_names_the_symptom(): + client = _FakeScrollClient([ + _chi_dinh_payload("paracetamol_acetaminophen", "Điều trị sốt, giảm đau nhẹ và vừa."), + _chi_dinh_payload("amoxicilin", "Điều trị nhiễm khuẩn đường hô hấp."), + ]) + retriever = QdrantRetriever(client, "duocthu_v1", embedder=None) + + hits = retriever.find_by_indication("sốt", limit=8) + + assert [h.document.drug_id for h in hits] == ["paracetamol_acetaminophen"] + + +def test_find_by_indication_requires_the_whole_phrase_not_a_scattered_match(): + """"sốt xuất huyết" (dengue) must not match a chunk that only says "sốt" + — the phrase itself has to appear, not just each of its words somewhere.""" + client = _FakeScrollClient([ + _chi_dinh_payload("paracetamol_acetaminophen", "Điều trị sốt, giảm đau."), + ]) + retriever = QdrantRetriever(client, "duocthu_v1", embedder=None) + + hits = retriever.find_by_indication("sốt xuất huyết", limit=8) + + assert hits == [] + + +def test_find_by_indication_matches_a_multi_word_phrase_contiguously(): + client = _FakeScrollClient([ + _chi_dinh_payload("paracetamol_acetaminophen", "Điều trị sốt cao ở người lớn."), + ]) + retriever = QdrantRetriever(client, "duocthu_v1", embedder=None) + + hits = retriever.find_by_indication("sốt cao", limit=8) + + assert [h.document.drug_id for h in hits] == ["paracetamol_acetaminophen"] + + +def test_find_by_indication_rejects_a_scattered_bag_of_common_words(): + """Found live 2026-08-07: a token-SUBSET match (every word present + *somewhere*, any order) let a long nonsense phrase built from common + filler words false-positive against real chi_dinh text — the words are + common enough to appear scattered through nearly anything. Phrase + matching closes it: none of these words are contiguous in the target + text the way they are in the query.""" + client = _FakeScrollClient([ + _chi_dinh_payload( + "paracetamol_acetaminophen", + "Điều trị sốt. Không dùng quá liều khuyến cáo trong sách hướng dẫn.", + ), + ]) + retriever = QdrantRetriever(client, "duocthu_v1", embedder=None) + + hits = retriever.find_by_indication( + "bệnh chưa từng ghi nhận trong sách abcxyz123", limit=8 + ) + + assert hits == [] + + +def test_find_by_indication_returns_at_most_one_hit_per_drug(): + client = _FakeScrollClient([ + _chi_dinh_payload("paracetamol_acetaminophen", "Điều trị sốt."), + {**_chi_dinh_payload("paracetamol_acetaminophen", "Điều trị sốt cao ở trẻ em."), + "chunk_id": "paracetamol_acetaminophen__chi_dinh__1"}, + ]) + retriever = QdrantRetriever(client, "duocthu_v1", embedder=None) + + hits = retriever.find_by_indication("sốt", limit=8) + + assert len(hits) == 1 + + +def test_find_by_indication_respects_the_limit(): + client = _FakeScrollClient([ + _chi_dinh_payload(f"drug_{i}", "Điều trị đau.") for i in range(5) + ]) + retriever = QdrantRetriever(client, "duocthu_v1", embedder=None) + + hits = retriever.find_by_indication("đau", limit=2) + + assert len(hits) == 2 diff --git a/apps/ai-service/tests/test_reasoning_loop.py b/apps/ai-service/tests/test_reasoning_loop.py deleted file mode 100644 index 5414039..0000000 --- a/apps/ai-service/tests/test_reasoning_loop.py +++ /dev/null @@ -1,244 +0,0 @@ -"""The loop must improve answers, and must be unable to run away. - -Bounded is the load-bearing property: an unbounded self-improvement loop on a -paid provider is a bill and a latency incident, and on a clinical tool it is -also an answer nobody is waiting for any more. -""" -from __future__ import annotations - -import pytest - -from rag.conversation import ConversationState, ResolvedQuestion -from rag.metrics import CLARIFY_ASKED, LOOP_REFINED, InMemoryMetrics -from rag.reasoning import ( - ClarifyReason, - DeterministicAssessor, - LoopTrace, - Sufficiency, - TurnBudget, - run_turn, -) - -ADULT = "Người lớn: uống 0,5 - 1 g/lần, cách 4 - 6 giờ; tối đa 4 g/ngày." -CHILD = "Trẻ em 6 - 12 tuổi: 240 - 250 mg mỗi lần." - - -def _q(text: str = "liều dùng paracetamol", population: str | None = None) -> ResolvedQuestion: - return ResolvedQuestion( - text=text, - drug_id="paracetamol", - section_key="lieu_luong_va_cach_dung", - population=population, - verbosity=None, - inherited_drug=False, - inherited_section=False, - ) - - -def _state() -> ConversationState: - return ConversationState("c1", turn_count=1) - - -class _Retriever: - """Returns a different evidence set on each round, recording calls.""" - - def __init__(self, *rounds: tuple[str, ...]) -> None: - self._rounds = list(rounds) - self.queries: list[str] = [] - - def __call__(self, resolved: ResolvedQuestion) -> tuple[str, ...]: - self.queries.append(resolved.text) - if self._rounds: - return self._rounds.pop(0) - return () - - -def _generator(answer: str | None): - calls = {"n": 0} - - def generate(resolved, evidence, state): - calls["n"] += 1 - return answer - - generate.calls = calls # type: ignore[attr-defined] - return generate - - -# --- the loop earns its rounds ------------------------------------------------ - - -def test_a_named_gap_buys_exactly_one_more_round(): - """Asked for adults, first round returned only paediatric text.""" - retriever = _Retriever((CHILD,), (ADULT, CHILD)) - metrics = InMemoryMetrics() - - outcome = run_turn( - _state(), - _q(population="nguoi_lon"), - retriever, - _generator("Người lớn: 0,5 - 1 g/lần [1]"), - metrics=metrics, - ) - - assert outcome.retrieval_rounds_used == 2 - assert outcome.generated is True - assert metrics.total(LOOP_REFINED, missing="population:nguoi_lon") == 1 - assert retriever.queries[1] != retriever.queries[0] - - -def test_a_satisfied_question_spends_one_round_only(): - retriever = _Retriever((ADULT,)) - - outcome = run_turn( - _state(), _q(population="nguoi_lon"), retriever, _generator("ok [1]") - ) - - assert outcome.retrieval_rounds_used == 1 - assert outcome.stopped_because == "sufficient" - - -def test_a_simple_question_does_not_loop(): - """No population asked for means nothing to be missing.""" - retriever = _Retriever((ADULT, CHILD)) - - outcome = run_turn(_state(), _q(), retriever, _generator("ok [1]")) - - assert outcome.retrieval_rounds_used == 1 - - -# --- the loop cannot run away ------------------------------------------------- - - -def test_retrieval_rounds_are_hard_capped(): - """Evidence never satisfies the assessor; the loop must still stop.""" - retriever = _Retriever((CHILD,), (CHILD,), (CHILD,), (CHILD,), (CHILD,)) - - outcome = run_turn( - _state(), - _q(population="nguoi_lon"), - retriever, - _generator("ok [1]"), - budget=TurnBudget(retrieval_rounds=2), - ) - - assert outcome.retrieval_rounds_used == 2 - assert outcome.stopped_because == "retrieval_budget" - assert len(retriever.queries) == 2 - - -def test_repairs_are_hard_capped_and_degrade_to_no_answer(): - """`generate` returning None means verification refused it every time.""" - generate = _generator(None) - - outcome = run_turn( - _state(), - _q(), - _Retriever((ADULT,)), - generate, - budget=TurnBudget(repairs=1, llm_calls=4), - ) - - assert outcome.answer is None - assert outcome.repairs_used == 1 - assert generate.calls["n"] == 2 # first attempt + one repair - assert outcome.stopped_because == "repair_budget" - - -def test_llm_call_budget_stops_generation_entirely(): - generate = _generator(None) - - outcome = run_turn( - _state(), _q(), _Retriever((ADULT,)), generate, budget=TurnBudget(llm_calls=0) - ) - - assert generate.calls["n"] == 0 - assert outcome.stopped_because == "llm_budget" - - -def test_a_refinement_that_changes_nothing_stops_the_loop(): - """Guards against a loop that keeps re-issuing the same query.""" - - class _SameQuery: - def assess(self, resolved, evidence): - return Sufficiency(False, missing="x", refined_query=resolved.text) - - retriever = _Retriever((CHILD,), (CHILD,)) - - outcome = run_turn( - _state(), _q(), retriever, _generator("ok [1]"), assessor=_SameQuery() - ) - - assert outcome.stopped_because == "query_unchanged" - assert len(retriever.queries) == 1 - - -def test_an_unnamed_gap_does_not_buy_a_round(): - """"Feels incomplete" is not a reason to spend the budget.""" - - class _Vague: - def assess(self, resolved, evidence): - return Sufficiency(False) - - retriever = _Retriever((CHILD,), (CHILD,)) - - outcome = run_turn(_state(), _q(), retriever, _generator("ok [1]"), assessor=_Vague()) - - assert outcome.stopped_because == "no_actionable_gap" - assert len(retriever.queries) == 1 - - -# --- clarify beats guessing --------------------------------------------------- - - -@pytest.mark.parametrize( - "signal", - [ClarifyReason.NO_ATTRIBUTE, ClarifyReason.AMBIGUOUS_DRUG, ClarifyReason.MULTI_ATTRIBUTE], -) -def test_a_clarify_signal_short_circuits_before_any_spend(signal): - retriever = _Retriever((ADULT,)) - generate = _generator("ok [1]") - metrics = InMemoryMetrics() - budget = TurnBudget() - - outcome = run_turn( - _state(), _q(), retriever, generate, clarify_signals=(signal,), budget=budget, metrics=metrics - ) - - assert outcome.clarification is not None - assert outcome.clarification.reason == signal - assert outcome.answer is None - assert retriever.queries == [] - assert generate.calls["n"] == 0 - assert budget.llm_calls == 4 and budget.retrieval_rounds == 2 - assert metrics.total(CLARIFY_ASKED, reason=signal) == 1 - - -def test_no_evidence_at_all_asks_rather_than_abstaining_silently(): - outcome = run_turn(_state(), _q(), _Retriever(()), _generator("ok [1]")) - - assert outcome.clarification is not None - assert outcome.clarification.reason == ClarifyReason.STILL_INSUFFICIENT - assert outcome.stopped_because == "no_evidence" - - -# --- the deterministic assessor ---------------------------------------------- - - -def test_assessor_only_reports_gaps_it_can_demonstrate(): - assessor = DeterministicAssessor() - - assert assessor.assess(_q(population="nguoi_lon"), (ADULT,)).sufficient is True - assert assessor.assess(_q(population="nguoi_lon"), (CHILD,)).sufficient is False - # No population asked for: nothing can be shown missing. - assert assessor.assess(_q(), (CHILD,)).sufficient is True - - -def test_trace_records_the_stages_walked(): - trace = LoopTrace() - - run_turn(_state(), _q(), _Retriever((ADULT,)), _generator("ok [1]"), trace=trace) - - assert trace.stages[0] == "understand" - assert "retrieve" in trace.stages - assert "assess" in trace.stages - assert trace.stages[-1] == "generate" diff --git a/apps/ai-service/tests/test_retrieval_service.py b/apps/ai-service/tests/test_retrieval_service.py index 5d7c8f4..cecc332 100644 --- a/apps/ai-service/tests/test_retrieval_service.py +++ b/apps/ai-service/tests/test_retrieval_service.py @@ -339,3 +339,78 @@ def test_recommendation_intent_is_refused_at_policy_boundary(): ) assert result.decision == EvidenceDecision.ABSTAIN assert result.reason == "recommendation_out_of_scope" + + +class _IndicationRetriever: + """A fake exposing only `find_by_indication`/`search_indication` (the + Qdrant adapter's shape for the reverse-lookup path), so + `retrieve_by_indication`'s own orchestration — keyword first, dense + fallback only when keyword finds nothing — is what's under test here, + not the matching algorithm itself (that's `test_qdrant_adapter.py`'s job).""" + + def __init__( + self, + keyword_hits: list[SearchHit] | None = None, + dense_hits: list[SearchHit] | None = None, + ) -> None: + self._keyword_hits = keyword_hits or [] + self._dense_hits = dense_hits or [] + self.dense_called = False + + def find_by_indication(self, indication_text, limit): # noqa: ARG002 + return self._keyword_hits + + def search_indication(self, query, limit): # noqa: ARG002 + self.dense_called = True + return self._dense_hits + + +def _indication_hit(drug_id: str) -> SearchHit: + return SearchHit( + document=RetrievalDocument( + doc_id=f"{drug_id}__chi_dinh__0", drug_id=drug_id, kind="prose", + section_key="chi_dinh", text="Điều trị sốt.", source_refs=(SOURCE,), + ), + score=1.0, + ) + + +def test_retrieve_by_indication_uses_keyword_hits_without_trying_dense(): + retriever = _IndicationRetriever(keyword_hits=[_indication_hit("paracetamol_acetaminophen")]) + service = RetrievalService(retriever, InMemoryParentStore([])) + + result = service.retrieve_by_indication("sốt") + + assert result.decision == EvidenceDecision.ANSWERABLE + assert len(result.evidence) == 1 + assert retriever.dense_called is False + + +def test_retrieve_by_indication_falls_back_to_dense_only_when_keyword_is_empty(): + retriever = _IndicationRetriever(dense_hits=[_indication_hit("ibuprofen")]) + service = RetrievalService(retriever, InMemoryParentStore([])) + + result = service.retrieve_by_indication("thân nhiệt tăng") + + assert result.decision == EvidenceDecision.ANSWERABLE + assert retriever.dense_called is True + + +def test_retrieve_by_indication_with_no_match_anywhere_abstains(): + retriever = _IndicationRetriever() + service = RetrievalService(retriever, InMemoryParentStore([])) + + result = service.retrieve_by_indication("bệnh chưa từng ghi nhận") + + assert result.decision == EvidenceDecision.ABSTAIN + assert result.reason == "no_indication_match" + + +def test_retrieve_by_indication_with_blank_text_abstains_without_calling_retrieval(): + retriever = _IndicationRetriever(keyword_hits=[_indication_hit("paracetamol_acetaminophen")]) + service = RetrievalService(retriever, InMemoryParentStore([])) + + result = service.retrieve_by_indication(" ") + + assert result.decision == EvidenceDecision.ABSTAIN + assert result.reason == "missing_indication" diff --git a/apps/ai-service/tests/test_section_routing.py b/apps/ai-service/tests/test_section_routing.py index 4adb1c1..5ac5076 100644 --- a/apps/ai-service/tests/test_section_routing.py +++ b/apps/ai-service/tests/test_section_routing.py @@ -77,7 +77,8 @@ CONTRA = [ ] INDICATION = [_doc("i1", "chi_dinh", "Giảm đau, hạ sốt, chống viêm.")] PHARMACOLOGY = [_doc("p1", "duoc_ly_va_co_che_tac_dung", "Ức chế cyclooxygenase.")] -ALL_DOCS = CONTRA + INDICATION + PHARMACOLOGY +PRECAUTION = [_doc("t1", "than_trong", "Thận trọng với người suy thận.")] +ALL_DOCS = CONTRA + INDICATION + PHARMACOLOGY + PRECAUTION def _service(retriever, resolver: SectionResolver | None) -> RetrievalService: @@ -207,6 +208,22 @@ class TestSectionRouting: assert retriever.section_calls == [] assert retriever.search_calls + def test_than_trong_also_pools_chong_chi_dinh(self) -> None: + """A precaution some drug's own "thận trọng" text never mentions can + still be filed under "chống chỉ định" (found live: Aspirin + loét dạ + dày). Pooling both keeps that answerable instead of a false "not in + this source" clarify/abstain.""" + retriever = SectionAwareRetriever(ALL_DOCS) + result = _service(retriever, SectionResolver()).retrieve( + "Thận trọng khi dùng aspirin là gì?", "aspirin" + ) + assert retriever.section_calls == [ + ("aspirin", "than_trong"), ("aspirin", "chong_chi_dinh"), + ] + assert {item.evidence_id for item in result.evidence} == { + "t1", "c1", "c2", "c3", "c4", "c5", + } + def test_named_but_empty_section_falls_back(self) -> None: """A drug with no such section must not abstain — similarity still tries.""" retriever = SectionAwareRetriever(INDICATION + PHARMACOLOGY) diff --git a/apps/ai-service/tests/test_understanding.py b/apps/ai-service/tests/test_understanding.py index ca9d1e0..9c41322 100644 --- a/apps/ai-service/tests/test_understanding.py +++ b/apps/ai-service/tests/test_understanding.py @@ -10,7 +10,13 @@ from __future__ import annotations import json -from rag.understanding import SECTION_KEY_HINTS, SECTION_KEYS, LlmQueryUnderstander +from rag.ports import AnswerGenerationUnavailable +from rag.understanding import ( + SECTION_KEY_HINTS, + SECTION_KEYS, + LlmQueryUnderstander, + QueryFrame, +) CATALOG = { "paracetamol_acetaminophen": "paracetamol acetaminophen, PARACETAMOL", @@ -23,6 +29,8 @@ class _FixedLlm: self._payload = payload def generate(self, system: str, user: str, schema: dict) -> str: + if isinstance(self._payload, BaseException): + raise self._payload if isinstance(self._payload, str): return self._payload return json.dumps(self._payload, ensure_ascii=False) @@ -139,11 +147,108 @@ def test_fuzzy_suggestion_bounds_a_typo_into_the_candidate_set(): assert frame.drugs == ("metformin",) +# --- F-10: a small battery of invented near-alias shapes, beyond the single +# "aspirinol" case above — each simulates a different way a name could be +# crafted to *look* like it should fuzzy-match a real drug --------------- + + +def test_a_real_drug_name_with_a_brand_like_suffix_is_not_substituted(): + resolver = _FakeResolver({}) # nothing in this turn resolves or suggests + understander = LlmQueryUnderstander(_FixedLlm({ + "turn_type": "drug_attribute", "drugs": [], + "unknown_drugs": ["metforminex"], "attribute": None, "population": None, + "weight_kg": None, "age_text": None, "indication": None, + "needs_clarify": False, "clarify_reason": None, + }), CATALOG, resolver) + frame = understander.understand("liều metforminex") + assert frame.drugs == () + assert frame.unknown_drugs == ("metforminex",) + + +def test_a_name_blending_two_real_drugs_is_not_substituted_for_either(): + resolver = _FakeResolver({}) + understander = LlmQueryUnderstander(_FixedLlm({ + "turn_type": "drug_attribute", "drugs": ["metformin", "paracetamol_acetaminophen"], + "unknown_drugs": [], "attribute": None, "population": None, + "weight_kg": None, "age_text": None, "indication": None, + "needs_clarify": False, "clarify_reason": None, + }), CATALOG, resolver) + frame = understander.understand("liều metformacetamol") + # Neither real id has deterministic candidate support for this turn — + # F-04's bound must reject both, not accept the ones that happen to be + # real catalog members. + assert frame.drugs == () + assert "metformin" in frame.unknown_drugs + assert "paracetamol_acetaminophen" in frame.unknown_drugs + + def test_malformed_json_fails_closed_to_a_clarify(): understander = LlmQueryUnderstander(_FixedLlm("not json"), CATALOG, RESOLVER) frame = understander.understand("gì đó") assert frame.turn_type == "out_of_scope" assert frame.needs_clarify is True + assert frame.quick_replies == () + + +def test_quick_replies_are_parsed_when_the_model_offers_them(): + understander = LlmQueryUnderstander(_FixedLlm({ + "turn_type": "dosing_calc", "drugs": ["paracetamol_acetaminophen"], + "unknown_drugs": [], "attribute": None, "population": None, + "weight_kg": None, "age_text": None, "indication": None, + "needs_clarify": True, "clarify_reason": "Người lớn hay trẻ em?", + "quick_replies": ["Người lớn", "Trẻ em"], + }), CATALOG, RESOLVER) + frame = understander.understand("liều paracetamol") + assert frame.quick_replies == ("Người lớn", "Trẻ em") + + +def test_missing_quick_replies_key_defaults_to_empty_not_a_crash(): + """The model is asked for `quick_replies` but structured-output providers + aren't guaranteed to include every optional key — a clarify without it + must still parse, just with no chips.""" + understander = LlmQueryUnderstander(_FixedLlm({ + "turn_type": "dosing_calc", "drugs": [], + "unknown_drugs": [], "attribute": None, "population": None, + "weight_kg": None, "age_text": None, "indication": None, + "needs_clarify": True, "clarify_reason": "Cân nặng bao nhiêu kg?", + }), CATALOG, RESOLVER) + frame = understander.understand("liều cho bé") + assert frame.quick_replies == () + + +def test_route_is_parsed_when_the_model_resolves_it(): + """The 2026-08-07 bug: a bare reply like 'Uống' answering the model's own + prior route question had nowhere to be recorded (QueryFrame had no route + field), so the model could only repeat its clarify_reason verbatim.""" + understander = LlmQueryUnderstander(_FixedLlm({ + "turn_type": "dosing_calc", "drugs": ["paracetamol_acetaminophen"], + "unknown_drugs": [], "attribute": "lieu_luong_va_cach_dung", + "population": "nguoi_lon", "weight_kg": None, "age_text": None, + "indication": None, "route": "uong", + "needs_clarify": False, "clarify_reason": None, + }), CATALOG, RESOLVER) + frame = understander.understand( + "Uống", + history=( + "Người dùng: Liều paracetamol hạ sốt là bao nhiêu?", + "Trợ lý: Người lớn hay trẻ em? Uống hay đặt trực tràng?", + "Người dùng: Người lớn", + "Trợ lý: Uống hay đặt trực tràng?", + ), + ) + assert frame.route == "uong" + assert frame.needs_clarify is False + + +def test_missing_route_key_defaults_to_none_not_a_crash(): + understander = LlmQueryUnderstander(_FixedLlm({ + "turn_type": "dosing_calc", "drugs": [], + "unknown_drugs": [], "attribute": None, "population": None, + "weight_kg": None, "age_text": None, "indication": None, + "needs_clarify": False, "clarify_reason": None, + }), CATALOG, RESOLVER) + frame = understander.understand("liều metformin") + assert frame.route is None def test_unrecognised_turn_type_falls_back_based_on_whether_a_drug_resolved(): @@ -200,3 +305,139 @@ def test_invalid_attribute_is_dropped_not_passed_through(): }), CATALOG, RESOLVER) frame = understander.understand("metformin") assert frame.attribute is None + + +# --- F-10: provider outage during the ONE call site that had no error +# handling at all ------------------------------------------------------------ + + +# --- F-11: prior-frame merge — the code-level backstop for the model +# dropping an already-established slot mid clarify-chain. Found live +# 2026-08-07 (50-question hand-typed browser audit): reproduced 3 times +# independently as either a non-terminating re-ask of the same clarify +# question, or a stale drug bleeding into an unrelated new topic. -------- + + +def test_prior_frame_known_fields_survive_a_short_reply_the_model_drops(): + """The Insulin/Azithromycin shape: the new call's own JSON comes back + with the just-answered field null (a real, observed failure — the model + is asked to restate it and sometimes doesn't), but since this turn named + no drug of its own (a short reply like "20kg" never does), the previously + established fields must survive via the merge, not be silently lost.""" + understander = LlmQueryUnderstander(_FixedLlm({ + "turn_type": "dosing_calc", "drugs": [], + "unknown_drugs": [], "attribute": None, "population": None, + "weight_kg": 20, "age_text": None, "indication": None, + "needs_clarify": False, "clarify_reason": None, + }), CATALOG, RESOLVER) + prior = QueryFrame( + turn_type="dosing_calc", drugs=("paracetamol_acetaminophen",), + attribute="lieu_luong_va_cach_dung", needs_clarify=True, + clarify_reason="Bé nặng bao nhiêu kg?", + ) + frame = understander.understand("bé nặng 20 cân", prior_frame=prior) + assert frame.drugs == ("paracetamol_acetaminophen",) + assert frame.attribute == "lieu_luong_va_cach_dung" + assert frame.weight_kg == 20 + + +def test_prior_frame_is_not_merged_when_the_turn_resolves_a_different_drug(): + """The headache/OMEPRAZOL bleed this guards against: a turn that itself + names a real, different drug is a genuine topic change and must not + inherit the old drug's population/weight/route — merging here would + reproduce the exact bug being fixed.""" + understander = LlmQueryUnderstander(_FixedLlm({ + "turn_type": "drug_attribute", "drugs": ["metformin"], + "unknown_drugs": [], "attribute": "chi_dinh", "population": None, + "weight_kg": None, "age_text": None, "indication": None, + "needs_clarify": False, "clarify_reason": None, + }), CATALOG, RESOLVER) + prior = QueryFrame( + turn_type="drug_attribute", drugs=("paracetamol_acetaminophen",), + population="tre_em", weight_kg=20, needs_clarify=True, + clarify_reason="Bé nặng bao nhiêu kg?", + ) + frame = understander.understand("chỉ định của metformin là gì", prior_frame=prior) + assert frame.drugs == ("metformin",) + assert frame.population is None + assert frame.weight_kg is None + + +def test_prior_frame_that_was_already_resolved_is_not_merged(): + """A prior turn that already answered (needs_clarify=False) has nothing + to continue — merging it into a brand-new turn would leak stale state + into an unrelated question that happens to follow it.""" + understander = LlmQueryUnderstander(_FixedLlm({ + "turn_type": "smalltalk", "drugs": [], + "unknown_drugs": [], "attribute": None, "population": None, + "weight_kg": None, "age_text": None, "indication": None, + "needs_clarify": False, "clarify_reason": None, + }), CATALOG, RESOLVER) + prior = QueryFrame( + turn_type="drug_attribute", drugs=("metformin",), + population="nguoi_lon", needs_clarify=False, + ) + frame = understander.understand("cảm ơn bạn", prior_frame=prior) + assert frame.drugs == () + assert frame.population is None + + +def test_known_facts_block_is_sent_to_the_model_on_a_clarify_continuation(): + captured = {} + + class _CapturingLlm: + def generate(self, system, user, schema): + captured["user"] = user + return json.dumps({ + "turn_type": "dosing_calc", "drugs": ["paracetamol_acetaminophen"], + "unknown_drugs": [], "attribute": None, "population": None, + "weight_kg": 20, "age_text": None, "indication": None, + "needs_clarify": False, "clarify_reason": None, + }) + + understander = LlmQueryUnderstander(_CapturingLlm(), CATALOG, RESOLVER) + prior = QueryFrame( + turn_type="dosing_calc", drugs=("paracetamol_acetaminophen",), + population="tre_em", needs_clarify=True, + clarify_reason="Bé nặng bao nhiêu kg?", + ) + understander.understand("20 cân", prior_frame=prior) + + assert "THÔNG TIN ĐÃ XÁC ĐỊNH" in captured["user"] + assert "paracetamol_acetaminophen" in captured["user"] + + +def test_no_known_facts_block_when_there_is_no_prior_clarify(): + captured = {} + + class _CapturingLlm: + def generate(self, system, user, schema): + captured["user"] = user + return json.dumps({ + "turn_type": "smalltalk", "drugs": [], + "unknown_drugs": [], "attribute": None, "population": None, + "weight_kg": None, "age_text": None, "indication": None, + "needs_clarify": False, "clarify_reason": None, + }) + + understander = LlmQueryUnderstander(_CapturingLlm(), CATALOG, RESOLVER) + understander.understand("chào bạn") + assert "THÔNG TIN ĐÃ XÁC ĐỊNH" not in captured["user"] + + +def test_provider_outage_fails_closed_to_a_clarify_not_an_unhandled_crash(): + """Found live 2026-08-07: unlike every other LLM call site in this + product, `understand()` had no try/except around its call at all — a + Bedrock outage here propagated straight through `RagAgent.handle()` + into an unhandled 500 (`routers/rag.py` only wraps the trace-save call, + not `agent.handle()`), instead of the graceful abstain every other + failure mode already gets.""" + understander = LlmQueryUnderstander( + _FixedLlm(AnswerGenerationUnavailable("Bedrock unreachable")), + CATALOG, RESOLVER, + ) + frame = understander.understand("liều paracetamol cho người lớn") + assert frame.needs_clarify is True + assert frame.clarify_reason is not None + assert frame.drugs == () + assert frame.quick_replies == () diff --git a/apps/web/Dockerfile b/apps/web/Dockerfile new file mode 100644 index 0000000..40c864f --- /dev/null +++ b/apps/web/Dockerfile @@ -0,0 +1,24 @@ +FROM node:20-slim AS base +RUN corepack enable +WORKDIR /repo + +FROM base AS deps +COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./ +COPY apps/web/package.json apps/web/package.json +COPY packages/shared-types/package.json packages/shared-types/package.json +COPY packages/api-client/package.json packages/api-client/package.json +COPY packages/ui/package.json packages/ui/package.json +COPY packages/config/package.json packages/config/package.json +RUN pnpm install --frozen-lockfile + +FROM deps AS build +COPY packages/ packages/ +COPY apps/web/ apps/web/ +RUN pnpm --filter @duoc-thu/web build + +FROM base AS runtime +ENV NODE_ENV=production +COPY --from=build /repo /repo +WORKDIR /repo/apps/web +EXPOSE 3000 +CMD ["pnpm", "start", "--", "-p", "3000", "-H", "0.0.0.0"] diff --git a/apps/web/app/_components/ChatPanel.tsx b/apps/web/app/_components/ChatPanel.tsx index 2709172..6d707ac 100644 --- a/apps/web/app/_components/ChatPanel.tsx +++ b/apps/web/app/_components/ChatPanel.tsx @@ -20,7 +20,7 @@ import { cn } from "@duoc-thu/ui"; interface ChatPanelProps { sessionId: string; initialQuery?: string; - onCitationClick?: (citation: Citation, index: number) => void; + onCitationClick?: (citation: Citation, index: number, allCitations: Citation[]) => void; onCitationsLoaded?: (citations: Citation[]) => void; activeCitationIndex?: number | null; className?: string; @@ -63,6 +63,7 @@ export function ChatPanel({ const [error, setError] = useState(null); const messagesEndRef = useRef(null); const abortControllerRef = useRef(null); + const initialQuerySentRef = useRef(undefined); const scrollToBottom = () => { messagesEndRef.current?.scrollIntoView({ behavior: "smooth" }); @@ -132,7 +133,15 @@ export function ChatPanel({ }; useEffect(() => { - if (initialQuery) { + // Guard against firing twice for the same query: React 18 Strict Mode + // (dev only) runs this effect setup twice on mount, and with no guard + // that sent every quick-prompt click as two identical live requests + // (found live 2026-08-07: duplicate "Chỉ định & Tác dụng không mong + // muốn của Aspirin" turns in the trace). The ref persists across the + // Strict Mode replay, so the second invocation for the same + // `initialQuery` is a no-op; a genuinely new query still sends once. + if (initialQuery && initialQuerySentRef.current !== initialQuery) { + initialQuerySentRef.current = initialQuery; handleSendMessage(initialQuery); } // eslint-disable-next-line react-hooks/exhaustive-deps @@ -291,15 +300,31 @@ export function ChatPanel({ {messages.length === 0 ? ( renderEmptyState() ) : ( - messages.map((msg) => ( - onCitationClick?.(citation, idx)} - activeCitationIndex={activeCitationIndex} - onRetry={() => handleSendMessage(msg.content)} - /> - )) + messages.map((msg, msgIdx) => { + // Retry must resend the ORIGINAL user question, not this + // bubble's own text — for an assistant bubble, `msg.content` is + // the answer/error text itself, so resending it fed the error + // message back in as if it were the next question (found live + // 2026-08-07: a trace row where the query text WAS literally + // "Dịch vụ đang gặp sự cố tạm thời..."). Walk back to the + // nearest preceding user turn instead. + const retryQuery = + msg.role === "assistant" + ? [...messages.slice(0, msgIdx)].reverse().find((m) => m.role === "user")?.content + : undefined; + return ( + + onCitationClick?.(citation, idx, allCitations) + } + activeCitationIndex={activeCitationIndex} + onRetry={retryQuery ? () => handleSendMessage(retryQuery) : undefined} + onQuickReply={(text) => handleSendMessage(text)} + /> + ); + }) )} {/* Loading Indicator */} diff --git a/apps/web/app/api/chat/route.ts b/apps/web/app/api/chat/route.ts index 12a73e7..f15ab8c 100644 --- a/apps/web/app/api/chat/route.ts +++ b/apps/web/app/api/chat/route.ts @@ -13,9 +13,11 @@ interface RagCitation { printed_page_start: number; printed_page_end: number; physical_page: number; + block_id?: string | null; + bbox?: [number, number, number, number] | null; + source_crop?: string | null; attachment?: string | null; - text_snippet?: string | null; - citation_reason?: string | null; + evidence_text?: string | null; } interface RagResponse { @@ -25,6 +27,8 @@ interface RagResponse { answer: string | null; resolved_drug_id: string | null; citations: RagCitation[]; + generated?: boolean; + quick_replies?: string[]; } const REFUSALS: Record = { @@ -32,31 +36,123 @@ const REFUSALS: Record = { "Chưa xác định được thuốc trong câu hỏi này, nên hệ thống không đưa ra nội dung chuyên môn. Vui lòng nêu rõ tên hoạt chất cần tra cứu (ví dụ: Paracetamol, Amoxicillin...).", drug_resolution_ambiguous: "Câu hỏi có thể ứng với nhiều thuốc khác nhau. Vui lòng nêu rõ tên hoạt chất cần tra cứu.", + drug_resolution_invalid_state: + "Có lỗi nội bộ khi xác định thuốc trong câu hỏi này. Vui lòng thử lại.", recommendation_out_of_scope: "Đây là câu hỏi xin tư vấn hoặc quyết định điều trị. Hệ thống chỉ tra cứu Dược thư và không đưa ra khuyến cáo điều trị — vui lòng hỏi bác sĩ hoặc dược sĩ.", out_of_scope_non_human: "Dược thư Quốc gia Việt Nam áp dụng cho người. Hệ thống không tra cứu cho đối tượng khác.", subject_scope_unknown: "Chưa rõ câu hỏi áp dụng cho đối tượng nào, nên hệ thống không trả lời.", + query_intent_unknown: + "Chưa rõ mục đích câu hỏi (tra cứu thông tin hay xin tư vấn điều trị). Vui lòng đặt lại câu hỏi cụ thể hơn.", query_embedding_unavailable: "Chưa tra được mục tương ứng cho câu hỏi này. Vui lòng nêu rõ thuộc tính cần tra (liều dùng, chống chỉ định, tương tác thuốc…).", insufficient_retrieval_score: "Không tìm thấy nội dung đủ liên quan trong Dược thư cho câu hỏi này.", + missing_query_or_drug: + "Câu hỏi hoặc tên thuốc chưa đủ rõ để tra cứu. Vui lòng nêu rõ tên thuốc và nội dung cần tra.", + missing_indication: + "Vui lòng nêu rõ triệu chứng hoặc chỉ định cần tra thuốc (ví dụ: sốt, đau đầu).", + no_indication_match: + "Không tìm thấy thuốc nào trong Dược thư ghi nhận chỉ định phù hợp với triệu chứng này.", + parent_hydration_failed: + "Có lỗi khi tổng hợp dữ liệu nhiều thuốc trong câu hỏi này. Vui lòng thử lại.", + missing_provenance: + "Không xác định được nguồn trang cho nội dung này nên hệ thống không thể trích dẫn. Vui lòng thử lại.", + missing_printed_page_provenance: + "Không xác định được nguồn trang cho nội dung này nên hệ thống không thể trích dẫn. Vui lòng thử lại.", + // The backend DID retrieve real evidence for every code below — none of + // these are missing-data cases. `rag/answer.py` now propagates the + // SPECIFIC safety check that rejected a generation instead of collapsing + // them all into "generation_unavailable" (found live 2026-08-07: the + // collapsed version made a real provider outage indistinguishable from + // ordinary entailment noise, both from here and from server metrics). + // Every one of these needs its own entry for the exact reason the + // now-fixed `generation_unavailable` case did: an unmapped reason here + // silently reads as "no data in the formulary", which is false. + request_budget_exhausted: + "Hệ thống mất quá nhiều thời gian xử lý câu hỏi này. Vui lòng thử lại.", + provider_unavailable: + "Không thể kết nối dịch vụ AI để tạo câu trả lời lúc này. Vui lòng thử lại sau ít phút.", + malformed_output: + "Hệ thống nhận được phản hồi không hợp lệ khi tạo câu trả lời. Vui lòng thử lại.", + evidence_insufficient: + "Dược thư có nội dung liên quan đến câu hỏi này, nhưng hệ thống chưa xác định đủ cơ sở để trả lời chắc chắn. Vui lòng thử lại hoặc nêu rõ hơn câu hỏi.", + ungrounded_number: + "Hệ thống phát hiện số liệu trong câu trả lời không khớp với nguồn nên đã huỷ để tránh sai sót. Vui lòng thử lại.", + invalid_citation: + "Hệ thống phát hiện trích dẫn không hợp lệ trong câu trả lời nên đã huỷ để tránh sai sót. Vui lòng thử lại.", + uncited_claim: + "Hệ thống phát hiện một phần câu trả lời không có trích dẫn nguồn rõ ràng nên đã huỷ để tránh sai sót. Vui lòng thử lại.", + unsupported_claim: + "Dược thư có nội dung liên quan đến câu hỏi này, nhưng bước đối chiếu lại chưa xác nhận được câu trả lời khớp hoàn toàn với nguồn. Vui lòng thử lại.", + // Kept as the fallback `answer.py` itself falls back to when, for some + // reason, none of the specific codes above was set. + generation_unavailable: + "Dược thư có nội dung liên quan đến câu hỏi này, nhưng hệ thống chưa tạo được câu trả lời đã kiểm chứng đầy đủ (có thể do lỗi tạm thời). Vui lòng bấm Thử lại.", + // `agent.py`'s clarify-loop circuit breaker (found live 2026-08-07: the + // understanding LLM could re-ask the same clarifying question forever, + // reproduced 3 times independently, one case never converged after 5 real + // turns). The backend always supplies its own `answer` text for this + // reason, so this entry is a fallback only. + clarify_loop_exhausted: + "Hệ thống chưa xác định đủ thông tin sau nhiều lần hỏi lại. Vui lòng gõ lại toàn bộ câu hỏi trong một tin nhắn đầy đủ, hoặc bấm \"Tạo phiên tra cứu mới\".", }; +// Only a truly unclassified reason code reaches this — every abstain path +// the backend actually produces (see rag/routing.py, rag/service.py, +// rag/answer.py) has a specific entry above. This must stay narrow: an +// unmapped reason silently reading as "no data in the formulary" is exactly +// the bug fixed 2026-08-07 (generation_unavailable was falling through here). const GENERIC_REFUSAL = - "Hệ thống không tìm thấy căn cứ trong Dược thư để trả lời câu hỏi này."; + "Hệ thống không thể xử lý câu hỏi này lúc này. Vui lòng thử lại."; -function toCitations(raw: RagCitation[], resolvedDrugId: string | null): Citation[] { - return raw.map((item) => { - const parts = item.chunk_id.split("__"); - const sectionName = parts.length > 1 ? parts[1] : ""; +// Chunk ids are always `{drug_id}__{section_key}__{part_index}` — drug_id and +// section_key use single underscores internally, so splitting on the double +// underscore reliably recovers both per citation. This must be derived per +// citation, not from the turn's single `resolved_drug_id`: a 2-drug +// interaction answer cites both drugs, and stamping every citation with one +// drug name would misattribute half of them. +// +// The backend emits one raw citation per `source_ref` of an evidence block — +// a quarantined chunk has both a plain-text ref (where the prose sits) and +// an attachment ref (where the table/formula actually sits, which can be a +// different physical page than the prose that mentions it — confirmed on +// real data, not assumed). Both refs share the same `chunk_id` and the same +// `evidence_text`, so they're grouped into ONE card here instead of showing +// two near-identical ones — the attachment ref's own page is kept as +// `quarantinePhysicalPage` rather than discarded. +function toCitations(raw: RagCitation[]): Citation[] { + const byChunk = new Map(); + for (const item of raw) { + const group = byChunk.get(item.chunk_id); + if (group) group.push(item); + else byChunk.set(item.chunk_id, [item]); + } + + return Array.from(byChunk.entries()).map(([chunkId, group]) => { + const primary = group.find((g) => !g.attachment) ?? group[0]; + const attachmentRef = group.find((g) => g.attachment); + const [drugSlug, sectionKey] = chunkId.split("__"); + const isQuarantined = Boolean(attachmentRef); return { - drugName: resolvedDrugId ?? parts[0] ?? item.chunk_id, - sectionType: sectionName, - sourcePageRange: [item.printed_page_start, item.printed_page_end], - snippet: item.text_snippet ?? undefined, - reason: item.citation_reason ?? `Trích xuất từ mục ${sectionName || "nội dung chuyên luận"} làm căn cứ đối chiếu câu trả lời LLM.`, + chunkId, + drugName: drugSlug ? drugSlug.replace(/_/g, " ").toUpperCase() : chunkId, + sectionType: sectionKey ?? "", + sourcePageRange: [primary.printed_page_start, primary.printed_page_end], + physicalPage: primary.physical_page, + snippet: primary.evidence_text ?? "", + isQuarantined, + quarantineNotice: attachmentRef + ? `Có bảng hoặc công thức tại trang in ${attachmentRef.printed_page_start}${ + attachmentRef.printed_page_end !== attachmentRef.printed_page_start + ? `–${attachmentRef.printed_page_end}` + : "" + } chưa được số hóa tự động — không suy ra số liệu từ đây, cần đối chiếu trực tiếp ảnh PDF gốc.` + : undefined, + quarantinePhysicalPage: attachmentRef?.physical_page, + sourceCropUrl: attachmentRef?.source_crop ?? undefined, }; }); } @@ -135,14 +231,19 @@ export async function POST(request: Request) { id: rag.trace_id || `msg-${Date.now()}`, role: "assistant", content: noAnswer ? (REFUSALS[rag.reason] ?? GENERIC_REFUSAL) : (rag.answer ?? GENERIC_REFUSAL), - citations: isAbstain || noAnswer ? [] : toCitations(rag.citations, rag.resolved_drug_id), + citations: isAbstain || noAnswer ? [] : toCitations(rag.citations), disclaimer: DISCLAIMER, traceId: rag.trace_id, decision: rag.decision, reason: rag.reason, grounded: !isAbstain && !noAnswer, + generated: !isAbstain && !noAnswer ? Boolean(rag.generated) : false, resolvedDrugId: rag.resolved_drug_id ?? undefined, createdAt: new Date().toISOString(), + quickReplies: + rag.decision === "clarify" && rag.quick_replies && rag.quick_replies.length > 0 + ? rag.quick_replies + : undefined, }; return NextResponse.json( diff --git a/apps/web/app/page.tsx b/apps/web/app/page.tsx index 0478b23..63f1460 100644 --- a/apps/web/app/page.tsx +++ b/apps/web/app/page.tsx @@ -73,16 +73,30 @@ export default function ChatPage() { setShowMobileSidebar(false); }; - const handleCitationClick = (citation: Citation, index: number) => { + const handleCitationClick = (citation: Citation, index: number, allCitations: Citation[]) => { + // Found live 2026-08-07: this used to only set the index into whatever + // `citations` array was last loaded (i.e. the MOST RECENT answer's), so + // clicking [1] on an older message showed a LATER message's unrelated + // drug in the evidence panel (reported live: clicking Omeprazol's own + // citation showed Kanamycin). The clicked message's own citation list + // must replace the panel's state, not just the index into a stale one. + setCitations(allCitations); setActiveCitationIndex(index); setShowMobileEvidence(true); }; const handleCitationsLoaded = (newCitations: Citation[]) => { setCitations(newCitations); - if (newCitations.length > 0) { - setActiveCitationIndex(1); - } + // Deliberately NOT auto-activating citation 1 here (removed + // 2026-08-07): this used to fire the beam connector line + card + // highlight on every single answer, unprompted, and — since + // `CitationBeamOverlay` only recomputes its coordinates on window + // resize/scroll, not on the content reflow a just-arrived answer + // itself causes — the line frequently ended up pointing at stale + // positions, i.e. exactly the "dây trích dẫn dính lung tung" (messy + // citation wire) reported live. The beam/highlight now only appears + // when the user actually clicks a citation, at which point the + // coordinates are computed fresh. }; return ( diff --git a/apps/web/app/tra-cuu/page.tsx b/apps/web/app/tra-cuu/page.tsx index 4b39396..8f5d013 100644 --- a/apps/web/app/tra-cuu/page.tsx +++ b/apps/web/app/tra-cuu/page.tsx @@ -13,10 +13,14 @@ export default function TraCuuPage() { function handleCitationClick(citation: Citation) { if (citation.sourcePageRange && citation.sourcePageRange[0]) { - const page = citation.sourcePageRange[0]; - setActivePage(page); + // Display the printed page (what's on the paper page, matches the + // clinician's physical copy), but navigate the PDF viewer by the + // physical page — they diverge by 1-3 pages depending on front-matter + // offset, confirmed against the real PDF (physical_page is PyMuPDF's + // 0-indexed page; the #page= fragment is 1-indexed). + setActivePage(citation.sourcePageRange[0]); setActiveDrug(citation.drugName); - setPdfSrc(`/api/pdf#page=${page}`); + setPdfSrc(`/api/pdf#page=${citation.physicalPage + 1}`); } } diff --git a/coordination/WORK_SPLIT_2026-08-10.md b/coordination/WORK_SPLIT_2026-08-10.md new file mode 100644 index 0000000..dd22a12 --- /dev/null +++ b/coordination/WORK_SPLIT_2026-08-10.md @@ -0,0 +1,24 @@ +# Work split — 2026-08-10 + +## Claude: deployment owner + +- Stop changing `apps/ai-service/rag/**` and `apps/ai-service/tests/**` after + finishing or handing off the currently open Aspirin precaution fix. +- Own Dockerfiles, runtime environment/secrets wiring, Docker Compose app + services, web hosting, health checks, smoke test, rollback notes, and the + demo deployment. +- Deployment paths: `infra/**`, app Dockerfiles, and deployment-only config. + +## Codex: RAG core owner + +- Own multi-query, hybrid dense/lexical retrieval, RRF fusion, parent/sibling + expansion, context packing, retrieval/output guardrails, tracing, and eval. +- Core paths: `apps/ai-service/rag/**`, retrieval adapters and their tests. +- Do not edit deployment files or `packages/ui/**` while Claude is working. + +## Collision rule + +- Do not modify a file currently changed by the other owner. +- Before each commit, check `git status --short` and preserve all pre-existing + changes. +- Keep deployment and core changes in separate commits. diff --git a/docs/adr/0007-conversational-reasoning-rag.md b/docs/adr/0007-conversational-reasoning-rag.md index a9c24a7..7726a2a 100644 --- a/docs/adr/0007-conversational-reasoning-rag.md +++ b/docs/adr/0007-conversational-reasoning-rag.md @@ -1,9 +1,39 @@ # ADR 0007: Conversational reasoning RAG — state, bounded loop, and how it is measured -**Status:** accepted, implementation in progress (2026-08-05) +**Status:** superseded by ADR 0008 (2026-08-07). See the note below before +reading this as a description of anything currently running. **Supersedes:** nothing. Extends ADR 0005 (segment output contract) and ADR 0006 (quarantined block references) rather than replacing them. +> **2026-08-07 — why this was superseded, not deleted.** An independent +> 7-agent audit on 2026-08-06 found `bootstrap.py` never constructs any of +> `rag/conversation.py` / `rag/reasoning.py` / `rag/conversational.py` — the +> live agent (`rag/agent.py::RagAgent`, wired in since the F-03 rebuild on +> 2026-08-06) is a fixed one-shot pipeline (understand → route → retrieve +> once → generate → ≤2 same-claim entailment retries), not the PLAN/RETRIEVE/ +> ASSESS/REFINE/VERIFY loop or the `Focus`/`ConversationState`/TTL state +> design below. This was a real, deliberate pivot mid-implementation, not an +> abandoned-but-still-intended plan: `rag/agent.py`'s own module docstring +> says outright that `ConversationalLoopService` + `conversation.py` were +> replaced because "the LLM reads a plain turn history and resolves +> ['thuốc đó' / 'còn liều thì sao'] itself" — simpler than maintaining +> `Focus`/TTL/turn-budget state by hand, and proven live across many +> multi-turn conversations since. Section 6 below ("Refused: an LLM +> confidence score as the loop's uncertainty signal") is the clearest +> evidence this is a genuine architecture change, not a gap: the live system +> now uses exactly that — an LLM sufficiency/clarify judgment — as its +> ask-or-answer signal, the opposite of what this ADR chose. +> +> The three modules this ADR specified (1,314 lines) and their five dedicated +> test files (42 tests) were deleted on 2026-08-07 rather than left as dead +> code, once confirmed to have zero live importers anywhere +> (`bootstrap.py`/`main.py`/`agent.py`/`answer.py`/`routers/rag.py`). This +> document is kept, unedited below this notice, as the historical record of +> why that design was chosen and what it traded off — see ADR 0008 for what +> actually runs today, including what this ADR got right that ADR 0008 +> still owes (a real request-scoped time/call budget — F-08, still open; a +> durable, cross-worker conversation store — currently an in-process dict). + ## Context The service answers one question at a time. `POST /v1/rag/query` carries no diff --git a/docs/adr/0008-llm-understanding-one-shot-rag.md b/docs/adr/0008-llm-understanding-one-shot-rag.md new file mode 100644 index 0000000..1f82e33 --- /dev/null +++ b/docs/adr/0008-llm-understanding-one-shot-rag.md @@ -0,0 +1,153 @@ +# ADR 0008: LLM query understanding + one-shot grounded RAG (what is actually live) + +**Status:** accepted, live since 2026-08-06 (F-03), extended 2026-08-07 +**Supersedes:** ADR 0007 (conversational reasoning RAG — the `Focus`/ +`ConversationState`/TTL state design and the PLAN/RETRIEVE/ASSESS/REFINE/ +VERIFY bounded loop). ADR 0007's own `rag/conversation.py`/`rag/reasoning.py`/ +`rag/conversational.py` were deleted 2026-08-07 once confirmed unreachable +from `bootstrap.py` — see the notice at the top of ADR 0007 for the full +reasoning. +**Extends:** ADR 0006 (quarantined block references) — unchanged and still +binding: a chunk with `has_quarantined_content` still forces `VERIFY_PDF` +and is never generated over. + +## Context + +This ADR exists because `docs/architecture.md` and ADR 0007 described a +design that was never fully built, and the modules that partially +implemented it were never wired into `bootstrap.py`. A 2026-08-06 +independent 7-agent audit found this the hard way — it cost real time +establishing that `QdrantRetriever.search()` (dense vector search) and the +entire reasoning-loop module set were dead code, contradicting what the +docs claimed was live. The fix is not "finish building ADR 0007" — the +project deliberately moved to a simpler design that already works, proven +across many real multi-turn conversations (see `docs/progress-log.md`, +2026-08-05 through 2026-08-07 entries). This ADR documents that design so +the next reader doesn't have to re-discover it by audit. + +## Decision + +### 1. One LLM call understands the turn; no separate state object + +`rag/understanding.py::LlmQueryUnderstander.understand(turn, history)` reads +the raw current turn plus a **plain list of past turn strings** +(`"Người dùng: …"` / `"Trợ lý: …"`, kept by `RagAgent._history`, a +per-conversation-id in-process dict) and returns a `QueryFrame`: turn type, +resolved `drug_id`s (validated against a candidate set a deterministic +fuzzy/alias pass bounds *before* the model runs — F-04), section attribute, +population, weight, age, indication, route, and a `needs_clarify`/ +`clarify_reason`/`quick_replies` triple. + +There is no `Focus` struct, no TTL, no separate summariser. The model +re-reads the same history window (last `HISTORY_TURNS * 2` = 12 lines) every +turn and re-derives what's still relevant — cheaper to build and, so far, +more robust than hand-maintained state: it naturally handles "còn trẻ em thì +sao?" and short replies to its own clarify questions (population/route/etc. +— the latter only after a 2026-08-07 fix; see progress-log) without a +resolver state machine to keep in sync. + +**Known gap, inherited from ADR 0007 and still open:** this history is an +in-process dict — lost on restart, not shared across workers if the service +ever scales beyond one. ADR 0007's `PostgresConversationStore` was never +built either. + +### 2. Routing is a single dispatch, not a loop + +`RagAgent._route()` reads `frame.turn_type` and dispatches once: +`interaction` (2+ drugs) → gather each drug's evidence, combine, decide; +`drug_attribute`/`drug_overview`/`dosing_calc`/fallback → one drug, one +retrieval call; `smalltalk`/`out_of_scope` → canned reply, no retrieval; +`symptom_to_drug` with no drug named → an honest "not built yet" clarify. +There is no PLAN/REFINE step and no retrieval-round budget, because there is +only ever one retrieval call per turn. + +### 3. Retrieval is deterministic routing, not similarity ranking + +`RetrievalService.retrieve_framed(drug_id, section_key, query)`: +- `section_key` given (the dominant case, since `understand()` almost always + resolves it) → `find_by_section`, an **exact Qdrant payload filter** + (`drug_id` + `section_key`), returning the whole section as a scroll. + Score is a hardcoded 1.0 — this is a filter, not a ranked search, and nothing + here is "confidence" in the sense ADR 0007's retrieval-confidence gate meant. +- No section resolved → `find_by_drug` (whole monograph, book order), + trimmed to identity sections for a bare name or reranked (Cohere + cross-encoder over the ~29 sections of that one drug, not a corpus search) + for a free-form question. +- `QdrantRetriever.search()` — real dense vector similarity over the whole + corpus — exists and is unit-tested, but `RagAgent` never calls it. It is + reachable only through the legacy `RetrievalService.retrieve()` entry + point, itself only reachable when `ANSWER_PROVIDER=disabled` (no agent + configured at all — retrieval-only mode). `docs/architecture.md`'s + "Retrieval-confidence gate: below a similarity threshold, skip the LLM + call entirely" describes this legacy-only path, not the live one; that + section has been corrected to say so. +- Measured, and the reason this design was chosen over similarity ranking + for the live path: routing by exact `section_key` moved contraindication + hit@1 from 0.05 to 1.00 (`[[project-retrieval-quality-gap]]`, 2026-08-04). + A quarantined chunk anywhere in the retrieved set still forces the whole + result to `VERIFY_PDF` (`RetrievalService.decide`, a public wrapper added + 2026-08-07 so `RagAgent._interaction` applies the same policy to a + combined multi-drug evidence pool instead of hand-rolling it). + +### 4. Generation is one call, verified twice, with no confidence score + +`GroundedAnswerService.answer_from_result`: sufficiency-check (ask instead of +guessing when the evidence spans multiple populations/routes and the turn +hasn't disambiguated) → generate → `grounding.verify` (every number and +citation traces to the block it cites) → `_verify_entailment` (a second LLM +pass confirming each cited claim's *content*, not just its numbers, is +actually stated by that block; one same-claim retry on a lone reject, since +this call is measurably noisy — 2026-08-06 finding). A generation that fails +any check **abstains** — it does not fall back to a raw extractive quote +when a generator is configured (`[[feedback_no_extractive_fallback_when_llm_configured]]`). + +No `MAX_LLM_CALLS`/`MAX_WALL_CLOCK_MS` budget object exists. Each call is +bounded only by its own provider timeout. **This is ADR 0007's F-08 finding, +inherited unchanged and still open** — a real end-to-end request deadline +threaded through `RagAgent`'s sequence of up to 5 sequential Bedrock calls +(understand → sufficiency → generate → ≤2 entailment) is real remaining +work, not solved by this ADR. Measured live 2026-08-07: a single answerable +turn costs ~8-9s wall clock, ~75-80% of it the 4 sequential LLM calls +(understand ~2.6-3.3s dominates — an 80B model doing a classification task +that likely doesn't need one); a clarify chain compounds this linearly since +each round is a fresh request repeating the same call sequence from scratch. + +### 5. Context resolved across turns is folded into one self-contained string + +Added 2026-08-07, closing a P0 the 2026-08-06 audit named: `frame.population`/ +`weight_kg`/`age_text`/`route`/`indication` were extracted by `understand()` +but never reached `retrieve_framed`/`answer_from_result`, which took only +the bare current-turn text — so a reply like "Uống" three turns into a dose +conversation reached the sufficiency/generation LLM calls as literally just +"Uống", with no notion that population=adult was already established two +turns back. `RagAgent._synthesize_query` now folds every resolved field into +one string (`"Uống. Đối tượng: người lớn. Đường dùng: uống."`) before it +reaches retrieval's rerank signal and generation's `query` argument. No-op +for a fresh single-shot question that already states its own context. + +## Consequences + +**Accepted.** No confidence score, no retrieval-round budget, no PLAN/REFINE +step — the tradeoff ADR 0007 explicitly refused ("an LLM confidence score... +is not a defensible basis for asking or not asking a clinician a question") +is exactly what this design uses instead (an LLM sufficiency/clarify +judgment), because in practice it has been reliable enough and dramatically +simpler to build, extend (route/quick_replies were one schema field + one +prompt rule each, not a new state machine), and debug — every session this +month that touched the ADR 0007 modules found new bugs in the state-machine +edges (TTL boundaries, Focus inheritance correctness) rather than in the +domain logic itself. + +**Refused (again, restated from ADR 0007, still true):** an LLM confidence +score as a hard gate for retrieval — `RetrievalService.decide`'s +`VERIFY_PDF`/`ABSTAIN` decisions remain deterministic (quarantine flag, +missing provenance), never a model's self-reported certainty. + +**Still open, named rather than hidden:** +- No request-scoped time/call budget (F-08). +- Conversation history is in-process, not durable/shared (inherited from + ADR 0007, never built either way). +- No production-path adversarial regression suite beyond one live-verified + end-to-end case (F-10's remaining scope). +- `dosing_calc` (a real mg/kg calculator) and `symptom_to_drug` (reverse + indication lookup) remain honest "not ready" clarifies, not answers. diff --git a/docs/architecture.md b/docs/architecture.md index 4a24564..f621173 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -17,12 +17,12 @@ disclaimer. | **auth-service** (NestJS) | Signup/login, password hashing, JWT issuance/refresh | Postgres (users); no dependency on other services | | **user-service** (NestJS) | Profile data, preferences, account settings | Postgres (profiles), called by gateway | | **chat-service** (NestJS) | Chat session lifecycle, message history persistence | Postgres (chat_sessions, chat_messages); calls ai-service per user message, persists both turns | -| **ai-service** (Python/FastAPI) | RAG orchestration: embed query → vector search in Qdrant → build grounded prompt → call OpenAI → return answer + citations | Qdrant (vector search), OpenAI API; stateless, does not own chat history | -| **ingestion** (Python, offline batch) | One-time/periodic job: parse PDF → monographs → chunks → embeddings → upsert to Qdrant | Qdrant (write), OpenAI embeddings API; runs as CLI/CI/k8s Job, never in the live request path | +| **ai-service** (Python/FastAPI) | RAG orchestration: understand query (LLM) → route to deterministic section/drug retrieval in Qdrant → generate + verify (LLM) → return answer + citations | Qdrant (payload-filtered retrieval), AWS Bedrock (Cohere embed-v4 for query embedding where used, Qwen3 via the Converse API for understanding/generation/entailment, Cohere rerank); conversation history is an in-process dict per `RagAgent`, not yet durable — see ADR 0008 | +| **ingestion** (Python, offline batch) | One-time/periodic job: parse PDF → monographs → chunks → embeddings → upsert to Qdrant | Qdrant (write), AWS Bedrock (`cohere.embed-v4:0`); runs as CLI/CI/k8s Job, never in the live request path | | **web** (Next.js) | Chat UI, auth UI, citation/disclaimer rendering, session list | Calls api-gateway only | **Sync vs async**: the live chat path (web → gateway → chat-service → -ai-service → Qdrant + OpenAI → back) is synchronous request/response. +ai-service → Qdrant + AWS Bedrock → back) is synchronous request/response. Ingestion is fully decoupled, offline, batch — it populates Qdrant ahead of time and is never triggered by a chat request, since parsing the 37MB PDF and embedding thousands of chunks takes minutes. Internal protocol is REST/JSON @@ -105,10 +105,12 @@ methodology, cross-tool comparison, and validation numbers. header/footer-boilerplate leak into section text (98.4% of monographs affected) must be fixed upstream before this design runs against real data. -4. **Embedding + load**: OpenAI `text-embedding-3-small` in batches, upserted - into a versioned Qdrant collection (`drug_monographs_v1`) keyed by - `chunk_id` for idempotent re-runs; collection aliasing allows re-ingesting - with a changed chunking strategy without downtime. +4. **Embedding + load**: AWS Bedrock `cohere.embed-v4:0` in batches + (cached by `(model_id, input_kind, text_sha256)` so a reload needs no + repeat cloud calls), upserted into Qdrant collection `duocthu_v1` + (15,100 points, live) keyed by `uuid5(chunk_id)` for idempotent re-runs; a + `__manifest` sidecar records the corpus sha/model/dimensions + and `ai-service` refuses to start against a mismatched one (F-05). 5. **Batch job, not synchronous**: runs as a CLI command locally, and as a Kubernetes `Job`/`CronJob` in production — never inside the ai-service request path. @@ -119,8 +121,15 @@ methodology, cross-tool comparison, and validation numbers. context, never state a dosage/contraindication/interaction not present in it, always append a disclaimer, and say "not found in the formulary" rather than guess when retrieval is irrelevant. -- **Retrieval-confidence gate**: below a similarity threshold, skip the LLM - call entirely and return a canned "consult a professional" response. +- **Deterministic routing, not a similarity-confidence gate.** The live + path resolves drug + section by exact payload filter (`section_key` + routing moved contraindication hit@1 from 0.05 to 1.00 — similarity + ranking alone was not reliable enough to gate on). A quarantined table/ + formula in the retrieved evidence, or missing page provenance, forces + `VERIFY_PDF`/abstain deterministically — never an LLM-reported confidence + score. Dense vector similarity search exists (`QdrantRetriever.search()`) + but is reachable only in the legacy no-generator-configured mode, not the + live agent path. See ADR 0008. - **Citations from metadata, not LLM prose**: the `citations` list is built directly from retrieved-chunk metadata, independent of what the LLM says, so the frontend can always show verifiable sources. @@ -136,9 +145,9 @@ methodology, cross-tool comparison, and validation numbers. 1. **Ingestion pipeline + populated, queryable vector DB.** Done when a CLI run populates Qdrant and a test script retrieves the correct drug/section chunk for a sample query — no API, no LLM call yet. -2. **ai-service (FastAPI) wrapping RAG + OpenAI.** Done when a `curl` to - `/query` returns a grounded answer with a traceable citation and an - always-present disclaimer. +2. **ai-service (FastAPI) wrapping RAG + AWS Bedrock.** Done when a `curl` to + `/v1/rag/query` returns a grounded answer with a traceable citation and an + always-present disclaimer. **Done** — live since 2026-08-05, see ADR 0008. 3. **auth/user/chat services + api-gateway.** Done when register → login → chat message flows end-to-end through the gateway only, persisted in Postgres. diff --git a/docs/progress-log.md b/docs/progress-log.md index a9d6d50..dc9df1b 100644 --- a/docs/progress-log.md +++ b/docs/progress-log.md @@ -1,5 +1,1181 @@ # Progress Log +## 2026-08-10 (cont. 16) — Recovered from the machine-trouble cutoff: Bug 2 live-reverified, quarantine path conclusively exercised, retry rate remeasured + +Picked up exactly where cont. 15 left off. Docker Desktop was down (machine +trouble from last session), so Postgres/Qdrant containers and both app +servers were all stopped. Restarted everything: Docker, `docker-postgres-1`/ +`docker-qdrant-1` (same volumes, no migration needed — `rag_conversation_turn`/ +`rag_retrieval_trace` tables and the Qdrant `duocthu_v1` collection's 15,100 +points were confirmed intact, not rebuilt), `ai-service` (`:8079`, no +`--reload`, per house rule) and `web` (`:3000`). Full `pytest -q`: 196 +passed, 5 skipped, no regression from the crash/restart. + +**Bug 2 re-verified live in the actual browser** (the one item cont. 15 +explicitly flagged as unfinished). Drove the Kanamycin eye-drop-dose +question through Chrome by hand, answering the clarify chain (người lớn → +indication/renal → weight → indication again) until the model converged. +Got exactly the specific, honest reason the fix was supposed to produce: +**"Dược thư không nêu liều dùng đường nhỏ mắt của thuốc này"** — not the old +generic "chưa xác định đủ cơ sở, vui lòng thử lại" boilerplate. Fix +confirmed working after the restart. + +**Quarantined-table citation path conclusively exercised** — the one gap +named in cont. 13's 50-question audit ("did not conclusively exercise... one +attempt correctly hit generic out-of-scope instead, not quarantine +specifically"). Found a clean known-quarantined chunk via direct Qdrant +payload query (`has_quarantined_content: true`): "Thuốc tương tự hormon giải +phóng Gonadotropin — Dược lý và cơ chế tác dụng" (p.1372), a +`block_descriptor` chunk whose entire content is a table lifted to +quarantine (page image only, no extracted text). Asked its mechanism-of- +action question live: response correctly tagged **"⚠️ CẦN ĐỐI CHIẾU PDF +GỐC"**, body text "Nguồn có bảng hoặc công thức cần đối chiếu trực tiếp với +ảnh PDF; không tự động trích số liệu," and the citation panel showed the +matching warning card with a working "Mở trang PDF gốc để đối chiếu" deep +link to printed page 1372. No fabricated number, source crop shown as +designed — matches [[project_quarantined_block_contract]] exactly. + +**Noisy-entailment retry rate remeasured under light (non-bursty) traffic**, +per cont. 13's flag that the old ~4% figure (cont. 11) might have been +inflated by that session's own heavy test load. Ran 15 sequential clean +single-turn factual questions via the live API (fresh `conversation_id` +each, ~2.5s pacing between calls, one manual retry on any non-answerable +first attempt) — script at +`ingestion`-adjacent scratch path, results not committed (throwaway probe). +- **11/15 (73%) answerable on the first attempt**, no retry needed. +- **2/15 (13.3%) hit a genuine `abstain` on the first attempt** + ("Tương tác thuốc của Warfarin là gì?", "Liều dùng Azithromycin cho người + lớn là bao nhiêu?" — both `unsupported_claim`). Warfarin recovered fully to + `answerable` on one retry — classic noise-and-recover. Azithromycin's retry + downgraded to a `clarify` (asking for indication/route) instead of + repeating the unsupported claim — the safety net choosing an honest + clarify over a second bad answer, not a full recovery but not a silent + wrong answer either. Matches the already-known, already-deferred + `dosing_calc`/indication-dependent-dosing gap, not a new bug. +- 1/15 (Cefazolin cách dùng) correctly hit `verify_pdf` first try (that + section genuinely has quarantined content, independently confirmed via the + same Qdrant query above) — **but the identical question retried fresh + returned `clarify` instead of `verify_pdf` the second time.** Flagging as a + new, small, non-blocking finding: routing/understanding isn't fully + deterministic run-to-run on this query, not measured further this session. +- 2/15 legitimately needed `clarify` (Insulin storage depends on + vial-vs-pen/opened-state; this is a fair question to ask back, not a + defect). + +**Honest reading of the number**: true first-attempt-abstain rate measured +at 2/15 ≈ 13.3%, higher than cont. 11's ~4% theoretical estimate — but +n=15 is small, and at least one of the two abstains (Azithromycin) looks +like a legitimate content-ambiguity case (multiple indication-specific +doses) rather than pure entailment noise, so this isn't an apples-to-apples +comparison with the old number. Under genuinely light traffic, no case +required more than one manual retry to reach either a correct answer or an +honest clarify — nothing looped, nothing hung, nothing fabricated. Not +proof the noisy-retry math from cont. 11 is wrong, but also not a clean +confirmation of the old ~4% figure; worth a larger-n rerun before using +either number for an SLA claim. + +**Also fixed while running the probe**: hit the known +`UnicodeEncodeError` on Vietnamese console output (`cp1258` codec) the first +run — re-ran with `PYTHONIOENCODING=utf-8` per the standing env gotcha +([[reference_env_operational_gotchas]]); also hit Python's stdout buffering +silently swallowing output when redirected to a file under +`run_in_background` — fixed with `python -u` (unbuffered) run as a detached +shell background process instead. + +**Still open, not touched this session**: `api-gateway`/`auth-service`/ +`user-service`/`chat-service` remain empty scaffolds — no auth, no rate +limiting, no `conversationId` ownership check; this is still the largest +structural gap standing between this build and production. The other 3 +persistent abstains from cont. 13's audit (Aspirin+ulcer caution, +Aspirin+Warfarin interaction specifically, Vancomycin rapid-infusion +caution) are untouched. The two minor precision bugs from that audit +(English-population-ignored, self-referential route question) are +untouched. Real Postgres connection pooling (F-09's named remainder) is +untouched. + +## 2026-08-07 (cont. 15) — 2 more real bugs owner caught live driving the browser, both fixed; session cut short before final re-verify + +Right after cont. 14's fix, owner drove the actual chat themselves (not me) +and hit two more real, live bugs. Both root-caused and fixed same session, +committed together in `a723f62`. Machine trouble cut the session short +before the second fix could be independently re-verified live — **do that +first next session**, see `project_production_readiness_audit_2026_08_07.md` +memory for the exact re-check steps. + +**Bug 1 — citation panel shows the wrong drug's evidence.** Clicking +citation `[1]` on an OLDER answer (Omeprazol's own mechanism-of-action +citation) displayed a completely unrelated LATER drug (Kanamycin) in the +"Bằng Chứng Dược Thư" panel, with the beam-connector line pointing at it +too. Root cause: `page.tsx`'s `handleCitationClick(citation, index)` +received the correct per-message `citation` object from `ChatBubble` but +discarded it, only ever setting `activeCitationIndex` — the panel's +`citations` array itself stayed whatever the MOST RECENTLY LOADED answer's +list was (set once by `onCitationsLoaded`), never refreshed per click. Any +older message's marker index just indexed into that stale, unrelated array. +Fixed by threading the clicked message's own citation array through the +whole chain (`ChatBubble.tsx`'s `onCitationClick` now passes `allCitations` +too → `ChatPanel.tsx` passes it through → `page.tsx` calls +`setCitations(allCitations)` before setting the index). Live-verified. + +**Bug 2 — a good, specific abstain reason gets thrown away for generic +boilerplate.** Asked Kanamycin's eye-drop strength; got "Dược thư có nội +dung liên quan... nhưng hệ thống chưa xác định đủ cơ sở, vui lòng thử lại" +— unhelpful. Manually retrying revealed the model's real, correct judgment +was available the whole time: "Bằng chứng không nêu liều dùng đường nhỏ +mắt của thuốc này" — specific, honest, actually useful. Root cause: +`rag/prompt.py`'s answer contract (rule 5 + `ANSWER_SCHEMA`) allowed +`clarifying_question` to stay `null` even when `evidence_sufficient=false` +for the "source genuinely lacks this content" case (rule 7's mandate only +covered the narrower "user needs to specify more" case) — so whenever the +model happened to omit it, `answer.py::_generate` fell through its one +internal retry straight to the generic `reject_reason="evidence_insufficient"` +→ `REFUSALS` boilerplate, discarding the specific reasoning the model +actually had. Fixed: prompt rule 5 and the schema's `clarifying_question` +description now REQUIRE a short honest explanation whenever +`evidence_sufficient=false`, covering both the "ask the user for more" and +the "the book doesn't cover this" cases. **Not yet independently +live-reverified after the last restart** — session ended mid-check. + +Full suite 196/196 passing at commit time (no test changes needed for +either fix — Bug 1 is TS-only, Bug 2 is a prompt-text-only change, no +logic/schema-shape change). + +## 2026-08-07 (cont. 14) — Fixed the P0 clarify-loop bug from cont. 13's audit + +User approved fixing the top blocker from the 50-question audit: the +non-terminating multi-turn clarify loop. Two complementary fixes, both in +`rag/understanding.py` and `rag/agent.py` (F-11), plus one more real bug the +user separately reported live mid-session. + +**Third live repro found while working**: user typed a correction — "tôi có +hỏi liều uống đặt trực tràng đâu" (a negation: "I never asked about the +rectal dose") — after the bot answered the wrong route. The bot just +repeated the same wrong-route answer, ignoring the correction entirely. Same +root cause family as the other two: no dedicated handling for "the user is +refuting my last answer," so the model re-derives the same wrong reading. + +**Fix 1 — structural, `understanding.py`**: `LlmQueryUnderstander.understand()` +now takes an optional `prior_frame: QueryFrame`. On a clarify-continuation +turn, its known fields (drugs/population/weight/age/route/indication/ +attribute) are (a) stated explicitly in the prompt as a "THÔNG TIN ĐÃ XÁC +ĐỊNH" block instead of relying on the model to re-derive them from a raw +text transcript, and (b) merged back onto the new turn's parsed frame in +code (`_merge_with_prior_frame`) whenever this turn doesn't itself resolve a +*different* drug — so a dropped field is a non-event, not a re-ask. Guarded: +merge/known-block only fire when `prior_frame.needs_clarify` was true (a +resolved prior turn has nothing to continue) and never overrides a turn that +names its own different drug (that's a real topic change, must not inherit +stale slots — this is the direction the OMEPRAZOL bleed ran). Two new +`_SYSTEM` prompt rules cover what the merge can't: (1) explicit "this is a +brand new unrelated topic, don't carry the old drug over" guidance for the +bleed case, (2) explicit "the user is negating/correcting my last answer, +don't repeat it — ask what they actually meant" guidance for the +rectal-dose correction case. + +**Fix 2 — circuit breaker, `agent.py`**: `RagAgent` tracks a per-conversation +consecutive-clarify streak. After `MAX_CONSECUTIVE_CLARIFY = 4` clarify +decisions in a row, it force-abstains with an actionable message ("gõ lại +toàn bộ câu hỏi... hoặc bấm Tạo phiên tra cứu mới") instead of asking again. +Any non-clarify decision resets the streak. This is the backstop that +guarantees no user gets stuck forever regardless of how well Fix 1 works — +every other failure mode in this file already degrades to a bounded abstain; +this was the one path with no bound at all. + +**Tests**: 9 new (5 `test_understanding.py` — merge survives a dropped +field, merge skipped on a genuine drug change, merge skipped when prior was +already resolved, known-facts block present/absent in the actual prompt +sent; 4 `test_agent.py` — breaker fires at the threshold, streak resets +after the hard stop so the conversation isn't permanently locked, a resolved +turn in between resets the streak, `prior_frame` is correctly threaded from +the previous turn). All 6 existing fake-understander test doubles in +`test_agent.py` updated for the new `prior_frame` kwarg. Full suite +187 -> 196 passed, 5 skipped, no regressions. + +**Live-verified in the actual browser** (ai-service restarted, no `--reload` +per house rule): replayed both reproduced bugs from cont. 13 end to end. +"Bảo quản Insulin" -> "Chưa mở lọ" -> "Insulin người" -> "Regular": no longer +loops — asks 3 genuinely different narrowing questions (real progress, not a +repeat) then the circuit breaker cleanly hard-stops with the actionable +message. "Cơ chế tác dụng của Omeprazole" (answered) -> "Tôi bị đau đầu nên +uống thuốc gì?": no longer mislabeled OMEPRAZOL or asks for body weight — +correctly asks "Anh/chị muốn dùng thuốc gì cho đau đầu? Ví dụ: paracetamol, +ibuprofen..." with no stale drug attached; answering "Paracetamol" converges +immediately to a correct, grounded, cited answer. Did not re-run the third +(rectal-dose correction) case live this session — covered by the same prompt +rule mechanism just verified working for the other two, not independently +browser-replayed. + +**Also added**: `clarify_loop_exhausted` entry in `apps/web/app/api/chat/ +route.ts`'s `REFUSALS` map (fallback only — the backend supplies its own +Vietnamese `answer` text for this reason, same pattern as every other agent- +inline abstain since cont. 9's fix). + +**Still open from cont. 13's audit**, not touched this session: +api-gateway/auth-service/user-service/chat-service remain unbuilt scaffolds; +the elevated live noisy-retry rate hasn't been re-measured outside heavy +test conditions; the 4 persistent (non-loop) abstains from the 50-question +run are unchanged; the quarantined-table/`VERIFY_PDF` citation path still +hasn't been conclusively exercised. + +## 2026-08-07 (cont. 13) — Post-reboot production-readiness audit: 50 hand-typed live browser questions, verdict NOT READY + +Machine crashed/rebooted mid-session (all background processes killed, Docker +Desktop down). Recovered clean: Postgres/Qdrant containers restarted, Qdrant +collection intact (15,100 pts), migrations re-applied, ai-service (`:8079`, +no `--reload`, per house rule) and web (`:3000`) restarted. Full pytest suite +187 passed/5 skipped immediately after — no regression from the crash. + +**User then directed a full manual audit**: read the codebase, then type 50 +real questions by hand into the actual Chrome UI (not curl) and judge pass/ +fail on the rendered answer + citation card, explicitly forbidding any other +verification method. Did exactly that — every one of the 50 below was typed +into the live textbox, submitted, and judged from the rendered DOM. + +**Architecture finding (new, not previously logged this precisely)**: +`api-gateway`, `auth-service`, `user-service`, `chat-service` are ALL still +empty scaffolds (`package.json` + `README.md` only, confirmed via directory +listing). `apps/web`'s `route.ts` talks directly to `ai-service:8079` — this +is the entire real live path, not a dev shortcut (matches cont. 8's +finding). No auth, no gateway rate-limiting, no persisted-by-a-real-backend +chat ownership exists yet. + +**Bug found and fixed before the 50-question run**: user separately reported +"mẫu tra cứu nhanh đang bị gửi 2 lần" (quick-prompt sidebar buttons +double-sending) — reproduced immediately via a stray click. Root cause: +`ChatPanel.tsx`'s `useEffect(() => { if (initialQuery) handleSendMessage(...) }, [initialQuery])` +had no guard, and Next.js dev-mode React 18 Strict Mode double-invokes +effect setup — each quick-prompt click fired two live identical `/api/chat` +POSTs. Fixed with a `useRef` sentinel that records the +last-sent `initialQuery` value, persists across the Strict Mode replay, and +still sends once for a genuinely new query (`ChatPanel.tsx`). Verified live: +one click -> one user bubble -> one POST in the Next.js server log. + +**50-question live results** (drug resolution + citation page always +spot-checked against real pharmacology, not just "did it answer"): +- **~40/50 eventually correct** (right drug, right section, citation page + matches evidence text) — many only after 1-3 manual "Thử lại" clicks. +- **4 persistent abstains** (still wrong after 2-3 retries, not noise): + "Thận trọng Aspirin + loét dạ dày" (`evidence_insufficient` 3/3), + pediatric weight-based Azithromycin dosing (`dosing_calc` — matches the + already-known open F-10 gap), Aspirin+Warfarin interaction + (`unsupported_claim` 2/2), Vancomycin rapid-infusion caution + (`unsupported_claim` 2/2). +- **A systemic multi-turn bug, independently reproduced 3 times in 3 + unrelated threads**: mid-clarify-chain, the understanding LLM loses + already-established context and either (a) re-asks the exact same + clarify question forever (Insulin storage: 5 real answered turns, never + converged), (b) forgets an already-stated population and re-asks it + (Azithromycin: "bé nặng 20 cân" established, then asked "trẻ em hay + người lớn?" again), or (c) drags in a stale unrelated drug from earlier + history into a brand-new topic (headache/tension question suddenly + labeled OMEPRAZOL, asking for the user's body weight for a + recommendation-seeking headache question). This is the single biggest + production blocker found this session — a real multi-turn user + conversation has a good chance of getting stuck in a non-terminating + clarify loop with no escape except starting a new session. +- **Two minor precision bugs**: an English-language query + ("...for adults") had its explicit population ignored, re-asked for a + child's weight; a self-referential question ("Cefotaxime dùng đường + nào?" — asking what routes exist) was misread as "which route do you + want," asking the user to pick one instead of just listing them. +- **Safety nets held up well** in every adversarial case: empty input + blocked client-side, gibberish/prompt-injection/English/very-long-repeat + input never hallucinated, non-human ("thuốc cho chó") and out-of-corpus + (Part 1/3 topics, BSA table) correctly abstained honestly, recommendation- + seeking ("tôi đau đầu nên uống gì") correctly did NOT hit the + recommendation-refusal gate (per [[feedback_no_recommendation_gate]]). +- **Noisy-entailment retry rate looked meaningfully higher than the + documented ~4% estimate** from cont. 11 — a large fraction of the 50 + needed at least one manual retry to get a real answer. Not conclusively + separated from this session's own heavy sequential test traffic; flagged + as worth re-measuring under light/normal traffic before trusting the old + 4% number for a capacity/SLA decision. +- Did not conclusively exercise the quarantined-table/`VERIFY_PDF` path — + one attempt (corticoid dose-equivalence table) correctly hit generic + out-of-scope instead, not quarantine specifically; needs a targeted + follow-up with a known quarantined chunk id. + +**Verdict**: chatbot is **NOT ready for production**. The RAG/grounding/ +citation core is genuinely strong (correct drug+section+page on the large +majority of single-turn questions, real safety gates holding under +adversarial input) but three things block a ship decision: (1) the +non-terminating multi-turn clarify loop — a real, frequent, user-facing +dead end, not an edge case; (2) `api-gateway`/`auth-service`/`chat-service` +are unbuilt, so there is no auth, no rate limiting, and conversation +history lives only in Postgres keyed by a client-supplied `conversationId` +with no ownership check; (3) the elevated live retry rate needs +re-measurement outside of heavy test conditions before any latency/cost SLA +is claimed. + +## 2026-08-07 (cont. 12) — REAL root cause of the repeated live failures found: O(history × aliases) candidate resolution, not Bedrock at all + +User, rightly frustrated that every fix so far was verified via curl/API calls +instead of the actual browser ("mở chrome lên gõ tay chat xem thế nào" — go +open Chrome, type by hand, see for yourself), directed hands-on browser +testing. That reproduced the failure directly: typed a real question by hand, +watched it load 30+ seconds, watched it fail with "Dịch vụ đang gặp sự cố +tạm thời" — the exact live symptom, not a hypothesis. + +**Diagnosis, with hard numbers, not guessing**: added timing instrumentation +to `RagAgent.handle()` (`t0..t4` around history/understand/route/remember). +First real capture: `understand=51.44s` — the understanding call was taking +nearly a minute, failing on `RequestBudgetExhausted` before ever reaching +Bedrock. Traced into `understanding.py::_candidate_ids`, which calls +`CatalogDrugResolver.resolve()` + `.suggest()` once per line of +`(turn, *history)` — up to 13 lines per turn. Direct isolated timing: +`resolve()` ≈0.65-0.7s, `suggest()` ≈0.94-0.97s per call, over the real +10,164-alias catalog (regex per alias in `resolve`, `SequenceMatcher` per +alias in `suggest` — both O(aliases)). **≈1.6-1.7s of pure CPU per history +line, called fresh on every single turn — including lines already resolved +in every prior turn of the same conversation.** A real multi-turn +conversation's accumulated history alone was enough to blow the 20s F-08 +budget before the first LLM call ever ran — this had nothing to do with +Bedrock, throttling, or the earlier `adaptive`-mode regression; those were +real but secondary. This is why the failure was reproducible and worsening +turn-over-turn in an actual chat session, not a flaky one-off a single curl +call would ever catch. + +**Fix**: `functools.lru_cache(maxsize=4096)` on `CatalogDrugResolver.resolve` +and `.suggest` (`rag/routing.py`) — both are pure functions of their +arguments (fixed `self._aliases`/`self._catalog` set once at construction, +single read-only caller). Verified in isolation: first pass over 3 lines +5.3s cold, identical second pass **0.0s** (full cache hit) — turns all but +the newest line into a dict lookup on every subsequent turn. Full suite 187 +passed after. **Live-verified in the actual browser** (not curl): fresh +session, "Chống chỉ định của Aspirin là gì?" answered correctly in <10s; +immediate follow-up "Liều dùng người lớn thì sao?" (real multi-turn, +history now populated) completed in ~15s with `resolved_drug_id` correctly +carried over from the first turn — no more `understanding_provider_unavailable`. +(That specific follow-up then hit `unsupported_claim` — the separate, +already-known noisy-entailment case from cont. 11, not this bug; reported +via the new granular reason with an honest message.) + +**Lesson, stated for the next session**: this was invisible to every +curl-based check this session ran, including dozens of them, because a +single stateless curl call never accumulates the history that made the cost +compound. Only driving the actual multi-turn chat surfaced it. The +standing house rule to drive the real chat, not just the API, is not +optional politeness — it is what found the actual bug after several +API-level "verified" claims that were all individually true but collectively +missed the real, user-facing failure. + +## 2026-08-07 (cont. 11) — Noisy-check retries, granular error codes, adaptive-mode regression found and reverted same session + +Follow-up to cont. 10 at the user's direct request: implement the 2 named +retry improvements, then a serious live incident hit mid-work. + +**Retry widening (`rag/answer.py`)**: `_verify_entailment` widened from 2 to +3 attempts (accept on any accept, discard only if all 3 reject) — math +check: single-call noise ~33% (2026-08-06 probe) makes 2-attempt discard +rate ≈ q²≈11%, matching the observed live abstain rate almost exactly; +3-attempt drops it to ≈q³≈4%. New `_attempt_generation`/`_RawAttempt` split +lets `_generate` retry once on a lone `evidence_sufficient=false` with no +`clarifying_question` — live-verified: the acid-ascorbic renal-threshold +case that abstained now returns the correct answer 3/3 on fresh retry. +**Trade-off stated in code, not hidden**: both changes let a genuinely bad +claim survive on 1-of-N noisy accepts instead of 1-of-2 — accepted since +the probed noise is symmetric, not because residual risk is zero. 5 new +tests (`test_grounded_generation.py`) lock in both the recovery and the +still-discards/still-abstains cases. Full suite 187 passed after. + +**Regression found and reverted same session**: while implementing the +above, switched Bedrock retry config from `mode: "standard"` to +`"adaptive"` (in `adapters/embedding.py`, `adapters/bedrock_converse.py`) — +intended to fix throttling, but adaptive mode's client-side rate limiter +remembers "throttled" ACROSS requests and paces down even healthy, +unrelated calls after a burst. This session's own heavy adversarial test +traffic (50-question harness + repeated manual retries) tripped it, and a +normal single answerable turn went from ~9s baseline to a measured 1-5 +minutes for the user, live, mid-session. Reverted to `mode: "standard"` +same session; 3 fresh timing checks after revert: 9.1s / 10s / 10.9s — back +to baseline. Lesson: `mode: "adaptive"`'s cross-request memory is exactly +wrong for a service that gets bursty *test* traffic sharing the same +client/quota as real traffic — `"standard"`'s per-request-independent +backoff has no such failure mode. `max_attempts: 3→4` kept either way. + +**Granular error codes (user's direct request, after this incident)**: +every abstain reason had already collapsed into `reason="generation_unavailable"` +by the time it reached the API/trace — a real provider outage was +indistinguishable from ordinary entailment noise without reading +`/metrics` by hand. `_GenOutcome` gained `reject_reason`, threaded through +every rejection branch of `_generate`, so `answer_from_result` now returns +the SPECIFIC reason (`provider_unavailable`, `malformed_output`, +`evidence_insufficient`, `ungrounded_number`, `invalid_citation`, +`uncited_claim`, `unsupported_claim`, `request_budget_exhausted`) instead +of the generic catch-all. Live-verified: the Risperidon contraindications +case (still genuinely abstaining after the retry fix — a real residual +case, not eliminated, now at least diagnosable) returns +`reason="evidence_insufficient"` instead of the old opaque +`"generation_unavailable"`. Also fixed `rag/understanding.py`'s +`except AnswerGenerationUnavailable` block, which silently swallowed the +real exception with ZERO logging anywhere — now logs +`type(exc).__name__: exc` and sets a new `QueryFrame.system_error` field so +`RagAgent._route` surfaces `understanding_provider_unavailable`/ +`understanding_malformed_output` instead of the same generic +`"needs_more_info"` a real clarifying question gets (these were previously +indistinguishable from the outside). `apps/web/app/api/chat/route.ts`'s +`REFUSALS` map extended with all 8 newly-surfaced `answer.py` codes — this +is the SAME class of bug fixed in cont. 9, reintroduced by this round's own +propagation change, caught and closed same session rather than left for a +future live report. + +**Also found and fixed**: `ChatPanel.tsx`'s "Thử lại" (retry) button +resent the ASSISTANT bubble's own text as the next query instead of the +original user question — live-confirmed via a trace row where the query +text WAS literally "Dịch vụ đang gặp sự cố tạm thời...". Fixed to walk back +to the nearest preceding `role: "user"` message. + +**Open, named rather than hidden**: the Risperidon case's root cause +(why THIS specific short contraindications passage draws a consistent +`evidence_insufficient` verdict across many attempts, not just noise) is +unsolved — worth a dedicated prompt-tuning look, not chased further this +round. User separately proposed a verbatim/no-LLM-generation fast path for +simple single-section lookups to cut the ~8-9s baseline under 7s — real +idea (the extractive mode already exists for `ANSWER_PROVIDER=disabled`, +this would be a per-request criterion instead) — not yet started, awaiting +go-ahead given its product-level UX impact. + +## 2026-08-07 (cont. 10) — Bedrock retry/backoff + full reason-code contract audit + +Follow-up to cont. 9: user agreed to skip `dosing_calc` (out of scope) and do +the other two named items. + +**Retry/backoff.** All 3 live boto3 Bedrock client sites +(`adapters/embedding.py`'s query embedder, `adapters/bedrock_converse.py`'s +answer generator AND reranker) were already retrying transient errors, but +only with `mode: "standard"` (reactive backoff-after-failure) and +`max_attempts: 3` — measurably not enough during today's throttling burst +(the two consecutive "Dịch vụ đang gặp sự cố tạm thời" turns from cont. 9). +Switched all 3 to `mode: "adaptive"` (client-side rate limiting that backs +off proactively once throttling is detected, botocore-native, no custom +retry code) with `max_attempts: 4`. Also gave `adapters/bedrock_claude.py`'s +Anthropic-SDK client `max_retries=4` (was the SDK default of 2) for +consistency, even though this provider isn't the one actually configured +(`ANSWER_PROVIDER=bedrock-converse` per `.env`) — kept in sync in case it's +ever selected. Full `pytest -q` suite (184 passed, 5 skipped) re-run clean +after the change; ai-service restarted and re-verified live against the +same Clonazepam query (still `answerable`/`grounded_evidence_available`, +2 citations). + +**Contract audit.** Enumerated every `reason=`/`EvidenceDecision.ABSTAIN` +site across `rag/routing.py`, `rag/service.py`, `rag/answer.py`, and +`rag/agent.py`'s own inline `AgentReply` construction — the full universe +of values `routers/rag.py` can put in the API response's `reason` field. +Cross-checked each against whether it can reach the frontend with +`answer: null` (the only case `REFUSALS` needs to cover — every agent.py +inline abstain path already supplies its own real answer text, bypassing +the map entirely) and confirmed all are now mapped after cont. 9's fix. +Also confirmed `chat-service` (the NestJS hop the documented production +topology routes through) has no source files yet — it's unbuilt scaffold — +so `apps/web`'s `route.ts` talking directly to `ai-service` on :8079 (per +`.env`'s `API_GATEWAY_URL`/`AI_SERVICE_URL`) really is the entire live +path today, not a dev-only shortcut with a separate prod code path that +could hide the same bug class. No second copy of this mapping exists +anywhere else to audit. + +## 2026-08-07 (cont. 9) — Live bug report investigated and fixed: misleading abstain message + +User reported "lỗi nghiêm trọng" (serious error) seen live in the chat UI. +Server logs showed zero exceptions/tracebacks and all HTTP 200s, so this was +not a crash — required reproducing the actual browser session to find it. + +**Root cause**: `apps/web/app/api/chat/route.ts`'s `REFUSALS` map only +covered 7 of the ~16 abstain `reason` codes the backend can actually emit +(enumerated by grepping every `EvidenceDecision.ABSTAIN`/`reason=` site +across `rag/routing.py`, `rag/service.py`, `rag/answer.py`). Any unmapped +reason silently fell through to a generic string that claims "Hệ thống +không tìm thấy căn cứ trong Dược thư" (system found no grounds in the +formulary) — **false** for the case that actually happened: asking +"dược động học của Clonazepam" (a real monograph section) hit +`reason="generation_unavailable"` (`rag/answer.py:208`) — retrieval +succeeded, generation/entailment failed its own safety check (this call is +documented-noisy, see ADR 0008) and correctly abstained — but the frontend +told the clinician the drug had no data at all. Confirmed by replaying the +exact same query directly against `/v1/rag/query`: it returned a full, +correctly-grounded, cited pharmacokinetics answer on retry. A doctor +reading "not found in the formulary" for a drug that IS in the formulary is +a real safety-adjacent UX bug, not a cosmetic one. + +**Fix**: added entries for all 9 previously-unmapped reason codes +(`generation_unavailable`, `query_intent_unknown`, +`drug_resolution_invalid_state`, `missing_query_or_drug`, +`missing_indication`, `no_indication_match`, `parent_hydration_failed`, +`missing_provenance`, `missing_printed_page_provenance`), each with an +accurate message — `generation_unavailable` explicitly says the formulary +DOES have related content and suggests retrying, instead of denying the +data exists. Narrowed `GENERIC_REFUSAL` itself from the false "no grounds +found" claim to a neutral "cannot process this right now, please retry", +since it should now only ever fire for a genuinely unclassified reason. +Verified live: hit `/api/chat` directly post-fix with the same Clonazepam +query (Next.js dev server hot-reloaded the route with no restart needed) +and got the correct grounded answer end-to-end through the real frontend +API route, not just the backend. + +**Also investigated, not a bug**: the same browser tab showed two earlier +turns ("Chống chỉ định & Thận trọng khi dùng Amoxicillin") failing with +"Dịch vụ đang gặp sự cố tạm thời" — `understanding.py`'s F-10 fail-closed +path for a real `understand()` LLM-call failure. Retried the identical +query 3x directly against the backend just now: all 3 succeeded with a +normal, correct clarify ("người lớn hay trẻ em?"). This looks like a +transient Bedrock hiccup/throttle from the session's own rapid testing +traffic, not a persistent fault — fail-closed behaved exactly as designed +(a graceful clarify message, not a crash), so no code change made here. + +## 2026-08-07 (cont. 8) — Phase 5/5 (final): faster-model A/B — negative result, current model kept + +Ran a live A/B (real Bedrock calls, real catalog/resolver) comparing the +current understanding model (`qwen.qwen3-next-80b-a3b`) against the fastest +plausible candidate already IAM-permitted (`qwen.qwen3-32b-v1:0`) on a +6-case battery covering today's actual hard cases: simple dose resolution, +a fake-drug safety check, 2-drug interaction, symptom_to_drug, weight +extraction, and the exact multi-turn route-resolution case fixed earlier +today. + +**Speed**: confirmed, qwen3-32b is genuinely faster — roughly 2.3-2.9s per +call vs 2.7-4.5s for the current model on most cases (~30-40% faster). + +**Two real regressions found, one safety-critical — recommendation: do NOT +wire it in.** +1. **Safety.** Asked about the fake drug "aspirinol" (this project's + standing regression case for F-04's catalog-bounding), the current model + correctly leaves it as `unknown_drugs=('aspirinol',)`. The 32B candidate + silently resolved it to the real `acid_acetylsalicylic_aspirin` with no + `unknown_drugs` entry at all — and because "aspirinol" fuzzy-matches + "aspirin" closely enough to appear in the turn's deterministic candidate + set, this substitution is NOT caught by F-04's own candidate-bound check + (`_resolve_id` only rejects an id that's outside the shown candidates; + this one is inside it). A live user asking about a genuinely nonexistent + drug would silently get an answer about aspirin instead, with no + indication their drug name didn't match anything. +2. **Instruction-following.** On the exact "Uống" multi-turn case fixed + earlier this session (route resolution after a short reply to the + model's own prior clarify question), the 32B candidate correctly + extracted `route=uong`/`population=nguoi_lon` into the frame fields — + the schema/field-level fix from earlier holds regardless of model — but + still set `needs_clarify=True` and re-asked a version of the original + question, undoing the point of today's earlier fix. The 80B model + correctly proceeded (`needs_clarify=False`). + +This matches a pattern already in memory from 2026-08-05 +(`[[project_llm_cloud_live]]`): DeepSeek V3.2 silently ignored the +clarify-don't-dump instruction on a different task, which is why Qwen3-80B +was chosen in the first place. Smaller models in the same family trading +away exactly this kind of careful instruction-following for speed is +consistent with that prior finding, not a one-off fluke. + +**No code changed** — per the plan's own stated criterion ("only wire it in +if the smaller model matches quality on the battery; otherwise document the +negative result"), this is a complete, valid Phase 5 outcome. `gpt-oss-20b` +was not tested — the qwen3-32b result already gives a clear, evidenced +negative for the "smaller Bedrock model for understanding" approach in +general, and further model exploration should wait for a specific reason to +revisit it rather than open-ended search. + +--- + +**All 5 phases of the owner-approved plan +(`~/.claude/plans/pure-wobbling-llama.md`) are now done or resolved**: +symptom_to_drug (built + live-verified), F-10 (built + found/fixed a real +unhandled-500 bug), F-08 (built + live-verified), durable conversation +history (built + verified across a real process restart), faster +understanding model (evaluated, negative result documented, current model +kept). `apps/ai-service`: 184 passed. + +## 2026-08-07 (cont. 7) — Phase 4/5: durable Postgres conversation history, verified across a real restart + +**Built**: `adapters/postgres.py::PostgresConversationStore` — same +established pattern as `PostgresTraceRepository` (`connect_timeout=5`, one +connection per call, no pooling — F-09's accepted tradeoff). Append-only +`rag_conversation_turn` table (`migrations/002_rag_conversation_turn.sql`); +`id bigserial` insertion order is the "oldest -> newest" ordering the +understanding prompt already expects, no separate turn-index column needed. +`recent(conversation_id, limit)` windows at READ time (`ORDER BY id DESC +LIMIT`), so — unlike the in-process dict it replaces — writes never need to +delete old rows; old history just sits unused past the window (same +unbounded-growth tradeoff the trace table already has, not a new gap). + +`RagAgent` gained an optional injected `store: ConversationStore | None` +(a tiny local Protocol — not a resurrection of the deleted `conversation.py`'s +`ConversationStore`, which was tied to the removed Focus/TTL design). +`None` (the default) keeps every existing behavior byte-for-byte unchanged. +When configured, `_get_history`/`_remember` read/write through the store +instead of the dict, and fail OPEN on any store error — same F-09 fail-open +convention as the trace writer, applied by direct analogy rather than a new +exception type: read failure → empty history this turn (fresh +understanding, not a 500); write failure → this turn's memory is silently +lost, the already-computed response still returns. Wired into +`bootstrap.py` next to `PostgresTraceRepository`. `migrate.py` now applies +both migrations. + +**Live-verified the actual capability being added, not just the plumbing**: +sent turn 1 ("Liều paracetamol hạ sốt là bao nhiêu?", `conversation_id` +set) to the real running server → clarify as expected. **Killed and +restarted the whole ai-service process** (a fresh Python process, empty +in-process dict — under the old design this conversation's memory would +be gone). Sent turn 2 ("Người lớn", no drug name at all) with the same +`conversation_id` → response's `resolved_drug_id` came back +`paracetamol_acetaminophen`, which is only possible if the understanding +call received turn 1's history from Postgres, since nothing in-process +survived the restart. This is the one live check that actually proves the +feature, as opposed to proving the code merely doesn't crash. + +`apps/ai-service`: **184 passed** (was 181; +3 unit tests with a fake +store covering round-trip/read-failure/write-failure). Also added +`test_real_postgres_conversation_store_round_trip` to +`test_live_datastores.py` (RUN_INTEGRATION=1-gated, matching the existing +pattern) — run against the real dev Postgres, passed, covers windowing at +the read boundary and an empty read for a never-seen `conversation_id`. + +**Known limitation, named not hidden**: no retention/cleanup job — the +table grows forever, same as `rag_retrieval_trace` already does. Not +addressed here; a reasonable follow-up if either table's growth becomes an +operational concern. + +## 2026-08-07 (cont. 6) — Phase 3/5: F-08 request-scoped budget built and live-verified + +**Built**: new `rag/budget.py` — `RequestBudget` (deadline + call-count, both +must hold) and `RequestBudgetExhausted` (subclasses `AnswerGenerationUnavailable` +deliberately, so every existing fail-open/fail-closed handler in the +codebase catches it with zero changes — budget exhaustion IS "the provider +is unavailable to us right now" from each call site's perspective). Not a +resurrection of the deleted `reasoning.py`'s heavier `TurnBudget` — that was +tied to the retrieval-refinement loop this system no longer has; this is +just a counter + a deadline, checked once per call. + +`RagAgent.handle()` constructs one `RequestBudget` per turn (defaults: +20s wall clock, 8 calls — sized with headroom above the measured normal +case of 4-5 calls / ~8-9s, so ordinary traffic never trips it) and threads +it through every LLM call site: `understanding.understand()`, +`answer.answer_from_result()` → `_check_sufficiency`/`_generate`/ +`_verify_entailment`/`_run_entailment_check`. Each calls `budget.require()` +immediately before its actual provider call — exhaustion means the real +network call never happens, not that it happens and then gets discarded. +Config: `Settings.max_wall_clock_ms`/`max_llm_calls_per_turn`, wired into +`bootstrap.py`'s `RagAgent` construction. + +One deliberate asymmetry, matching each site's existing failure-direction: +`_check_sufficiency` fails OPEN on budget exhaustion (skips the clarify +heuristic, proceeds to generate — it's a UX heuristic, not a safety gate); +every other site fails CLOSED (abstain/reject) — this was already true for +provider outages before F-08, budget exhaustion now follows the identical +rule at each site rather than introducing a third behavior. + +**Live-verified two ways**: normal query with the default budget answers +unchanged (~7.4s, same as before F-08). A `max_llm_calls_per_turn=1` agent +against the real Bedrock/Qdrant stack correctly aborts after the one +understand call, cleanly abstains (`generation_unavailable`, no crash, +no fabricated answer) instead of proceeding — proving the mechanism holds +end to end, not just in unit tests. Note on what this does and doesn't +prove: the latency saving in this specific case was modest (~6.2s vs +~7.4s) because `understand()` alone already dominates a normal turn's cost +— the budget's real value is bounding the pathological case (one call +stuck retrying for minutes against `read_timeout=60s` × up to 3 attempts), +which was not separately fault-injected live this session; that would need +a deliberately broken/slow fake provider, a reasonable next step if this +needs stronger evidence. + +`apps/ai-service`: **181 passed** (was 173; +8: 6 direct `RequestBudget` +unit tests, 2 end-to-end `RagAgent` tests proving a spent budget blocks the +generator from ever being called, with a control test proving the same +setup succeeds normally under the default budget). + +## 2026-08-07 (cont. 5) — Phase 2/5: F-10 adversarial battery — found and fixed a real unhandled-500 bug + +**Real bug found, not just tests added.** `rag/understanding.py::LlmQueryUnderstander.understand()` +was the ONE LLM call site in the whole product with no error handling +around it — every other call (`answer.py`'s sufficiency/generate/entailment) +catches `AnswerGenerationUnavailable` and fails closed, but `understand()`'s +`self._llm.generate(...)` had no try/except, and `routers/rag.py` only wraps +the trace-save call, not `agent.handle()` itself. A Bedrock outage during +understanding — the FIRST call of every single turn — would have propagated +into an unhandled 500 instead of a graceful abstain. Found by asking "what +does F-10's provider-outage-mid-conversation category actually cover today" +and checking each of the 4 call sites by hand, not by running anything. +Fixed: wrapped, fails closed to the same `needs_clarify` shape the JSON- +parse-failure path already uses, with an honest "dịch vụ đang gặp sự cố" +message instead of "tôi chưa hiểu câu hỏi" (the failure is the service's, +not a misunderstanding of the user's phrasing). + +**Also pinned, not previously tested**: `_check_sufficiency`'s outage +behavior is a deliberate fail-OPEN (skip the clarify heuristic, proceed to +generate — grounding/entailment remain the real safety net), unlike every +other failure mode in the service which fails closed to abstain. This was +already the code's behavior; now there's a regression test locking it in +as intentional rather than an accident nobody would notice changing. + +**New coverage**: `conversation_id` presence/absence reaches the same +decision on a fresh turn (by construction — both see empty history — now a +regression-guarded fact, not just an inference from reading the code); 2 +more fake-drug-near-alias shapes beyond the existing `aspirinol` case +(brand-like suffix on a real name, a name blending two real drugs) both +confirming F-04's candidate-bound rejects even a real catalog id with no +turn-specific support. Prompt-injection resistance and the entailment-judge +noise case were **not** newly tested — the former only really tests +anything with a fake LLM if the "compromise" changes the OUTPUT shape +(covered by the near-alias/catalog-bound tests above, which are exactly +that); genuine adversarial prompt resistance needs the real model, and +today's many live queries already incidentally exercised it without +incident. The entailment-judge noise case (warfarin/aspirin, 2026-08-06) was +not specifically re-run live this session — time-scoped out, not forgotten. + +`apps/ai-service`: **173 passed** (was 168). Server restarted, confirmed +normal operation unaffected by the fix. + +## 2026-08-07 (cont. 4) — Phase 1/5: symptom_to_drug reverse lookup built and live-verified + +Owner approved a 5-phase plan (`~/.claude/plans/pure-wobbling-llama.md`) for +the remaining backlog: symptom_to_drug, F-10 adversarial tests, F-08 request +budget, durable conversation history, faster understanding model. Phase 1 done. + +**Built**: `QdrantRetriever.find_by_indication` (keyword phrase match on +`chi_dinh`-section prose chunks, deterministic) + `search_indication` (dense +vector fallback restricted to `chi_dinh`, tried only when keyword finds +nothing — the one place in the live path dense search is actually used, per +ADR 0008). `RetrievalService.retrieve_by_indication` orchestrates the two. +`RagAgent._symptom_to_drug` wires this into the `symptom_to_drug` turn type +(previously an honest "not ready" clarify), reusing +`GroundedAnswerService.answer_from_result` with a new `list_mode` flag so +citations/grounding/entailment apply unchanged. + +**Two real bugs found and fixed by driving it live**, not just unit tests: +1. Without `list_mode`, the generation prompt picked ONE drug out of 8 real + symptom matches and silently dropped the rest — `rag/prompt.py::build_request` + gained a `list_mode=True` branch instructing the model to enumerate every + matching drug (never a ranking — `[[feedback_no_recommendation_gate]]`), + and `answer_from_result`/`_generate` thread it through; the sufficiency + clarify (right for a single dose question) is skipped in this mode since + it doesn't fit a reverse lookup. Verified: "sốt" now correctly cites both + paracetamol and artesunat, not just one. +2. The first keyword-matching design (token-SUBSET: every word present + *somewhere*, any order) let a long nonsense query built from common + filler words ("bệnh chưa từng ghi nhận trong sách…") false-positive + against real `chi_dinh` text, reaching a wasted generation call before + entailment correctly rejected it. Switched to a word-boundary-anchored + CONTIGUOUS phrase match — precise by construction, dense search remains + the deliberate fallback for genuine paraphrases. + +**Known remaining imperfection, not chased further today**: the dense +fallback's `minimum_score` gate (reused from `EvidencePolicy`, 0.12) doesn't +reject a nonsense query's weak matches before generation — Cohere embed-v4 +similarity for unrelated Vietnamese medical text apparently sits above 0.12 +often enough that the gate rarely fires. The **safety outcome is still +correct** (grounding/entailment cleanly abstains, no fabrication, verified +live) — this is a wasted-generation-call efficiency cost, not a correctness +gap, and tuning the exact right threshold is a separate exercise from +today's scope. + +`apps/ai-service`: **168 passed** (was 153 before this phase). Live-verified +against the real running server (restarted after each code change): "sốt" +→ real 2-drug answer with citations; a nonsense phrase → clean abstain. + +## 2026-08-07 (cont. 3) — Item 3 of yesterday's Top 3 closed: dead reasoning-loop deleted, docs reconciled with what's live + +Owner said to go ahead and fix the last of yesterday's "Top 3 picked for +next session" items: reconcile `architecture.md`/ADR 0007 with what's +actually live. + +**Investigated before touching anything.** Confirmed by grep, not +assumption: `rag/reasoning.py`, `rag/conversation.py`, `rag/conversational.py` +(1,314 lines) have zero live importers — not in `bootstrap.py`, `main.py`, +`agent.py`, `answer.py`, or `routers/rag.py`. Their only consumers were their +own 5 dedicated test files (42 tests). `rag/ports.py` never actually gained +the `ConversationStore`/`Summariser`/`Planner`/`SufficiencyAssessor` +protocols ADR 0007 planned for it, and `adapters/postgres.py` never gained +`PostgresConversationStore` either — the whole design was implemented as +free-standing modules, then never wired in, confirming the audit's finding +that it's genuinely dead, not "integration pending." + +**Decision: delete + document reality, not wire the old design in.** The +live `RagAgent` (LLM-driven one-shot pipeline, plain-history multi-turn) has +been proven working across many real multi-turn conversations today and +yesterday — including cases ADR 0007's design was explicitly written to +handle (follow-up inheritance, under-specified dose clarify). Reviving +`Focus`/`ConversationState`/TTL/the PLAN-REFINE loop would mean +reintroducing exactly the state-machine complexity `agent.py`'s own +docstring says was deliberately removed. ADR 0007 section 6 ("Refused: an +LLM confidence score as the loop's uncertainty signal") is itself evidence +this was a genuine architecture pivot, not an unfinished build — the live +system now uses exactly that judgment as its ask/answer signal. + +**Done:** +- Deleted the 3 dead modules + 5 dedicated test files. `apps/ai-service`: + **153 passed** (was 195; 42 tests removed with the dead code, nothing else + broke — confirms they were truly isolated). Server restarted, boots clean, + a real query still answers correctly. +- `docs/adr/0007-conversational-reasoning-rag.md`: status changed to + "superseded by ADR 0008," with a note explaining why and pointing to what + it got right that's still owed (F-08 request budget, a durable + cross-worker conversation store). Kept unedited below the notice — an ADR + is a historical decision record, not something to rewrite in place. +- New `docs/adr/0008-llm-understanding-one-shot-rag.md`: documents what + actually runs today — one LLM call understands the turn against plain + history, a single deterministic retrieval dispatch (no PLAN/REFINE round + budget because there's only ever one retrieval call), generation verified + twice (grounding + entailment) with no confidence score, and the + 2026-08-07 context-synthesis fix. States plainly what's still open (F-08, + in-process-only history, no adversarial regression suite beyond one + case) instead of letting the new doc drift stale the same way the old one did. +- `docs/architecture.md`: fixed the audit-flagged false claim ("Retrieval- + confidence gate: below a similarity threshold, skip the LLM call + entirely" — never true of the live path, only the legacy no-generator + fallback) to describe the real deterministic-routing/quarantine-gate + design. While in the same sections: also fixed adjacent, equally-stale + claims noticed along the way — every "OpenAI" reference (the service + actually calls AWS Bedrock: Cohere embed-v4, Qwen3 via Converse, Cohere + rerank) and the wrong Qdrant collection name (`drug_monographs_v1` → + actual live `duocthu_v1`). Did not do a full audit of the rest of the + file (build-roadmap phase claims for auth/chat-service/k8s) — out of + scope for this specific reconciliation. + +`apps/web`: no changes this entry (backend/docs only); typecheck unaffected. + +## 2026-08-07 (cont. 2) — Second audit P0 fixed: population/weight/age/route now reach retrieval and generation, not just the frame + +Owner asked to check why the quick-reply chip loop kept re-asking the same +question ("Uống" → same "uống hay đặt trực tràng?" back), and separately +asked which of yesterday's "Top 3 picked for next session" items were done. +Checked the file directly (`docs/progress-log.md` line 261-267, cont. 12): +(1) interaction-quarantine drop — done earlier today; (2) wire population/ +weight/age into `retrieve_framed` for real — not done, and turned out to be +exactly the root cause of the chip-loop bug; (3) reconcile `architecture.md`/ +ADR 0007 with live reality — still untouched, not started this session either. + +**Root-caused the chip-loop bug with full prompt/response visibility**, not +guessing: wrote a throwaway script that monkeypatched the live generator to +capture the exact system+user prompt and raw LLM JSON for the failing turn. +Two real findings, not one: + +1. Conversation history **was** reaching the LLM correctly — the captured + prompt showed all 4 prior turns verbatim, and the model correctly read + `population=nguoi_lon` from two turns back. Multi-turn history plumbing + itself was never the problem. +2. **`QueryFrame` had no field to hold "route of administration."** When the + model correctly recognized "Uống" as answering its own prior route + question, it had nowhere in its output schema to record that — only + `population`/`weight_kg`/`age_text`/`indication` existed. With nothing to + write, it could only re-emit the identical `clarify_reason` it asked + before. Confirmed directly from the captured raw JSON response. +3. **A second, independent gap, matching exactly the audit's P0-2** named in + yesterday's cont. 12 entry: even where the frame *does* correctly resolve + population/weight/age, nothing downstream ever reads those fields. + `GroundedAnswerService.answer_from_result(query, result)` takes only a + bare `query` string with no notion of conversation history — confirmed by + `grep`, zero references to `history` anywhere in `rag/answer.py`. So the + sufficiency-check and generation LLM calls that decide whether to answer + or ask again would have seen only the literal current turn ("Uống"), + blind to everything resolved in earlier turns, regardless of whether + route existed as a frame field. + +**Fixed both together** (fixing only one wouldn't have closed the loop): +- `rag/understanding.py`: `QueryFrame` gained `route: str | None`; + `FRAME_SCHEMA`/`_SYSTEM` updated with an explicit rule — a short reply + following the model's own last clarify question must be read as resolving + that dimension, keep already-known fields, and flip `needs_clarify=false` + once population + route (+ age/weight if a child) are all known, instead + of re-emitting the same `clarify_reason` verbatim. +- `rag/agent.py`: new `_synthesize_query(turn, frame)` folds every resolved + frame field (population/age_text/weight_kg/route/indication) into a + self-contained question string — e.g. `"Uống. Đối tượng: người lớn. Đường + dùng: uống."` — used in `_single_drug` for both `retrieve_framed`'s rerank + signal and (more importantly) as the `query` handed to + `answer_from_result`, so sufficiency-check/generation are no longer blind + to context resolved in earlier turns. No-op (returns `turn` unchanged) when + the frame has no resolved fields, so a fresh single-shot question is + unaffected. 6 new tests across `test_understanding.py`/`test_agent.py`, + including a direct regression test asserting the exact prior failure case + now produces `needs_clarify=false` and a context-carrying query. + +**Verified live, twice, against the real running server** (restarted after +the code change, per house rule): first with a direct script reproducing the +exact 3-turn conversation that failed before (`agent.handle()` called 3 +times against the live Qdrant/Bedrock stack) — turn 3 ("Uống") now returns a +real grounded answer scoped to the oral dose, not a repeated question. Then +again through the actual browser UI end to end (chip click → "Người lớn" → +typed "Uống") — same result: real answer, `ENTAILED & GROUNDED` + +`AI diễn giải, đã kiểm chứng`, 2 real citations, citation beam working. + +`apps/ai-service`: **195 passed** (was 191, +4 net: 2 route-parsing tests in +`test_understanding.py`, 2 query-synthesis tests in `test_agent.py`). +`apps/web` typechecks clean (no frontend changes this entry). + +**Still open, unchanged from this morning:** item (3) from yesterday's Top +3 — reconciling `architecture.md`/ADR 0007 with what's actually live. Also +still open: the model-latency investigation's proposed fix (a smaller/faster +model for the `understand` step only) — diagnosed, not attempted. + +## 2026-08-07 (cont.) — Interaction-quarantine P0 fixed, citation duplicates merged, quick-reply chips added + +Owner said to go do the outstanding items from the session above, plus asked +for clickable quick-reply options on clarifying questions (like this tool's +own option-picker). + +**P0 fixed: `_interaction` no longer silently drops a quarantined drug's +evidence.** `rag/agent.py::_interaction` used to keep only `part.decision == +ANSWERABLE` parts before combining two drugs' interaction evidence, then +hardcoded the combined `RetrievalResult` to `ANSWERABLE` — so if one drug's +`tuong_tac_thuoc` section had a quarantined table, its evidence (and the +"table exists, verify PDF" notice the quarantine contract requires) was +dropped instead of surfaced; a confident interaction answer could omit a real +unverified contraindication table for one of the two drugs +([[project-quarantined-block-contract]]). Fixed: added +`RetrievalService.decide()` (a public wrapper around the existing `_decide` +policy) and `_interaction` now keeps both `ANSWERABLE` and `VERIFY_PDF` parts, +then re-derives the combined decision through `decide()` instead of +hand-rolling it — the same quarantine policy the single-drug path already +applies. New regression test +(`test_interaction_with_one_drug_quarantined_never_answers_confidently`) +locks this in with a synthetic quarantined case. **Could not be demonstrated +live end-to-end**: checked the real corpus and found 0 of 487 quarantined +chunks are in `tuong_tac_thuoc` — no real drug pair exists today where this +exact path fires, so the fix is proven by unit test + live regression-check +of the normal (non-quarantined) interaction case (warfarin+aspirin, unchanged +behavior, 2 citations, `generated=true`), not by a live quarantined-interaction +probe. + +**Citation duplication fixed, but it turned out not to be pure duplication.** +Investigated the "near-duplicate citation cards" rough edge named at the end +of the previous entry. Traced a real case (Acetazolamid, quarantined +`duoc_ly_va_co_che_tac_dung` table): the two citations for one evidence block +have DIFFERENT physical pages — the prose paragraph sits on physical page 108 +(printed 109), the table it mentions sits on physical page 109 (printed 110). +So merging them naively would have hidden real information. Fixed properly in +`route.ts::toCitations`: citations are grouped by `chunk_id` into one card, +using the plain-text ref's page as the card's primary location and keeping +the attachment ref's own page as a new `quarantinePhysicalPage` field — +`CitationCard`'s "Mở trang PDF gốc" link now opens the TABLE's own page, not +the prose's page. `Citation` DTO gained `quarantinePhysicalPage?: number`. + +**Quick-reply chips added for clarifying questions**, per owner's request +("thêm câu trả lời cho câu hỏi thêm kiểu lựa chọn như của claude ấy"). Two +independent clarify sources both needed wiring — found the hard way by +testing live: +1. `GroundedAnswerService._check_sufficiency` (the dose-under-specified + check) — `rag/prompt.py`'s `SUFFICIENCY_SCHEMA` gained `quick_replies: + string[]`, `_check_sufficiency` now returns `(question, quick_replies)`, + threaded through `GroundedAnswer.quick_replies` → `AgentReply.quick_replies` + → `RagQueryResponse.quick_replies`. +2. **The path real traffic actually hits** (confirmed live — every clarify in + this session's testing came from here, not #1): `understanding.py`'s + `LlmQueryUnderstander` sets `QueryFrame.needs_clarify`/`clarify_reason` + directly from its own single LLM call, and `RagAgent._route()` returns + that immediately, short-circuiting before retrieval/`GroundedAnswerService` + ever runs. Initially wired only #1 and shipped it — live-tested and found + quick_replies came back empty every time; root-caused to this second, + dominant path and fixed it too: `QueryFrame` gained `quick_replies`, + `FRAME_SCHEMA`/`_SYSTEM` prompt updated, `_parse()` extracts it, `_route` + passes it through. 5 new tests across `test_agent.py`/ + `test_understanding.py`/`test_citation_and_intro.py`. + +Frontend: `ChatMessage.quickReplies?: string[]`; `route.ts` only surfaces them +when `decision === "clarify"` and the list is non-empty; `ChatBubble` renders +them as clickable chips (only under a real `clarify` decision) that call +`onQuickReply`, wired in `ChatPanel` straight into `handleSendMessage` — a +click sends that exact text as the next turn, no different from typing it. + +**Verified live** (server restarted after each backend change, per house +rule): "Liều paracetamol hạ sốt là bao nhiêu?" → clarify with 4 real chips +("Người lớn", "Trẻ em <1 tuổi", "Trẻ em 1-5 tuổi", "Trẻ em 6-12 tuổi"), +clicking "Người lớn" correctly auto-sent it and produced a follow-up clarify +("Uống hay đặt trực tràng?") with its own 2 chips — the chip mechanism itself +(render → click → auto-send → new response) works end to end. + +**New issue found while verifying, not fixed today:** clicking "Uống" (the +chip's own suggested answer) got the SAME "uống hay đặt trực tràng?" question +back, twice in a row, even though `_remember()` does put "Người dùng: Uống" +in the history the very next call reads. The understanding LLM isn't reliably +resolving a terse one-word reply against its own immediately-preceding +`clarify_reason` — a conversational-memory prompt weakness in +`understanding.py`, separate from the chip UI itself (which correctly sent +the text every time). Worth a dedicated pass: likely needs the prompt to +explicitly say "a short reply with no drug name answers your own last +clarify_reason" rather than relying on the model to infer that from bare +history lines. + +`apps/ai-service`: **191 passed** (was 186 before this cont., +5 for the P0 +regression test and quick-reply coverage). `apps/web` typechecks clean. + +## 2026-08-07 — Citation UI now shows real retrieved data instead of fabricated placeholders + +Owner asked to fix the UI/UX so it shows precisely what was retrieved and how +the LLM answered from it, and to read all memories first. Traced the citation +pipeline end to end (`rag/answer.py` → `routers/rag.py` → `apps/web/app/api/ +chat/route.ts` → `CitationCard.tsx`) and found it was showing manufactured +data at several points, not real data: + +1. **`route.ts`'s `RagCitation` interface declared `text_snippet`/ + `citation_reason` fields that don't exist on the real backend + `CitationResponse`** — always `undefined`, so every citation's snippet was + blank and its "reason" silently fell back to a canned boilerplate sentence + ("Trích xuất từ mục X làm căn cứ...") presented as if it were real + entailment reasoning. +2. **The backend never exposed the retrieved chunk text at all.** `Citation` + (`rag/answer.py`) carried only page/block pointers, so even a frontend fix + alone could not have shown real evidence. +3. **`CitationCard.tsx`'s `SECTION_LABELS` map used guessed section-key + slugs** (`lieu_dung`, `duoc_ly`, `tac_dung_phu`, `qua_lieu`, `bao_quan`) + that don't match the corpus's real 19-field schema (`lieu_luong_va_cach_ + dung`, `duoc_ly_va_co_che_tac_dung`, `tac_dung_khong_mong_muon`, ...) — + every citation fell back to the raw slug instead of a label. +4. **`tra-cuu/page.tsx`'s PDF-jump used the printed page number as the + `#page=` fragment.** Verified against the real PDF (rendered physical + pages 106-110 with PyMuPDF and read the text) that physical page ≠ printed + page — off by 1-3 depending on front-matter offset, confirmed across 1,431 + sampled chunks. Right by coincidence in the majority case, wrong the rest + of the time. Fixed to use `physical_page + 1` (physical_page is PyMuPDF's + 0-indexed page; the `#page=` fragment is 1-indexed — verified directly by + opening the resulting PDF tab and reading the rendered page). + +**Fixed backend** (`rag/answer.py`, `routers/rag.py`): `Citation`/ +`CitationResponse` gained `evidence_text` — the literal chunk text handed to +the generator/entailment check, not a paraphrase. `RagQueryResponse` gained +`generated: bool` so the UI can honestly distinguish an LLM paraphrase +(passed grounding + entailment) from a verbatim extractive quote (the +`ANSWER_PROVIDER=disabled` mode, or a configured generator's canned +`VERIFY_PDF` message). + +**Fixed frontend:** `Citation` DTO rewritten to match real fields (`chunkId`, +`physicalPage`, real `snippet`, `isQuarantined`/`quarantineNotice` in place of +the fabricated `reason`); `route.ts` now derives `drugName`/`sectionType` +**per citation** from `chunk_id.split("__")` instead of stamping every +citation with the turn's single `resolved_drug_id` (wrong on the 2-drug +interaction path — verified live with a warfarin+aspirin query, both +citations correctly show "WARFARIN", not the old combined string); +`CitationCard` renders the real evidence text, a genuine quarantine banner +(with a working "open PDF at the right page" link) only when the source +pipeline actually flagged that chunk, and the corrected section labels; +`ChatBubble` gained a truthful "AI diễn giải, đã kiểm chứng" vs "Trích dẫn +nguyên văn" pill — gated to `decision === "answerable"` only, after live +testing caught it mislabeling a clarifying question as "verbatim quote." +`verify_pdf` and `clarify` decisions now get their own distinct header +badges instead of borrowing the grounded/ungrounded binary. + +**Verified live, not just unit tests** (per house rule): stood up local +Qdrant + Postgres (Docker, pre-existing volumes — 15,100 pts intact) and the +real ai-service + web servers, drove three real queries through the actual +browser: +- Simple dose question (paracetamol) → real `evidence_text` shown, correct + section label, "AI diễn giải, đã kiểm chứng" pill. +- Interaction question (warfarin + aspirin) → both citations correctly show + "WARFARIN" (both drawn from warfarin's own `tuong_tac_thuoc` section). +- Quarantined-table question (Acetazolamid dược lý, page 110) → + "CẦN ĐỐI CHIẾU PDF GỐC" badge, quarantine banner rendered, clicked "Mở + trang PDF gốc" and confirmed in the opened PDF tab that it lands exactly + on the physical page showing the real quarantined table (the pharmacokinetic + timing table) — the page-jump is now provably correct, not just plausible. + +`apps/ai-service`: **186 passed**, no regressions. `apps/web` typechecks clean +(`tsc --noEmit`). + +**Known rough edges, named rather than hidden, not fixed today:** +- `_indexed_citations` emits one `Citation` per `source_ref`, so a quarantined + chunk (base prose ref + attachment ref) produces two near-duplicate citation + cards with identical snippet text. Pre-existing data shape, not introduced + today. A dedup pass needs to preserve the quarantine flag from whichever ref + carries it — not attempted, to avoid rushing something that could silently + drop the quarantine signal. +- This UI fix makes a quarantined citation genuinely visible **when the + backend sends it**, but does not fix the already-known P0 where the 2-drug + interaction path (`agent.py::_interaction`) silently drops a quarantined + drug's evidence instead of surfacing "table exists" for it + ([[project-quarantined-block-contract]]). Still next-session work. +- `source_crop` is `None` across the entire live corpus (checked: 0/15,100 + chunks) — the table-reconstruction pass that would populate it is a + separate in-progress track (11 crop PNGs generated so far, not yet loaded). + The `` rendering path in `CitationCard` is wired but dormant; it + activates automatically once that data lands. Until then, quarantined + citations fall back to the "open PDF at the right page" link, which is + itself now verified-correct. + +## 2026-08-06 (cont. 12) — Independent senior-engineer audit, 7 parallel agents, read-only (no fixes applied yet) + +Owner asked for a full RAG audit (parsing → chunking → retrieval → query +understanding/reasoning → grounding/safety → evaluation → production +engineering), API-only architecture, no fine-tuning proposals, no redesign, +report only. Ran 7 subagents in parallel, each required to read real +implementation + run real tests before concluding. Headline finding: the +code running in production (`rag/agent.py`, wired via `bootstrap.py`) is +**not** the architecture described in `docs/architecture.md` or ADR 0007 — +three separate live/dead-code mismatches independently surfaced by three +different agents: + +- **Dense vector search is dead code live.** `retrieve_framed` (the only + method `RagAgent` calls) only ever does exact `find_by_section`/ + `find_by_drug` payload-filter scroll, never `QdrantRetriever.search()`. + "Hybrid retrieval" doesn't exist in production either (only in + `rag/in_memory.py`'s test fallback). +- **`architecture.md`'s "retrieval-confidence gate: skip LLM below a + similarity threshold" is false for the live path.** The threshold only + exists on the legacy `RetrievalService.retrieve()`, which `RagAgent` + never calls. +- **ADR 0007's entire reasoning-loop design (`reasoning.py`, + `conversation.py`, `conversational.py` — Focus/TTL, turn budget, + sufficiency-driven retrieval refinement) is dead code.** `bootstrap.py` + builds `RagAgent` with none of it; the live agent is a fixed one-shot + pipeline (understand → route → retrieve once → generate → ≤2 entailment + retries), not an iterative loop that feeds back into retrieval. + +Two new bugs found (not previously known): +1. **P0 — `_interaction` (agent.py:144-149) silently drops a drug's + interaction evidence if it's quarantined (`VERIFY_PDF`)**, without + telling the user — violates the [[project-quarantined-block-contract]] + obligation ("must make the answer say a table exists") specifically on + the 2-drug interaction path; the single-drug path already obeys it. +2. **P0 — `QueryFrame.population/weight_kg/age_text/indication` are + extracted by `understanding.py` but never passed into + `retrieve_framed`/`answer_from_result`.** This is exactly the bug ADR + 0007 was written to fix ("liều paracetamol cho người lớn" vs "liều + paracetamol" can retrieve identically) — the structured signal exists, + the plumbing into retrieval that would guarantee it does not. + +Ingestion side (parsing/chunking) came out strong and independently +verified against real whole-corpus artifacts, not docs: back-index +recall 96.2%/precision 99.1% (live CLI run), all 30 `chunk-ready` gates +PASS on the real 684-monograph/15,100-chunk corpus, deterministic rebuild +confirmed by sha256 diff + live-Qdrant idempotent-upsert test. Two smaller +ingestion bugs found: `extract/spans.py`'s reading-order sort treats every +`full_width` block as page-header material — falsified by 9 real +mid-page full-width tables in `table_regions.json` (2/9 traced through to +final output were fine, 7/9 unverified); and `chunk/chunker.py`'s +`_SUBGROUP_LABEL` regex (the guard against splitting mid-subsection) is +missing pregnancy/breastfeeding terms (`phụ nữ|mang thai|thai|cho con +bú`), 11 real occurrences in-corpus, no confirmed bad split yet but +uncovered by the guard. + +Evaluation-coverage gap: full unit suites pass for real (`apps/ai-service` +186 passed/4 skipped, `ingestion` 296 passed/0 skipped), but there is +**no automated regression re-run of the golden sets** — `run_eval.py` is +unusable as committed (missing fixtures), `evals/manual_adversarial_ +hard10.jsonl` (has exactly the table/formula/vet-abstain/cross-page cases +needed) is never read by any script or test, no CI workflow exists, and +NDCG/Precision@K are entirely absent repo-wide (MRR exists only in the +ingestion embedding benchmark, unwired to retrieval eval). + +Production engineering: no hardcoded secrets found (checked). k8s/Helm/ +Terraform/Dockerfiles are genuinely empty scaffolding (Phase 6, as +roadmapped — not a surprise). No circuit breaker, no exception handling +around the Qdrant scroll calls actually used live (outage → raw HTTP 500, +not a graceful abstain), no end-to-end request timeout budget (F-08 still +open — worst case several minutes, no aggregate cutoff), `/health` +unconditionally returns ok with no downstream check, 6 of the metric names +defined in `rag/metrics.py` aren't registered in `adapters/prometheus.py` +(silent no-op if incremented), conversation history is an in-process dict +(lost on restart, not shared across workers). + +**Top 3 picked for next session** (see full report in this session's +transcript for file:line detail on every item above): (1) fix the +interaction-quarantine silent drop, (2) wire population/weight/age into +`retrieve_framed` for real, (3) reconcile `architecture.md`/ADR 0007 with +what's actually live — either add the retrieval-confidence floor for +real and delete the two dead reasoning-loop modules, or wire them in; stop +carrying two contradictory architectures side by side. + +No code changed this session — read-only audit per owner's explicit +instruction. Full agent-by-agent findings (parsing, chunking, retrieval, +query-understanding/reasoning-loop, grounding/safety, evaluation, +production) not reproduced here in full; re-run the same 7-way audit +prompt if the detail is needed again, or ask the owner for the chat +transcript. + ## 2026-08-06 (cont. 11) — Real bug found by actually running the golden eval set: "thận trọng" silently answered as "chống chỉ định" Owner pointed at a golden dataset (`Golden Dataset/golden_e2e_v1.csv` +4 diff --git a/infra/docker/Caddyfile b/infra/docker/Caddyfile new file mode 100644 index 0000000..e082f80 --- /dev/null +++ b/infra/docker/Caddyfile @@ -0,0 +1,3 @@ +realvuxbaro.me { + reverse_proxy web:3000 +} diff --git a/infra/docker/docker-compose.prod.yml b/infra/docker/docker-compose.prod.yml new file mode 100644 index 0000000..a7238e8 --- /dev/null +++ b/infra/docker/docker-compose.prod.yml @@ -0,0 +1,58 @@ +# Production topology for a single EC2 box. No GPU, no team k3s — Bedrock +# calls go out over the instance's IAM role (see coordination/WORK_SPLIT_ +# 2026-08-10.md), so no AWS access keys live in this file or its env files. +services: + postgres: + image: postgres:16-alpine + environment: + POSTGRES_USER: duoc_thu + POSTGRES_PASSWORD: duoc_thu + POSTGRES_DB: duoc_thu + volumes: + - postgres-data:/var/lib/postgresql/data + restart: unless-stopped + + qdrant: + image: qdrant/qdrant:latest + volumes: + - qdrant-data:/qdrant/storage + restart: unless-stopped + + ai-service: + build: + context: ../.. + dockerfile: apps/ai-service/Dockerfile + env_file: ../../apps/ai-service/.env.prod + depends_on: + - postgres + - qdrant + restart: unless-stopped + + web: + build: + context: ../.. + dockerfile: apps/web/Dockerfile + environment: + AI_SERVICE_URL: http://ai-service:8000 + depends_on: + - ai-service + restart: unless-stopped + + caddy: + image: caddy:2-alpine + ports: + - "80:80" + - "443:443" + volumes: + - ./Caddyfile:/etc/caddy/Caddyfile:ro + - caddy-data:/data + - caddy-config:/config + depends_on: + - web + restart: unless-stopped + +volumes: + postgres-data: + qdrant-data: + caddy-data: + caddy-config: diff --git a/packages/shared-types/src/dto/chat.ts b/packages/shared-types/src/dto/chat.ts index 2171f2c..19b8810 100644 --- a/packages/shared-types/src/dto/chat.ts +++ b/packages/shared-types/src/dto/chat.ts @@ -1,9 +1,26 @@ export interface Citation { + chunkId: string; drugName: string; sectionType: string; + /** Printed page numbers as they appear in the book — what a clinician reads off their paper copy. */ sourcePageRange: [number, number]; - snippet?: string; - reason?: string; + /** PyMuPDF 0-indexed page of the physical PDF file — NOT the printed page. Add 1 for a `#page=` viewer fragment. */ + physicalPage: number; + /** The exact chunk text retrieved and handed to the LLM/grounding check — not a paraphrase. */ + snippet: string; + /** True when this chunk carries a table/formula the pipeline quarantined (never linearised into text). */ + isQuarantined: boolean; + /** Present only when `isQuarantined` — a truthful notice, not boilerplate. */ + quarantineNotice?: string; + /** + * The quarantined table/formula's OWN physical page, when it differs from + * `physicalPage` (a table often sits on the page after the paragraph that + * mentions it — verified on real data, not assumed). Falls back to + * `physicalPage` when absent. Only meaningful when `isQuarantined`. + */ + quarantinePhysicalPage?: number; + /** A rendered crop of the source table/formula, when reconstruction has produced one. Usually absent today. */ + sourceCropUrl?: string; } export interface ChatMessage { @@ -16,8 +33,17 @@ export interface ChatMessage { decision?: string; reason?: string; grounded?: boolean; + /** True: the LLM paraphrased the evidence and it passed grounding+entailment. False: verbatim source quote (no generator configured, or a configured one that failed and abstained). */ + generated?: boolean; resolvedDrugId?: string; createdAt: string; + /** + * Short suggested replies for a `decision: "clarify"` turn — e.g. ["Người + * lớn", "Trẻ em"] for an age-band question. Optional: the model doesn't + * always produce clean short options (an open-ended clarify has none), + * and the UI must fall back to free text either way. + */ + quickReplies?: string[]; } export interface SendMessageRequest { diff --git a/packages/ui/src/ChatBubble.tsx b/packages/ui/src/ChatBubble.tsx index b8b2d01..f09c1bf 100644 --- a/packages/ui/src/ChatBubble.tsx +++ b/packages/ui/src/ChatBubble.tsx @@ -14,14 +14,25 @@ import { RotateCcw, ExternalLink, BookOpen, + CornerDownRight, } from "lucide-react"; import { cn } from "./lib/utils"; interface ChatBubbleProps { message: ChatMessage; - onCitationClick?: (citation: Citation, index: number) => void; + // `allCitations` is THIS message's own citation list (`message.citations`) + // — found live 2026-08-07: the caller previously had no way to know which + // message a click came from, so it kept showing whatever citation array a + // LATER message had most recently loaded. Clicking [1] on an old answer + // (e.g. Omeprazol's mechanism) displayed a completely unrelated later + // drug's evidence (e.g. Kanamycin) in the source panel — a real, live- + // reported bug for a product whose entire value proposition is a + // verifiable citation trail. + onCitationClick?: (citation: Citation, index: number, allCitations: Citation[]) => void; activeCitationIndex?: number | null; onRetry?: () => void; + /** Fired when the user picks a quick-reply chip instead of typing — sends that text as the next turn. */ + onQuickReply?: (text: string) => void; className?: string; } @@ -30,6 +41,7 @@ export function ChatBubble({ onCitationClick, activeCitationIndex, onRetry, + onQuickReply, className, }: ChatBubbleProps) { const [copied, setCopied] = useState(false); @@ -51,83 +63,92 @@ export function ChatBubble({ ); } + // Citation markers must render INLINE within whatever block (paragraph/ + // bullet/heading) they end the sentence of — never split out as a bare + // top-level element. Found live 2026-08-07: the old approach split the + // WHOLE content on `[n]` first and only then broke each piece into lines, + // so a citation sitting on its own source line (a common shape for a + // multi-band dosing answer, e.g. "...5,4 g/ngày\n[1]\n; cấp tính...") ended + // up as a bare ` + ); + }); + }; + // Helper to parse citations [1], [2] in markdown content const renderStructuredContent = (content: string, citations?: Citation[]) => { - // Split content by citations like [1], [2], etc. - const parts = content.split(/(\[\d+\])/g); - - return parts.map((part, i) => { - const match = part.match(/^\[(\d+)\]$/); - if (match) { - const citationIndex = parseInt(match[1], 10); - const citationObj = citations && citations[citationIndex - 1]; - const isActive = activeCitationIndex === citationIndex; + const lines = content.split("\n"); + return lines.map((line, lineIdx) => { + if (!line.trim()) return
; + // Heading 2 or 3 + if (line.startsWith("### ") || line.startsWith("## ")) { return ( - +

+ + {renderInline(line.replace(/^#+\s*/, ""), `h-${lineIdx}`, citations)} +

); } - // Format markdown-like text lines - const lines = part.split("\n"); + // Bullet points + if (line.trim().startsWith("- ") || line.trim().startsWith("* ")) { + return ( +
  • + {renderInline(line.trim().replace(/^[-*]\s*/, ""), `li-${lineIdx}`, citations)} +
    • + ); + } + + // Warning block / Note + if (line.includes("Chống chỉ định") || line.includes("Cảnh báo") || line.includes("Thận trọng")) { + return ( +
    + +
    {renderInline(line, `w-${lineIdx}`, citations)}
    +
    + ); + } + + // The primary clinical answer text — the one thing on this whole card + // a clinician actually needs to read, so it carries more visual + // weight than the surrounding header/footer chrome (found live + // 2026-08-07: previously the same low-emphasis size/weight as + // everything else, easy to skim past). return ( - - {lines.map((line, lineIdx) => { - if (!line.trim()) return
    ; - - // Heading 2 or 3 - if (line.startsWith("### ") || line.startsWith("## ")) { - return ( -

    - - {line.replace(/^#+\s*/, "")} -

    - ); - } - - // Bullet points - if (line.trim().startsWith("- ") || line.trim().startsWith("* ")) { - return ( -
  • - {formatBoldText(line.trim().replace(/^[-*]\s*/, ""))} -
    • - ); - } - - // Warning block / Note - if (line.includes("Chống chỉ định") || line.includes("Cảnh báo") || line.includes("Thận trọng")) { - return ( -
    - -
    {formatBoldText(line)}
    -
    - ); - } - - return ( -

    - {formatBoldText(line)} -

    - ); - })} -     +

    + {renderInline(line, `p-${lineIdx}`, citations)} +

    ); }); }; @@ -175,11 +196,21 @@ export function ChatBubble({
    - {message.grounded !== false ? ( + {message.decision === "answerable" && message.grounded !== false ? ( ENTAILED & GROUNDED + ) : message.decision === "verify_pdf" ? ( + + + CẦN ĐỐI CHIẾU PDF GỐC + + ) : message.decision === "clarify" ? ( + + + CẦN LÀM RÕ CÂU HỎI + ) : ( @@ -187,6 +218,33 @@ export function ChatBubble({ )} + {/* How the answer was produced from the retrieved evidence — only + meaningful for a real answerable turn (rag/answer.py's two + operating modes: LLM paraphrase vs. verbatim quote). A clarify + or verify_pdf turn is neither, so it gets no source-mode pill. */} + {message.decision === "answerable" && message.grounded !== false && message.generated !== undefined && ( + + {message.generated ? ( + <> + + AI diễn giải, đã kiểm chứng + + ) : ( + <> + + Trích dẫn nguyên văn + + )} + + )} + @@ -198,6 +256,23 @@ export function ChatBubble({ {renderStructuredContent(message.content, message.citations)}
    + {/* Quick-reply chips — only for a clarify turn the model gave a few + natural discrete answers to; free text always still works. */} + {message.quickReplies && message.quickReplies.length > 0 && ( +
    + {message.quickReplies.map((reply, idx) => ( + + ))} +
    + )} + {/* Disclaimer Section inside document */} {message.disclaimer && (
    diff --git a/packages/ui/src/CitationCard.tsx b/packages/ui/src/CitationCard.tsx index 2bca09c..c05ed4d 100644 --- a/packages/ui/src/CitationCard.tsx +++ b/packages/ui/src/CitationCard.tsx @@ -2,7 +2,7 @@ import React from "react"; import type { Citation } from "@duoc-thu/shared-types"; -import { BookOpen, FileText, CheckCircle2, ChevronRight } from "lucide-react"; +import { BookOpen, FileText, ChevronRight, AlertTriangle, ExternalLink } from "lucide-react"; import { cn } from "./lib/utils"; interface CitationCardProps { @@ -13,16 +13,31 @@ interface CitationCardProps { className?: string; } +// Matches the real 19-field section_key enum the corpus is chunked on +// (apps/ai-service/rag/understanding.py SECTION_KEY_HINTS) — the previous +// map here used guessed keys (lieu_dung, tac_dung_phu, duoc_ly, qua_lieu, +// bao_quan) that never matched a real chunk, so every citation silently +// fell back to the raw slug instead of a readable label. const SECTION_LABELS: Record = { + ten_chung_quoc_te: "Tên chung quốc tế", + ten_thuong_mai: "Tên thương mại", + ma_atc: "Mã ATC", + loai_thuoc: "Phân loại thuốc", + dang_thuoc_va_ham_luong: "Dạng thuốc & Hàm lượng", + duoc_ly_va_co_che_tac_dung: "Dược lý & Cơ chế tác dụng", chi_dinh: "Chỉ định", chong_chi_dinh: "Chống chỉ định", - lieu_dung: "Liều lượng & Cách dùng", - tac_dung_phu: "Tác dụng không mong muốn (ADR)", + than_trong: "Thận trọng", + thoi_ky_mang_thai: "Thời kỳ mang thai", + thoi_ky_cho_con_bu: "Thời kỳ cho con bú", + tac_dung_khong_mong_muon: "Tác dụng không mong muốn (ADR)", + huong_dan_xu_tri_adr: "Hướng dẫn xử trí ADR", + lieu_luong_va_cach_dung: "Liều lượng & Cách dùng", tuong_tac_thuoc: "Tương tác thuốc", - duoc_ly: "Dược lý & Cơ chế tác dụng", - than_trong: "Thận trọng khi dùng", - qua_lieu: "Quá liều & Xử trí", - bao_quan: "Bảo quản", + qua_lieu_va_xu_tri: "Quá liều & Xử trí", + do_on_dinh_va_bao_quan: "Độ ổn định & Bảo quản", + tuong_ky: "Tương kỵ", + thong_tin_quy_che: "Thông tin quy chế", }; export function CitationCard({ @@ -95,21 +110,43 @@ export function CitationCard({
    - {/* Snippet / Source Excerpt */} + {/* Snippet / Source Excerpt - the exact retrieved chunk text, verbatim */} {citation.snippet && ( -
    - “ +
    + “ {citation.snippet} - ” + ”
    )} - {/* Reason / Entailment Note */} - {citation.reason && ( -

    - - {citation.reason} -

    + {/* Quarantine notice - only rendered when the source pipeline actually + flagged this chunk (table/formula lifted out of prose), never a + generic boilerplate line for an ordinary citation. */} + {citation.isQuarantined && ( + )}