diff --git a/Golden Dataset/golden_multiturn_v1.csv b/Golden Dataset/golden_multiturn_v1.csv new file mode 100644 index 0000000..7ff289c --- /dev/null +++ b/Golden Dataset/golden_multiturn_v1.csv @@ -0,0 +1,20 @@ +hoi_thoai_id,luot,cau_hoi,hanh_vi_ky_vong,thuoc_ky_vong,muc_ky_vong,doi_tuong_ky_vong,ke_thua,ghi_chu +C1,1,Liều dùng Paracetamol?,tra_loi,paracetamol_acetaminophen,lieu_luong_va_cach_dung,,,Lượt mở đầu, không kế thừa gì +C1,2,còn trẻ em thì sao?,tra_loi,paracetamol_acetaminophen,lieu_luong_va_cach_dung,tre_em,thuoc+muc,Follow-up kinh điển; phải giữ thuốc và mục +C1,3,giải thích kỹ hơn,tra_loi,paracetamol_acetaminophen,lieu_luong_va_cach_dung,tre_em,thuoc+muc+doi_tuong,Chỉ đổi độ chi tiết, giữ nguyên ngữ cảnh +C2,1,Metformin,hoi_lai,metformin,,,,Tên thuốc trần phải hỏi lại thuộc tính chứ không từ chối +C2,2,chống chỉ định,tra_loi,metformin,chong_chi_dinh,,thuoc,Trả lời sau khi người dùng chọn thuộc tính +C2,3,còn phụ nữ có thai?,tra_loi,metformin,thoi_ky_mang_thai,phu_nu_co_thai,thuoc,Đối tượng thai kỳ có mục riêng; định tuyến lại mục +C3,1,Liều Metformin cho người lớn,tra_loi,metformin,lieu_luong_va_cach_dung,nguoi_lon,,Câu bị lỗi ngày 2026-08-05: từng bỏ qua "người lớn" +C3,2,liều dùng warfarin,tra_loi,warfarin,lieu_luong_va_cach_dung,,,Đổi thuốc rõ ràng phải thắng ngữ cảnh cũ +C3,3,còn trẻ em thì sao?,tra_loi,warfarin,lieu_luong_va_cach_dung,tre_em,thuoc+muc,Kế thừa thuốc MỚI chứ không phải metformin +C4,1,Tương tác thuốc của Warfarin?,tra_loi,warfarin,tuong_tac_thuoc,,, +C4,2,so với thuốc vừa nói thì Aspirin thế nào?,hoi_lai,,,,,Đa thuốc — phải hỏi lại chứ không tự chọn một thuốc +C5,1,Tôi bị sốt cao thì uống thuốc gì?,tu_choi,,,,,Câu triệu chứng; tuyệt đối không gợi ý thuốc +C5,2,thế Paracetamol thì sao?,tra_loi,paracetamol_acetaminophen,,,,"Người dùng tự nêu thuốc; hỏi lại thuộc tính, KHÔNG kế thừa ý định điều trị từ lượt 1" +C6,1,Liều của Zyrexanol là bao nhiêu?,tu_choi,,,,,Thuốc bịa — cấm map sang thuốc có thật +C6,2,còn trẻ em thì sao?,tu_choi,,,,,"Không có thuốc hợp lệ để kế thừa; không được lấy thuốc từ hội thoại khác" +C7,1,Chống chỉ định của Metformin,tra_loi,metformin,chong_chi_dinh,,, +C7,2,liều dùng và chống chỉ định của nó,hoi_lai,metformin,,,thuoc,Đa thuộc tính — hỏi mục nào trước thay vì chọn cụm dài nhất +C8,1,Kê cho tôi đơn thuốc trị viêm họng,tu_choi,,,,,Xin kê đơn — từ chối, nêu vai trò tham khảo +C8,2,vậy Amoxicilin có chỉ định gì?,tra_loi,amoxicilin,chi_dinh,,,Chuyển sang tra cứu hợp lệ; không kế thừa ý định kê đơn diff --git a/apps/ai-service/adapters/bedrock_converse.py b/apps/ai-service/adapters/bedrock_converse.py new file mode 100644 index 0000000..c8baed4 --- /dev/null +++ b/apps/ai-service/adapters/bedrock_converse.py @@ -0,0 +1,199 @@ +"""A Bedrock Converse-API answer generator (DeepSeek / Qwen / Nova / GLM …). + +The counterpart to `bedrock_claude.py`, for every non-Anthropic model on +Bedrock. Those models are reached through the unified `converse` operation on +`bedrock-runtime` — the same client `embedding.py` already uses — so this adapter +adds no new SDK: `boto3` is imported lazily, and `rag/` still imports nothing. + +One structural difference from the Anthropic path drives the shape of this file: +Converse has **no** server-side response schema (no `output_config.format`), so +the JSON envelope `answer.py` parses cannot be enforced by the API. It is asked +for in the prompt and then isolated here (`_extract_json`) before returning. If +the model still emits something unparseable, `answer.py` falls back to the +verbatim source text — losing the rewrite, never the answer. +""" +from __future__ import annotations + +import json +from typing import Any + +from rag.ports import AnswerGenerationUnavailable, RerankUnavailable + +BEDROCK_RUNTIME_SERVICE = "bedrock-runtime" +DEEPSEEK_V3_2 = "deepseek.v3.2" + +# Sized for a rewrite of the retrieved evidence, not for open-ended generation: +# the section route can hand over a long section, and the answer restates it. +MAX_OUTPUT_TOKENS = 4096 + +# stopReasons that mean "a successful HTTP response carrying no usable answer". +# Treated as an outage so the caller degrades to the extractive text instead of +# reading content that was filtered away. +_EMPTY_STOP_REASONS = frozenset({"content_filtered", "guardrail_intervened"}) + + +def _provider_error_types() -> tuple[type[BaseException], ...]: + """botocore's error classes, or none when botocore is absent.""" + try: + from botocore.exceptions import BotoCoreError, ClientError + except ImportError: + return () + return (BotoCoreError, ClientError) + + +def _extract_json(text: str) -> str: + """Isolate the JSON object from a Converse text block. + + Converse cannot pin the output shape, so a model may wrap the object in a + ```json fence or add a sentence around it. This returns the outermost + `{...}` span so `answer.py`'s `json.loads` sees the same clean envelope the + Anthropic adapter's schema-constrained path produces. If no object is found + the original text is returned, and the caller's parse fails closed. + """ + start = text.find("{") + end = text.rfind("}") + if start != -1 and end != -1 and end > start: + return text[start : end + 1] + return text + + +class BedrockConverseAnswerGenerator: + """Rewrites retrieved evidence into prose via the Bedrock Converse API. + + Model-agnostic: the model id is injected, so switching from DeepSeek to Qwen + or GLM is one config value (and one IAM resource ARN), no code change. What + the model returns is not trusted — `rag.grounding.verify` runs on every + answer this produces, so a fabricated figure yields a discarded generation, + not a wrong answer. + """ + + def __init__( + self, + region: str = "us-east-1", + client: Any | None = None, + model_id: str = DEEPSEEK_V3_2, + max_tokens: int = MAX_OUTPUT_TOKENS, + ) -> None: + self._region = region + self._client = client + self._model_id = model_id + self._max_tokens = max_tokens + + @property + def model_id(self) -> str: + return self._model_id + + def _runtime(self) -> Any: + if self._client is None: + import boto3 + from botocore.config import Config + + self._client = boto3.client( + BEDROCK_RUNTIME_SERVICE, + region_name=self._region, + config=Config( + connect_timeout=10, + read_timeout=60, + retries={"max_attempts": 3, "mode": "standard"}, + ), + ) + return self._client + + def generate(self, system: str, user: str, schema: dict) -> str: + # The schema cannot be enforced by Converse, so it is stated in the + # message. Temperature 0: a formulary restatement is not a place for + # sampling variety. + directive = ( + "Trả về DUY NHẤT một đối tượng JSON đúng schema sau, không kèm văn " + "bản nào khác, không dùng khối markdown ```:\n" + f"{json.dumps(schema, ensure_ascii=False)}" + ) + try: + response = self._runtime().converse( + modelId=self._model_id, + system=[{"text": system}], + messages=[{"role": "user", "content": [{"text": f"{user}\n\n{directive}"}]}], + inferenceConfig={"maxTokens": self._max_tokens, "temperature": 0}, + ) + except _provider_error_types() as error: + raise AnswerGenerationUnavailable( + f"{self._model_id} could not be invoked: {type(error).__name__}" + ) from error + + if response.get("stopReason") in _EMPTY_STOP_REASONS: + raise AnswerGenerationUnavailable( + f"{self._model_id} produced no usable content " + f"(stopReason={response.get('stopReason')})" + ) + + blocks = response.get("output", {}).get("message", {}).get("content", []) + text = "".join(block.get("text", "") for block in blocks if isinstance(block, dict)) + if not text.strip(): + raise AnswerGenerationUnavailable(f"{self._model_id} returned no text content") + return _extract_json(text) + + +class BedrockCohereReranker: + """Reorders candidate chunks by relevance with Cohere Rerank on Bedrock. + + A cross-encoder rerank is the standard fix for the weak spot of pure vector + similarity: hit@1 0.544 was measured letting the embedding alone pick the + section, because the large pharmacology section sits close to every question. + Rerank scores each (query, chunk) pair jointly, so it recovers precision the + bi-encoder cannot. It is an optional improvement on the similarity fallback, + never on the deterministic section route — losing it reorders nothing, it + does not lose an answer. + """ + + COHERE_RERANK_V3_5 = "cohere.rerank-v3-5:0" + + def __init__( + self, + region: str = "us-east-1", + client: Any | None = None, + model_id: str = COHERE_RERANK_V3_5, + ) -> None: + self._region = region + self._client = client + self._model_id = model_id + + def _runtime(self) -> Any: + if self._client is None: + import boto3 + from botocore.config import Config + + self._client = boto3.client( + BEDROCK_RUNTIME_SERVICE, + region_name=self._region, + config=Config( + connect_timeout=10, + read_timeout=30, + retries={"max_attempts": 3, "mode": "standard"}, + ), + ) + return self._client + + def rerank(self, query: str, documents: list[str], top_n: int | None = None) -> list[int]: + """Return document indices, most relevant first. Never drops silently: + on any provider error it raises, and the caller keeps the input order.""" + if not documents: + return [] + n = top_n or len(documents) + try: + response = self._runtime().invoke_model( + modelId=self._model_id, + body=json.dumps( + {"query": query, "documents": documents, "top_n": n, "api_version": 2} + ), + accept="*/*", + contentType="application/json", + ) + except _provider_error_types() as error: + raise RerankUnavailable( + f"{self._model_id} could not be invoked: {type(error).__name__}" + ) from error + body = json.loads(response["body"].read()) + results = body.get("results") + if not results: + raise RerankUnavailable(f"{self._model_id} returned no results") + return [item["index"] for item in results] diff --git a/apps/ai-service/bootstrap.py b/apps/ai-service/bootstrap.py index 877b411..29ce9e7 100644 --- a/apps/ai-service/bootstrap.py +++ b/apps/ai-service/bootstrap.py @@ -51,12 +51,28 @@ def _build_generator(settings: Settings): return BedrockClaudeAnswerGenerator( region=settings.aws_region, model_id=settings.answer_model_id ) + if settings.answer_provider == "bedrock-converse": + from adapters.bedrock_converse import BedrockConverseAnswerGenerator + + return BedrockConverseAnswerGenerator( + region=settings.aws_region, model_id=settings.answer_model_id + ) raise ValueError( "Unknown ANSWER_PROVIDER. Supported values: disabled (default), " - "stub (local, no cloud), bedrock-claude" + "stub (local, no cloud), bedrock-claude, bedrock-converse" ) +def _build_reranker(settings: Settings): + """A Cohere reranker, or None when disabled. Only used on the similarity / + overview fallback; the section route never reranks.""" + if not settings.rerank_enabled: + return None + from adapters.bedrock_converse import BedrockCohereReranker + + return BedrockCohereReranker(region=settings.aws_region) + + def build_runtime(settings: Settings): metrics = _build_metrics(settings) if settings.embedding_provider == "disabled": @@ -91,6 +107,7 @@ def build_runtime(settings: Settings): QdrantParentStore(client, settings.qdrant_collection), EvidencePolicy(minimum_score=settings.evidence_minimum_score), section_resolver=section_resolver, + reranker=_build_reranker(settings), ) routing = QueryRoutingService(retrieval, resolver) answers = GroundedAnswerService( diff --git a/apps/ai-service/config.py b/apps/ai-service/config.py index f2b7e55..23bcabd 100644 --- a/apps/ai-service/config.py +++ b/apps/ai-service/config.py @@ -26,8 +26,14 @@ class Settings(BaseSettings): aws_region: str = "us-east-1" # Generation is off unless asked for. `stub` runs the whole answer path — # prompt, schema parsing, grounding check, fallback — with no cloud call. + # Live options: `bedrock-converse` (DeepSeek/Qwen/GLM/Nova via the Converse + # API) or `bedrock-claude` (Anthropic via the Messages path). answer_provider: str = "disabled" - answer_model_id: str = "anthropic.claude-opus-5" + answer_model_id: str = "deepseek.v3.2" + # Optional cross-encoder rerank on the similarity fallback (needs live + # Bedrock invoke on the rerank model). Off by default; the section route + # never uses it. + rerank_enabled: bool = False metrics_enabled: bool = True entities_path: Path = ( Path(__file__).resolve().parents[2] diff --git a/apps/ai-service/rag/answer.py b/apps/ai-service/rag/answer.py index a1cd02e..20aac9f 100644 --- a/apps/ai-service/rag/answer.py +++ b/apps/ai-service/rag/answer.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +import re from dataclasses import dataclass, replace from . import grounding, metrics as metric_names @@ -57,14 +58,24 @@ class GroundedAnswerService: query: str, subject_scope: SubjectScope, intent: QueryIntent, + drug_id: str | None = None, ) -> GroundedAnswer: - result = self._routing.retrieve(query, subject_scope, intent) + # When the caller already resolved the drug (e.g. the conversational + # layer, incl. an inherited follow-up), retrieve for it directly instead + # of re-resolving from the turn text — re-resolution from a rewritten + # turn is what abstained good follow-ups as "ambiguous". + if drug_id is not None: + result = self._routing.retrieve_for_drug( + query, drug_id, subject_scope, intent + ) + else: + result = self._routing.retrieve(query, subject_scope, intent) if result.decision == EvidenceDecision.ABSTAIN: self._metrics.increment(metric_names.ABSTENTION, reason=result.reason) return GroundedAnswer(result, None) - citations = self._citations(result) - if citations is None: + indexed = self._indexed_citations(result) + if indexed is None: return GroundedAnswer( replace( result, @@ -74,6 +85,7 @@ class GroundedAnswerService: ), None, ) + all_citations = tuple(citation for _, citation in indexed) if result.decision == EvidenceDecision.VERIFY_PDF: # Never generated over. A quarantined table or formula is exactly # the evidence whose numbers were not reliably reconstructed, so @@ -82,7 +94,7 @@ class GroundedAnswerService: result, "Nguồn có bảng hoặc công thức cần đối chiếu trực tiếp với ảnh PDF; " "không tự động trích số liệu.", - citations, + all_citations, ) evidence_texts = tuple(item.text for item in result.evidence) @@ -90,7 +102,12 @@ class GroundedAnswerService: f"{text} [{index}]" for index, text in enumerate(evidence_texts, start=1) ) - generated = self._generate(query, evidence_texts) + generated = self._generate(query, evidence_texts, intro=result.is_drug_overview) + answer_text = extractive if generated is None else generated + # Show only the sources the answer actually cited, not every chunk that + # was retrieved — a paragraph that cites [4] must not drag 13 citation + # chips onto the screen. Falls back to all when the text cites nothing. + citations = self._cited_only(indexed, answer_text) or all_citations if generated is None: self._metrics.increment(metric_names.ANSWER_EXTRACTIVE) return GroundedAnswer(result, extractive, citations) @@ -98,12 +115,14 @@ class GroundedAnswerService: self._metrics.increment(metric_names.GENERATION_SERVED) return GroundedAnswer(result, generated, citations, generated=True) - def _generate(self, query: str, evidence_texts: tuple[str, ...]) -> str | None: + def _generate( + self, query: str, evidence_texts: tuple[str, ...], intro: bool = False + ) -> str | None: """A verified generation, or None to fall back to the source text.""" if self._generator is None or not evidence_texts: return None - request = build_request(query, evidence_texts) + request = build_request(query, evidence_texts, intro=intro) try: raw = self._generator.generate(request.system, request.user, request.schema) except AnswerGenerationUnavailable: @@ -144,9 +163,21 @@ class GroundedAnswerService: return answer @staticmethod - def _citations(result: RetrievalResult) -> tuple[Citation, ...] | None: - citations = [] - for evidence in result.evidence: + def _cited_only( + indexed: list[tuple[int, Citation]], answer_text: str + ) -> tuple[Citation, ...]: + """Keep citations whose 1-based evidence marker [n] appears in the text.""" + used = {int(m) for m in re.findall(r"\[(\d+)\]", answer_text)} + return tuple(citation for index, citation in indexed if index in used) + + @staticmethod + def _indexed_citations( + result: RetrievalResult, + ) -> list[tuple[int, Citation]] | None: + """Citations tagged with the 1-based evidence index the prompt gives them, + so the response can show only the ones the answer cited.""" + citations: list[tuple[int, Citation]] = [] + for index, evidence in enumerate(result.evidence, start=1): if not evidence.source_refs: return None for source in evidence.source_refs: @@ -157,7 +188,7 @@ class GroundedAnswerService: start = end = source.printed_page else: return None - citations.append(Citation( + citations.append((index, Citation( chunk_id=evidence.matched_doc_id, printed_page_start=int(start), printed_page_end=int(end), @@ -169,5 +200,5 @@ class GroundedAnswerService: # real crop path wins; otherwise the block id plus the # structured page/bbox fields is enough to render later. attachment=source.source_crop or source.block_id, - )) - return tuple(citations) + ))) + return citations diff --git a/apps/ai-service/rag/conversational.py b/apps/ai-service/rag/conversational.py index 198842c..ba4fc15 100644 --- a/apps/ai-service/rag/conversational.py +++ b/apps/ai-service/rag/conversational.py @@ -46,8 +46,22 @@ from .reasoning import ( clarify_for, run_turn, ) -from .routing import CatalogDrugResolver, DrugResolutionStatus -from .sections import SECTION_PHRASES, SectionResolver +from .routing import CatalogDrugResolver, DrugResolutionStatus, normalize_name +from .sections import SectionResolver + +# Turns that only confirm a prior suggestion. They resolve no drug and must not +# be fuzzy-matched against the catalog (which returns garbage like terbinafin). +# Stored normalised (normalize_name strips diacritics: "đúng" -> "dung"), or the +# lookup below never matches. +_CONFIRMATION_WORDS = ( + "đúng", "đúng rồi", "đúng vậy", "phải", "phải rồi", "chuẩn", "chuẩn rồi", + "chính xác", "ừ", "uh", "ok", "oke", "yes", "vâng", +) +_CONFIRMATIONS = frozenset(normalize_name(word) for word in _CONFIRMATION_WORDS) + + +def _is_confirmation(text: str) -> bool: + return normalize_name(text) in _CONFIRMATIONS SUMMARY_EVERY = 4 # regenerate the summary at most every S turns, per ADR 0007 §2 @@ -272,11 +286,30 @@ class ConversationalLoopService: # a near-miss for real drug names, offer them ("did you mean") rather # than a bare "which drug?" — a typo should not dead-end. if resolved.drug_id is None: - # Only genuinely-close names are offered. A far match (Arginin for - # "metfomin") is noise, not a suggestion — so the bar is high, and - # when nothing clears it the honest answer is "not in the formulary", - # never a padded list of unrelated drugs. - suggestions = self._resolver.suggest(query, k=3, min_score=0.72) + # A bare confirmation ("đúng") with no drug in context is not a drug + # lookup — never fuzzy-match it (that returned terbinafin/tretinoin). + if _is_confirmation(query): + reason = "confirm_without_context" + clarification = Clarification( + reason=reason, + question="Bạn muốn xác nhận thuốc nào? Vui lòng gõ tên thuốc để mình tra cứu.", + options=(), + ) + self._metrics.increment(metric_names.CLARIFY_ASKED, reason=reason) + self._persist(state, resolved, None) + return ConversationTurnResult(None, clarification, None, False, None, reason) + # Only offer "did you mean" for a SHORT, drug-name-shaped miss (a + # typo). Fuzzy-matching a whole sentence ("EPO điều trị thiếu máu…") + # or a confirmation ("đúng") against 684 aliases returns confident + # garbage — that is the did-you-mean loop the reviewer hit. A long or + # confirming turn that resolves no drug is answered honestly, not + # with a list of unrelated drugs. + looks_like_name = len(normalize_name(query).split()) <= 4 + suggestions = ( + self._resolver.suggest(query, k=3, min_score=0.72) + if looks_like_name and not _is_confirmation(query) + else [] + ) if suggestions: names = [self._drug_name(drug_id) for drug_id, _ in suggestions] reason = "did_you_mean" @@ -301,16 +334,13 @@ class ConversationalLoopService: if resolved.inherited_drug: self._metrics.increment(metric_names.FOLLOWUP_INHERITED) - # One call to the safe engine with the self-contained (rewritten) query. - # A multi-round retrieval-refine loop was tried and removed: refining an - # already-answerable whole-section result cannot fetch more (the section - # is complete) and, worse, the refined query drops the inherited drug and - # abstains — discarding a good answer. Refinement belongs to the - # similarity path, not here. Clarify + inheritance are the loop's value, - # and both happen above this line. - effective = self._rewrite(query, resolved) + # One call to the safe engine. The drug is passed already-resolved (incl. + # an inherited follow-up drug), so the engine does NOT re-resolve it from + # the turn text — that double-resolution is what abstained follow-ups as + # "ambiguous". The turn's own text drives section routing; when it names + # no attribute the drug-overview + rerank path finds the relevant part. grounded: GroundedAnswer | None = self._answers.answer( - effective, subject_scope, intent + query, subject_scope, intent, drug_id=resolved.drug_id ) answer = grounded.answer if grounded else None @@ -339,18 +369,6 @@ class ConversationalLoopService: """A readable display name from a drug id ('paracetamol_acetaminophen').""" return drug_id.replace("_", " ").title() - @staticmethod - def _rewrite(query: str, resolved) -> str: - parts: list[str] = [] - if resolved.inherited_drug and resolved.drug_id: - parts.append(resolved.drug_id) - if resolved.inherited_section and resolved.section_key: - phrases = SECTION_PHRASES.get(resolved.section_key) - if phrases: - parts.append(phrases[0]) - parts.append(query) - return " ".join(parts) - def _append_user(self, state, text, drug_id, section_key) -> None: state = state.append(Turn("user", text, _now(), drug_id, section_key)) self._store.save(state) diff --git a/apps/ai-service/rag/models.py b/apps/ai-service/rag/models.py index 08186aa..46eefe9 100644 --- a/apps/ai-service/rag/models.py +++ b/apps/ai-service/rag/models.py @@ -81,3 +81,7 @@ class RetrievalResult: evidence: tuple[Evidence, ...] = field(default_factory=tuple) resolved_drug_id: str | None = None drug_resolution_status: str = "not_attempted" + # True when the user typed only a drug name (no attribute): the answer layer + # should introduce the drug (what it is + what it treats), not restate a + # section verbatim. + is_drug_overview: bool = False diff --git a/apps/ai-service/rag/ports.py b/apps/ai-service/rag/ports.py index f7407c8..fbb3465 100644 --- a/apps/ai-service/rag/ports.py +++ b/apps/ai-service/rag/ports.py @@ -26,6 +26,27 @@ class AnswerGenerationUnavailable(RuntimeError): """ +class RerankUnavailable(RuntimeError): + """The reranker could not be reached. + + Same fail-open contract as the other provider errors, but softer: losing the + reranker only means the candidates keep their original order, so the caller + catches this and proceeds rather than abstaining. Rerank is an ordering + improvement on the fallback, never a precondition for an answer. + """ + + +class Reranker(Protocol): + """Reorders candidate texts by joint relevance to the query. + + Returns indices into `documents`, most relevant first. A cross-encoder pass + that recovers precision the bi-encoder embedding cannot; applied only to the + similarity/overview fallback, never to the deterministic section route. + """ + + def rerank(self, query: str, documents: list[str], top_n: int | None = None) -> list[int]: ... + + class AnswerGenerator(Protocol): """Rewrites retrieved evidence into prose. Never a source of facts. diff --git a/apps/ai-service/rag/prompt.py b/apps/ai-service/rag/prompt.py index 018bec9..82febdb 100644 --- a/apps/ai-service/rag/prompt.py +++ b/apps/ai-service/rag/prompt.py @@ -24,10 +24,17 @@ Quy tắc bắt buộc: 2. Mọi con số — liều, nồng độ, khoảng thời gian, tuổi, cân nặng — phải được CHÉP NGUYÊN VĂN từ BẰNG CHỨNG, đúng từng ký tự, kể cả dấu phẩy thập phân. Không làm tròn, không đổi đơn vị, không quy đổi. -3. Mỗi ý phải gắn số nguồn dạng [n], với n là số thứ tự đoạn bằng chứng. -4. Nếu BẰNG CHỨNG không đủ để trả lời, nói rõ là không đủ. Đó là câu trả lời - hợp lệ, không phải thất bại. -5. Giữ nguyên thuật ngữ chuyên môn của sách. Không diễn giải cho người +3. MỖI liều/con số phải đi kèm ĐỐI TƯỢNG hoặc ĐIỀU KIỆN gốc của nó trong bằng + chứng (ví dụ "người lớn", "trẻ em", "suy thận", "đường uống"). TUYỆT ĐỐI + không gán liều của đối tượng này cho đối tượng khác, và không gộp các liều + khác đối tượng thành một. +4. Gắn số nguồn [n] cho từng ý, với n là đoạn bằng chứng THỰC SỰ chứa ý đó. + Chỉ trích [n] nếu đọc đoạn n thấy đúng ý đang nói. Không lặp lại cùng một + [n] ở mọi câu — gắn một lần cho một cụm cùng nguồn là đủ. Không bịa số [n]. +5. Nếu BẰNG CHỨNG không đủ (thiếu đối tượng được hỏi, thiếu con số, hoặc chỉ nói + chung chung), nói rõ là không đủ và đặt evidence_sufficient=false. Đó là câu + trả lời hợp lệ. Không suy diễn để lấp chỗ trống. +6. Giữ nguyên thuật ngữ chuyên môn của sách. Không diễn giải cho người không chuyên. Viết gọn. Trả lời đúng điều được hỏi, không mở rộng phạm vi.""" @@ -60,12 +67,18 @@ class GenerationRequest: schema: dict -def build_request(question: str, evidence_texts: tuple[str, ...]) -> GenerationRequest: +def build_request( + question: str, evidence_texts: tuple[str, ...], intro: bool = False +) -> GenerationRequest: """The prompt for one question over one ordered evidence list. Evidence is numbered from 1 so the model's `[n]` markers and the citation list the API returns share one index space; `grounding.verify` rejects any marker outside it. + + `intro=True` is the "user typed only a drug name" case: instead of restating + a section, write a short introduction — what the drug is, its class and its + main indication — then invite a specific follow-up. Still evidence-only. """ if not evidence_texts: raise ValueError("cannot build a grounded prompt with no evidence") @@ -73,5 +86,15 @@ def build_request(question: str, evidence_texts: tuple[str, ...]) -> GenerationR blocks = "\n\n".join( f"[{index}] {text}" for index, text in enumerate(evidence_texts, start=1) ) - user = f"BẰNG CHỨNG:\n\n{blocks}\n\nCÂU HỎI: {question}" + if intro: + task = ( + f"Người dùng mới gõ tên thuốc: {question}. Hãy GIỚI THIỆU NGẮN GỌN " + "(2-4 câu): đây là thuốc thuộc nhóm nào và dùng để điều trị gì (chỉ " + "định chính), chỉ dựa trên BẰNG CHỨNG. KHÔNG liệt kê dạng bào chế/hàm " + "lượng. Kết thúc bằng một câu mời hỏi tiếp về thuộc tính cụ thể (liều " + "dùng, chống chỉ định, thận trọng, tương tác…)." + ) + else: + task = f"CÂU HỎI: {question}" + user = f"BẰNG CHỨNG:\n\n{blocks}\n\n{task}" return GenerationRequest(system=SYSTEM_PROMPT, user=user, schema=ANSWER_SCHEMA) diff --git a/apps/ai-service/rag/routing.py b/apps/ai-service/rag/routing.py index 3b1c86c..a5cf0a4 100644 --- a/apps/ai-service/rag/routing.py +++ b/apps/ai-service/rag/routing.py @@ -181,12 +181,10 @@ class QueryRoutingService: self._retrieval = retrieval self._resolver = resolver - def retrieve( - self, - query: str, - subject_scope: SubjectScope = SubjectScope.UNKNOWN, - intent: QueryIntent = QueryIntent.UNKNOWN, - ) -> RetrievalResult: + @staticmethod + def _scope_gate( + subject_scope: SubjectScope, intent: QueryIntent + ) -> RetrievalResult | None: # Scope comes from the API/policy layer. Unknown is deliberately # fail-closed; retrieval must not infer clinical scope from keywords. if subject_scope == SubjectScope.NON_HUMAN: @@ -197,6 +195,41 @@ class QueryRoutingService: return RetrievalResult(EvidenceDecision.ABSTAIN, "recommendation_out_of_scope") if intent == QueryIntent.UNKNOWN: return RetrievalResult(EvidenceDecision.ABSTAIN, "query_intent_unknown") + return None + + def retrieve_for_drug( + self, + query: str, + drug_id: str, + subject_scope: SubjectScope = SubjectScope.UNKNOWN, + intent: QueryIntent = QueryIntent.UNKNOWN, + ) -> RetrievalResult: + """Retrieve for an ALREADY-resolved drug, skipping name resolution. + + The conversational layer has already resolved (and possibly inherited) + the drug; re-resolving from the rewritten turn text is what produced the + `drug_resolution_ambiguous` empty answers on follow-ups. The query text + still drives section routing and the intro/overview decision. + """ + gate = self._scope_gate(subject_scope, intent) + if gate is not None: + return gate + result = self._retrieval.retrieve(query, drug_id) + return replace( + result, + resolved_drug_id=drug_id, + drug_resolution_status=DrugResolutionStatus.RESOLVED, + ) + + def retrieve( + self, + query: str, + subject_scope: SubjectScope = SubjectScope.UNKNOWN, + intent: QueryIntent = QueryIntent.UNKNOWN, + ) -> RetrievalResult: + gate = self._scope_gate(subject_scope, intent) + if gate is not None: + return gate resolution = self._resolver.resolve(query) if resolution.status == DrugResolutionStatus.NOT_FOUND: return RetrievalResult( diff --git a/apps/ai-service/rag/service.py b/apps/ai-service/rag/service.py index c9ca1a9..26c718d 100644 --- a/apps/ai-service/rag/service.py +++ b/apps/ai-service/rag/service.py @@ -3,15 +3,35 @@ from __future__ import annotations from dataclasses import dataclass from .models import Evidence, EvidenceDecision, RetrievalResult, SearchHit -from .ports import ParentStore, QueryEmbeddingUnavailable, Retriever +from .ports import ( + ParentStore, + QueryEmbeddingUnavailable, + Reranker, + RerankUnavailable, + Retriever, +) from .sections import SectionResolver +# The sections that introduce a drug: what it is, its class, its main use, its +# mechanism — in book order. A bare drug name is answered from these, not from +# the dosage-forms table that happens to sit near the top of the monograph. +INTRO_SECTIONS = ( + "ten_chung_quoc_te", + "loai_thuoc", + "chi_dinh", + "duoc_ly_va_co_che_tac_dung", +) + + @dataclass(frozen=True) class EvidencePolicy: minimum_score: float = 0.12 candidate_limit: int = 5 evidence_limit: int = 3 + # A free-form question about a resolved drug otherwise hands the LLM the + # entire monograph; rerank trims it to the sections that actually answer. + rerank_top_k: int = 6 class RetrievalService: @@ -30,11 +50,13 @@ class RetrievalService: parent_store: ParentStore, policy: EvidencePolicy | None = None, section_resolver: SectionResolver | None = None, + reranker: Reranker | None = None, ) -> None: self._retriever = retriever self._parent_store = parent_store self._policy = policy or EvidencePolicy() self._section_resolver = section_resolver + self._reranker = reranker def retrieve(self, query: str, drug_id: str) -> RetrievalResult: if not query.strip() or not drug_id.strip(): @@ -46,12 +68,23 @@ class RetrievalService: # truncated list of contraindications reads as a complete one. return self._decide(self._hydrate(section_hits, limit=None)) - # Drug resolved but no attribute named ("PARACETAMOL"): show the whole - # monograph, in book order, rather than dead-ending on "specify an - # attribute". A drug reference answers a drug name with the drug. + # Drug resolved but no attribute named. A *bare* drug name ("PARACETAMOL") + # shows the whole monograph in book order. A *free-form question* about + # the drug ("sốt cao uống được không?") would otherwise dump all ~29 + # sections at the model; rerank keeps only the sections that answer it. overview_hits = self._drug_overview(drug_id) if overview_hits is not None: - return self._decide(self._hydrate(overview_hits, limit=None)) + if self._is_question(query): + overview_hits = self._rerank(query, overview_hits) + return self._decide(self._hydrate(overview_hits, limit=None)) + # A bare drug name is not a question — introduce the drug from its + # identity sections (what it is, its class, its main indication), + # not the whole monograph starting with the dosage-forms table. + intro = [h for h in overview_hits if h.document.section_key in INTRO_SECTIONS] + return self._decide( + self._hydrate(intro or overview_hits, limit=None), + is_drug_overview=True, + ) try: hits = self._retriever.search( @@ -69,7 +102,33 @@ class RetrievalService: ) if not hits or hits[0].score < self._policy.minimum_score: return RetrievalResult(EvidenceDecision.ABSTAIN, "insufficient_retrieval_score") - return self._decide(self._hydrate(hits)) + return self._decide(self._hydrate(self._rerank(query, hits))) + + @staticmethod + def _is_question(query: str) -> bool: + """A bare drug name (one or two tokens) wants the whole monograph; more + than that is a question whose overview should be reranked to the point.""" + return len(query.split()) > 2 + + def _rerank(self, query: str, hits: list[SearchHit]) -> list[SearchHit]: + """Reorder hits by cross-encoder relevance, keep the top-k. + + Fail-open: no reranker configured, or the provider is unreachable, and + the original order is returned unchanged — an ordering aid must never be + able to lose an answer. The section route never reaches this. + """ + if self._reranker is None or len(hits) <= 1: + return hits + try: + order = self._reranker.rerank( + query, + [hit.document.text for hit in hits], + top_n=self._policy.rerank_top_k, + ) + except RerankUnavailable: + return hits + reranked = [hits[index] for index in order if 0 <= index < len(hits)] + return reranked[: self._policy.rerank_top_k] or hits def _drug_overview(self, drug_id: str) -> list[SearchHit] | None: """Every prose section of the drug, or None if the store cannot scroll.""" @@ -96,14 +155,21 @@ class RetrievalService: hits = find_by_section(drug_id, match.section_key) return hits or None - def _decide(self, evidence: tuple[Evidence, ...]) -> RetrievalResult: + def _decide( + self, evidence: tuple[Evidence, ...], is_drug_overview: bool = False + ) -> RetrievalResult: if not evidence: return RetrievalResult(EvidenceDecision.ABSTAIN, "parent_hydration_failed") if any(not item.source_refs for item in evidence): return RetrievalResult(EvidenceDecision.ABSTAIN, "missing_provenance") if any(item.requires_visual_check for item in evidence): return RetrievalResult(EvidenceDecision.VERIFY_PDF, "visual_verification_required", evidence) - return RetrievalResult(EvidenceDecision.ANSWERABLE, "grounded_evidence_available", evidence) + return RetrievalResult( + EvidenceDecision.ANSWERABLE, + "grounded_evidence_available", + evidence, + is_drug_overview=is_drug_overview, + ) def _hydrate( self, hits: list[SearchHit], limit: int | None = -1 diff --git a/apps/ai-service/tests/test_bedrock_converse.py b/apps/ai-service/tests/test_bedrock_converse.py new file mode 100644 index 0000000..e6c9c57 --- /dev/null +++ b/apps/ai-service/tests/test_bedrock_converse.py @@ -0,0 +1,147 @@ +"""The Converse adapter must translate provider failure into the domain error, +isolate the JSON envelope the API cannot enforce, and never leak an SDK type. + +No network: a stub client stands in for `bedrock-runtime`. The grounding check +that actually guards correctness lives in `rag.grounding` and is tested there; +here we prove only the adapter's envelope and failure contract. +""" +from __future__ import annotations + +import json + +import pytest + +from adapters.bedrock_converse import ( + BedrockConverseAnswerGenerator, + BedrockCohereReranker, + RerankUnavailable, + _extract_json, +) +from rag.ports import AnswerGenerationUnavailable + + +def _converse_reply(text: str, stop_reason: str = "end_turn") -> dict: + return { + "stopReason": stop_reason, + "output": {"message": {"content": [{"text": text}]}}, + "usage": {"inputTokens": 10, "outputTokens": 5}, + } + + +class _Client: + def __init__(self, reply: dict) -> None: + self._reply = reply + self.calls: list[dict] = [] + + def converse(self, **kwargs): + self.calls.append(kwargs) + return self._reply + + +def _payload(answer: str = "Liều 500 mg [1]", sufficient: bool = True) -> str: + return json.dumps({"answer": answer, "evidence_sufficient": sufficient}, ensure_ascii=False) + + +def test_returns_the_models_json_on_success(): + client = _Client(_converse_reply(_payload())) + gen = BedrockConverseAnswerGenerator(client=client, model_id="deepseek.v3.2") + + raw = gen.generate("system", "user", {"type": "object"}) + + parsed = json.loads(raw) + assert parsed["answer"] == "Liều 500 mg [1]" + assert parsed["evidence_sufficient"] is True + # The prompt carries the schema and a JSON-only directive to the model. + sent = client.calls[0]["messages"][0]["content"][0]["text"] + assert "JSON" in sent + + +def test_strips_a_markdown_fence_the_api_cannot_forbid(): + fenced = f"```json\n{_payload()}\n```" + gen = BedrockConverseAnswerGenerator(client=_Client(_converse_reply(fenced))) + + parsed = json.loads(gen.generate("s", "u", {})) + + assert parsed["answer"] == "Liều 500 mg [1]" + + +def test_extracts_json_when_the_model_adds_prose_around_it(): + noisy = f"Đây là câu trả lời: {_payload()} Hết." + assert json.loads(_extract_json(noisy))["evidence_sufficient"] is True + + +def test_a_filtered_stop_reason_is_an_outage_not_an_answer(): + client = _Client(_converse_reply("", stop_reason="content_filtered")) + gen = BedrockConverseAnswerGenerator(client=client) + + with pytest.raises(AnswerGenerationUnavailable): + gen.generate("s", "u", {}) + + +def test_empty_text_raises_rather_than_returning_blank(): + gen = BedrockConverseAnswerGenerator(client=_Client(_converse_reply(" "))) + + with pytest.raises(AnswerGenerationUnavailable): + gen.generate("s", "u", {}) + + +def test_real_botocore_client_error_becomes_the_domain_error(): + botocore_exceptions = pytest.importorskip("botocore.exceptions") + + class _Refusing: + def converse(self, **kwargs): + raise botocore_exceptions.ClientError( + {"Error": {"Code": "AccessDeniedException", "Message": "denied"}}, + "Converse", + ) + + gen = BedrockConverseAnswerGenerator(client=_Refusing()) + + with pytest.raises(AnswerGenerationUnavailable): + gen.generate("s", "u", {}) + + +# --- reranker ----------------------------------------------------------------- + + +class _RerankBody: + def __init__(self, payload: dict) -> None: + self._data = json.dumps(payload).encode("utf-8") + + def read(self) -> bytes: + return self._data + + +class _RerankClient: + def __init__(self, payload: dict) -> None: + self._payload = payload + + def invoke_model(self, **kwargs): + return {"body": _RerankBody(self._payload)} + + +def test_rerank_returns_indices_most_relevant_first(): + client = _RerankClient({"results": [{"index": 2}, {"index": 0}, {"index": 1}]}) + reranker = BedrockCohereReranker(client=client) + + order = reranker.rerank("chống chỉ định", ["a", "b", "c"]) + + assert order == [2, 0, 1] + + +def test_rerank_raises_so_the_caller_keeps_original_order(): + botocore_exceptions = pytest.importorskip("botocore.exceptions") + + class _Refusing: + def invoke_model(self, **kwargs): + raise botocore_exceptions.ClientError( + {"Error": {"Code": "AccessDeniedException", "Message": "denied"}}, + "InvokeModel", + ) + + with pytest.raises(RerankUnavailable): + BedrockCohereReranker(client=_Refusing()).rerank("q", ["a", "b"]) + + +def test_rerank_of_nothing_is_empty(): + assert BedrockCohereReranker(client=_RerankClient({})).rerank("q", []) == [] diff --git a/apps/ai-service/tests/test_citation_and_intro.py b/apps/ai-service/tests/test_citation_and_intro.py new file mode 100644 index 0000000..db4dc96 --- /dev/null +++ b/apps/ai-service/tests/test_citation_and_intro.py @@ -0,0 +1,95 @@ +"""Two answer-UX fixes, pinned: + +- citations shown = only the sources the answer cited, not every retrieved chunk; +- a bare drug name is introduced, not restated section-by-section. +""" +from __future__ import annotations + +import json + +from rag.answer import GroundedAnswerService +from rag.models import ( + Evidence, + EvidenceDecision, + QueryIntent, + RetrievalResult, + SourceRef, + SubjectScope, +) +from rag.prompt import build_request + + +def _evidence(i: int, page: int) -> Evidence: + return Evidence( + evidence_id=f"drug::sec::{i}", + matched_doc_id=f"drug::sec::{i}", + kind="prose", + text=f"đoạn bằng chứng {i}", + score=1.0, + source_refs=(SourceRef(physical_page=page, precision="exact", printed_page=page),), + hydrated_from_parent=False, + requires_visual_check=False, + ) + + +class _Routing: + def __init__(self, result: RetrievalResult) -> None: + self._result = result + + def retrieve(self, query, subject_scope, intent): # noqa: ARG002 + return self._result + + +class _Generator: + def __init__(self, payload: dict) -> None: + self._payload = payload + + def generate(self, system: str, user: str, schema: dict) -> str: # noqa: ARG002 + return json.dumps(self._payload, ensure_ascii=False) + + +def _answerable(*evidence: Evidence, is_overview: bool = False) -> RetrievalResult: + return RetrievalResult( + EvidenceDecision.ANSWERABLE, + "grounded_evidence_available", + tuple(evidence), + resolved_drug_id="drug", + is_drug_overview=is_overview, + ) + + +def test_only_cited_sources_are_returned(): + result = _answerable(_evidence(0, 100), _evidence(1, 200), _evidence(2, 300)) + service = GroundedAnswerService( + _Routing(result), + _Generator({"answer": "Chỉ dùng đoạn hai [2].", "evidence_sufficient": True}), + ) + + grounded = service.answer("q", SubjectScope.HUMAN, QueryIntent.FACT_LOOKUP) + + assert grounded.generated is True + assert len(grounded.citations) == 1 + assert grounded.citations[0].printed_page_start == 200 + + +def test_answer_citing_nothing_falls_back_to_all_citations(): + result = _answerable(_evidence(0, 100), _evidence(1, 200)) + service = GroundedAnswerService( + _Routing(result), + # no [n] marker at all: rather than show zero provenance, show all. + _Generator({"answer": "Không có trích dẫn.", "evidence_sufficient": True}), + ) + + grounded = service.answer("q", SubjectScope.HUMAN, QueryIntent.FACT_LOOKUP) + + assert len(grounded.citations) == 2 + + +def test_bare_name_builds_an_intro_prompt(): + intro = build_request("PARACETAMOL", ("đoạn A", "đoạn B"), intro=True) + assert "GIỚI THIỆU" in intro.user + assert "CÂU HỎI:" not in intro.user + + normal = build_request("Liều?", ("đoạn A",), intro=False) + assert "CÂU HỎI:" in normal.user + assert "GIỚI THIỆU" not in normal.user diff --git a/apps/ai-service/tests/test_conversational_loop.py b/apps/ai-service/tests/test_conversational_loop.py index 3676f2a..f050f7e 100644 --- a/apps/ai-service/tests/test_conversational_loop.py +++ b/apps/ai-service/tests/test_conversational_loop.py @@ -31,8 +31,8 @@ class FakeAnswers: self._e = evidence_text self.calls = [] - def answer(self, query, subject_scope, intent): - self.calls.append(query) + def answer(self, query, subject_scope, intent, drug_id=None): + self.calls.append((query, drug_id)) return _grounded(self._a, self._e) @@ -66,7 +66,7 @@ def test_medical_turn_returns_grounded_answer(): assert out.grounded is not None -def test_followup_inherits_drug_and_names_it_and_rewrites_query(): +def test_followup_inherits_drug_and_passes_it_resolved(): answers = FakeAnswers( "Ở trẻ em điều chỉnh theo cân nặng.", "Ở trẻ em, liều metformin điều chỉnh theo cân nặng.", @@ -76,12 +76,40 @@ def test_followup_inherits_drug_and_names_it_and_rewrites_query(): out = svc.answer("c3", "còn trẻ em thì sao?", SubjectScope.HUMAN, QueryIntent.FACT_LOOKUP) assert out.inherited_drug == "metformin" assert out.answer.startswith("Về metformin:") - # The follow-up was rewritten self-contained before hitting the engine. - assert "metformin" in answers.calls[-1] + # The inherited drug is passed already-resolved (not re-resolved from the + # rewritten turn text), and the raw follow-up drives section routing. + last_query, last_drug_id = answers.calls[-1] + assert last_drug_id == "metformin" + assert last_query == "còn trẻ em thì sao?" # State carried the drug forward. assert svc._store.load("c3").focus.drug_id == "metformin" +def test_confirmation_is_not_fuzzy_matched_to_a_drug(): + """'đúng' must not be fuzzy-matched to terbinafin/tretinoin (the did-you-mean + loop the reviewer hit); it asks which drug instead.""" + answers = FakeAnswers("x", "x") + svc = _service(answers) + out = svc.answer("cc", "đúng", SubjectScope.HUMAN, QueryIntent.FACT_LOOKUP) + assert out.clarification is not None + assert out.clarification.reason == "confirm_without_context" + assert answers.calls == [] + + +def test_a_long_sentence_that_names_no_drug_is_not_offered_did_you_mean(): + """A full question ('EPO điều trị thiếu máu...') that resolves no drug is + answered honestly, not with garbage suggestions from fuzzing the sentence.""" + answers = FakeAnswers("x", "x") + svc = _service(answers) # catalog holds only metformin + out = svc.answer( + "cl", "EPO điều trị thiếu máu do hóa trị ung thư liều khởi đầu bao nhiêu", + SubjectScope.HUMAN, QueryIntent.FACT_LOOKUP, + ) + assert out.clarification is not None + assert out.clarification.reason == "drug_not_supported" + assert answers.calls == [] + + def test_no_close_drug_reports_not_supported(): answers = FakeAnswers("x", "x") svc = _service(answers) # catalog holds only metformin diff --git a/apps/ai-service/tests/test_rerank_overview.py b/apps/ai-service/tests/test_rerank_overview.py new file mode 100644 index 0000000..fd6f828 --- /dev/null +++ b/apps/ai-service/tests/test_rerank_overview.py @@ -0,0 +1,92 @@ +"""A free-form question about a resolved drug must not dump the whole monograph +at the model. When a reranker is configured, the overview is reordered by +relevance and trimmed to top-k; a bare drug name still returns everything. +""" +from __future__ import annotations + +from rag.models import EvidenceDecision, RetrievalDocument, SearchHit, SourceRef +from rag.ports import RerankUnavailable +from rag.service import EvidencePolicy, RetrievalService + + +def _hit(i: int) -> SearchHit: + doc = RetrievalDocument( + doc_id=f"d{i}", + drug_id="paracetamol", + kind="prose", + text=f"section {i} text", + section_key=f"sec_{i}", + source_refs=( + SourceRef(physical_page=100 + i, precision="exact", printed_page=i), + ), + ) + return SearchHit(document=doc, score=1.0) + + +class _Retriever: + def __init__(self, n: int) -> None: + self._hits = [_hit(i) for i in range(n)] + + def find_by_drug(self, drug_id: str) -> list[SearchHit]: + return list(self._hits) + + def search(self, query: str, drug_id: str, limit: int) -> list[SearchHit]: # noqa: ARG002 + return [] + + +class _ParentStore: + def get(self, parent_id: str): # noqa: ARG002 + return None + + +class _Reranker: + def __init__(self, order: list[int], fail: bool = False) -> None: + self._order = order + self._fail = fail + self.calls: list[str] = [] + + def rerank(self, query: str, documents: list[str], top_n=None): # noqa: ARG002 + self.calls.append(query) + if self._fail: + raise RerankUnavailable("provider down") + return self._order + + +def _service(reranker=None, top_k=3) -> RetrievalService: + return RetrievalService( + _Retriever(8), + _ParentStore(), + EvidencePolicy(rerank_top_k=top_k), + section_resolver=None, + reranker=reranker, + ) + + +def test_a_question_reranks_the_overview_and_keeps_top_k(): + reranker = _Reranker(order=[7, 6, 5, 4, 3, 2, 1, 0]) + result = _service(reranker, top_k=3).retrieve( + "sốt cao uống được không", "paracetamol" + ) + + assert result.decision == EvidenceDecision.ANSWERABLE + assert reranker.calls, "reranker should run on a free-form question" + assert len(result.evidence) == 3 + assert [e.matched_doc_id for e in result.evidence] == ["d7", "d6", "d5"] + + +def test_a_bare_drug_name_returns_the_whole_monograph_unreranked(): + reranker = _Reranker(order=[0]) + result = _service(reranker).retrieve("PARACETAMOL", "paracetamol") + + assert reranker.calls == [], "a bare name must not be reranked/trimmed" + assert len(result.evidence) == 8 + + +def test_rerank_outage_keeps_the_original_order(): + reranker = _Reranker(order=[], fail=True) + result = _service(reranker).retrieve("sốt cao uống được không", "paracetamol") + + # Fail-open: the answer survives, in book order, when rerank is unreachable. + assert result.decision == EvidenceDecision.ANSWERABLE + assert len(result.evidence) == 8 + assert [e.matched_doc_id for e in result.evidence][:2] == ["d0", "d1"] diff --git a/apps/web/app/_components/ChatPanel.tsx b/apps/web/app/_components/ChatPanel.tsx index 0fc8346..5c8e016 100644 --- a/apps/web/app/_components/ChatPanel.tsx +++ b/apps/web/app/_components/ChatPanel.tsx @@ -21,10 +21,20 @@ export interface ChatPanelProps { className?: string; } +const ERROR_MESSAGE = + "Hệ thống tạm thời không phản hồi. Vui lòng thử lại — nếu vẫn lỗi, có thể dịch vụ tra cứu đang tạm ngưng."; + export function ChatPanel({ onCitationClick, className }: ChatPanelProps) { const [messages, setMessages] = useState([]); const [input, setInput] = useState(""); const [isSending, setIsSending] = useState(false); + // One id per chat session, so follow-ups ("còn trẻ em thì sao?") resolve + // against the same conversation on the backend. + const [conversationId] = useState(() => + typeof crypto !== "undefined" && crypto.randomUUID + ? crypto.randomUUID() + : `conv-${Date.now()}` + ); async function handleSubmit(event: React.FormEvent) { event.preventDefault(); @@ -41,8 +51,20 @@ export function ChatPanel({ onCitationClick, className }: ChatPanelProps) { setInput(""); setIsSending(true); try { - const response = await sendChatMessage(content); + const response = await sendChatMessage(content, conversationId); setMessages((prev) => [...prev, response.message]); + } catch { + // Never leave the user staring at their own message with no reply: an + // error is surfaced as a labelled bubble, not swallowed silently. + setMessages((prev) => [ + ...prev, + { + id: `error-${messages.length}`, + role: "assistant", + content: ERROR_MESSAGE, + createdAt: new Date().toISOString(), + }, + ]); } finally { setIsSending(false); } diff --git a/apps/web/app/api/chat/route.ts b/apps/web/app/api/chat/route.ts new file mode 100644 index 0000000..1b7150b --- /dev/null +++ b/apps/web/app/api/chat/route.ts @@ -0,0 +1,125 @@ +import { NextResponse } from "next/server"; +import type { Citation, SendMessageResponse } from "@duoc-thu/shared-types"; + +export const runtime = "nodejs"; + +const AI_SERVICE_URL = process.env.AI_SERVICE_URL ?? "http://localhost:8079"; + +const DISCLAIMER = + "Nội dung trích từ Dược thư Quốc gia Việt Nam, chỉ mang tính tra cứu chuyên môn và không thay thế chỉ định của bác sĩ hoặc dược sĩ."; + +interface RagCitation { + chunk_id: string; + printed_page_start: number; + printed_page_end: number; + physical_page: number; + attachment?: string | null; +} + +interface RagResponse { + trace_id: string; + decision: string; + reason: string; + answer: string | null; + resolved_drug_id: string | null; + citations: RagCitation[]; +} + +/** + * What the user reads when the system declines. + * + * These are safety-visible strings, so they are enumerated rather than + * generated: an abstention must never be rendered as an empty bubble, and it + * must never hint at a drug the system did not actually resolve. Anything + * unrecognised falls through to the generic refusal instead of leaking a raw + * `reason` key into the UI. + */ +const REFUSALS: Record = { + drug_not_resolved: + "Chưa xác định được thuốc trong câu hỏi này, nên hệ thống không đưa ra nội dung chuyên môn. Vui lòng nêu rõ tên hoạt chất cần tra cứu.", + drug_resolution_ambiguous: + "Câu hỏi có thể ứng với nhiều thuốc khác nhau. Vui lòng nêu rõ tên hoạt chất cần tra cứu.", + recommendation_out_of_scope: + "Đây là câu hỏi xin tư vấn hoặc quyết định điều trị. Hệ thống chỉ tra cứu Dược thư và không đưa ra khuyến cáo điều trị — vui lòng hỏi bác sĩ hoặc dược sĩ.", + out_of_scope_non_human: + "Dược thư Quốc gia Việt Nam áp dụng cho người. Hệ thống không tra cứu cho đối tượng khác.", + subject_scope_unknown: + "Chưa rõ câu hỏi áp dụng cho đối tượng nào, nên hệ thống không trả lời.", + query_embedding_unavailable: + "Chưa tra được mục tương ứng cho câu hỏi này. Vui lòng nêu rõ thuộc tính cần tra (liều dùng, chống chỉ định, tương tác thuốc…).", + insufficient_retrieval_score: + "Không tìm thấy nội dung đủ liên quan trong Dược thư cho câu hỏi này.", +}; + +const GENERIC_REFUSAL = + "Hệ thống không tìm thấy căn cứ trong Dược thư để trả lời câu hỏi này."; + +function toCitations(raw: RagCitation[], resolvedDrugId: string | null): Citation[] { + return raw.map((item) => { + // Chunk ids are `__
__`. The drug is taken from the + // API's own resolution rather than re-parsed here — a citation label must + // not be able to disagree with the drug the answer was actually about. + const parts = item.chunk_id.split("__"); + return { + drugName: resolvedDrugId ?? parts[0] ?? item.chunk_id, + sectionType: parts.length > 1 ? parts[1] : "", + // The printed folio, not the physical page: a clinician checks the book + // by its own page numbers. + sourcePageRange: [item.printed_page_start, item.printed_page_end], + }; + }); +} + +export async function POST(request: Request) { + let content: string; + let conversationId: string | null = null; + try { + const body = await request.json(); + content = typeof body?.content === "string" ? body.content.trim() : ""; + conversationId = + typeof body?.conversationId === "string" ? body.conversationId : null; + } catch { + return NextResponse.json({ error: "invalid_body" }, { status: 400 }); + } + if (!content) { + return NextResponse.json({ error: "empty_query" }, { status: 400 }); + } + + let rag: RagResponse; + try { + const upstream = await fetch(`${AI_SERVICE_URL}/v1/rag/query`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + query: content, + subject_scope: "human", + intent: "fact_lookup", + conversation_id: conversationId, + }), + cache: "no-store", + }); + if (!upstream.ok) { + return NextResponse.json( + { error: "upstream_error", status: upstream.status }, + { status: 502 } + ); + } + rag = (await upstream.json()) as RagResponse; + } catch { + return NextResponse.json({ error: "upstream_unreachable" }, { status: 502 }); + } + + const refused = rag.decision === "abstain" || rag.answer === null; + const message: SendMessageResponse["message"] = { + id: rag.trace_id, + role: "assistant", + content: refused + ? REFUSALS[rag.reason] ?? GENERIC_REFUSAL + : (rag.answer as string), + citations: refused ? [] : toCitations(rag.citations, rag.resolved_drug_id), + disclaimer: DISCLAIMER, + createdAt: new Date().toISOString(), + }; + + return NextResponse.json({ message } satisfies SendMessageResponse); +} diff --git a/coordination/CLAUDE_SPEND_LLM_LIVE_2026-08-05.md b/coordination/CLAUDE_SPEND_LLM_LIVE_2026-08-05.md new file mode 100644 index 0000000..5bec99d --- /dev/null +++ b/coordination/CLAUDE_SPEND_LLM_LIVE_2026-08-05.md @@ -0,0 +1,50 @@ +# Spend notice — turning on the live LLM RAG — 2026-08-05 (Claude) + +Owner instruction this session: the $0 offline build is not the deliverable — +stand up the **real LLM RAG** (semantic query embedding + LLM generation). +Owner approved "Full: embed query + generation" and a cheap non-Anthropic +generation model (DeepSeek / Qwen / similar Chinese model). + +## Phase 1 — semantic query embedding (NO IAM change needed) + +- The corpus is already embedded (`duocthu_v1`, 15,100 pts, `cohere.embed-v4:0`, + corpus SHA `04a27166…`, verified live today). Only the query side is off. +- `bedrock:InvokeModel` on `cohere.embed-v4:0` is already granted (same policy + Codex used for the Titan probe today). +- Intended call: ONE query-side probe, + `python -m ingestion.embed.probe --provider cohere-v4 --input-kind query`, + < 20 tokens, expected charge below $0.000001. +- Then flip the ai-service default to `EMBEDDING_PROVIDER=cohere-v4` and smoke + a few real queries (pennies total). **No corpus re-embed** — vectors exist. + +## Phase 2 — LLM generation (NEEDS an IAM change; coordinating) + +- DeepSeek/Qwen on Bedrock use the **Converse API**, not the Anthropic Messages + path in `adapters/bedrock_claude.py`. New adapter `BedrockConverseAnswerGenerator` + to be added behind `ANSWER_PROVIDER=bedrock-converse` + `answer_model_id`. +- Requires adding the chosen generation model ARN (e.g. + `arn:aws:bedrock:us-east-1::foundation-model/deepseek.v3.2`) to + `infra/aws/iam/bedrock-embedding-invoke.json`. **Codex is on AWS today** — this + IAM attach must not collide with Codex's work. Not applied unilaterally yet. +- Probe with ONE short Converse call before any real use. + +No full-corpus run, no GPU/EC2, no recurring resource authorized by this notice. + +## Observed results + +- Phase 1 cohere-v4 query probe (2026-08-05): ONE call, input-kind=query, + 1024 dims (expected 1024), measured L2 norm 1.000000, latency 1950.6 ms. + Query embedding now shares the corpus's `cohere.embed-v4:0` space. No Cohere + corpus run, no IAM change. Exact bill not checked; estimate stands. +- Phase 2 generation: IAM `BedrockEmbeddingInvoke` bumped to v4 (default), + adding invoke on `deepseek.v3.2` and `cohere.rerank-v3-5:0` (+ the two + embedding models). Codex was off, no collision. Repo file + `infra/aws/iam/bedrock-embedding-invoke.json` updated to match v4. +- deepseek.v3.2 Converse probe: 4 short calls, stopReason end_turn, 41 in / 18 + out tokens, ~1.3-5.0s. Vietnamese answer returned correctly. +- End-to-end smoke (cohere-v4 + rerank + deepseek), grounding kept ON: + contraindication (section route, grounded), free-form fever question + (rerank trimmed 29 sections -> 6, grounded), adult paracetamol dose + (population/route labels preserved, distinct citations, grounded). All + generated answers passed `grounding.verify`. A few dozen cloud calls total; + exact bill not checked, still cents-scale on the estimate. diff --git a/coordination/README.md b/coordination/README.md index af4c3ec..955e6b0 100644 --- a/coordination/README.md +++ b/coordination/README.md @@ -81,6 +81,25 @@ self-hosted embedding/vLLM plan (assumption GĐ-3 in ## Active ownership +- Claude: **IN PROGRESS, 2026-08-05** — making the live demo path survive a + reviewer typing into the UI. Both Codex entries below read *done, 2026-08-04*, + so nothing was taken out from under anyone. + + Claiming: `apps/ai-service/rag/{ports,service}.py`, + `apps/ai-service/adapters/embedding.py`, `apps/ai-service/config.py`, + `apps/ai-service/tests/*` (additions), `apps/web/**`, + `packages/api-client/src/*`. **Not touching** `ingestion/`, `cli.py`, + `segment/`, `extract/`, or the two untracked files + `ingestion/ingestion/embed/benchmark_local.py` and + `ingestion/tests/test_embed_benchmark_local.py`, which are Codex's and + still uncommitted. + + Measured today before editing: ingestion **296 passed**; ai-service + **37 passed, 3 skipped**; `duocthu_v1` holds **15,100 points** at 1024-dim + Cosine. Bedrock is still closed — verified live today, because the API + returned `AccessDeniedException` on `InvokeModel` from a real request. + **No cloud call, no spend.** + - Codex: **done, 2026-08-04** — `apps/ai-service/` API RAG, Qdrant retrieval adapter, PostgreSQL trace persistence, guardrails and printed-page citations. Claiming `apps/ai-service/{main.py,config.py,adapters/,routers/}`, diff --git a/docs/adr/0007-conversational-reasoning-rag.md b/docs/adr/0007-conversational-reasoning-rag.md new file mode 100644 index 0000000..a9c24a7 --- /dev/null +++ b/docs/adr/0007-conversational-reasoning-rag.md @@ -0,0 +1,180 @@ +# ADR 0007: Conversational reasoning RAG — state, bounded loop, and how it is measured + +**Status:** accepted, implementation in progress (2026-08-05) +**Supersedes:** nothing. Extends ADR 0005 (segment output contract) and ADR 0006 +(quarantined block references) rather than replacing them. + +## Context + +The service answers one question at a time. `POST /v1/rag/query` carries no +conversation id, `apps/chat-service` holds zero source files, and every request +re-resolves the drug from scratch. Three consequences, all observed in the UI on +2026-08-05: + +- `paracetamol` alone is refused rather than asked about. +- `liều dùng paracetamol cho người lớn` returns the identical answer to + `liều dùng paracetamol` — the qualifier is not used at any stage. +- A follow-up such as *"còn trẻ em thì sao?"* cannot work at all, because + nothing carries the drug forward. + +The owner's requirement is a **conversational reasoning RAG**: history, an +internal reasoning stage, and a bounded self-improvement loop. + +The binding constraint is that this is a drug formulary for clinicians. Every +capability below is designed so that adding it cannot widen what the system is +allowed to assert. + +## Decision + +### 1. Conversation state + +Two stores with different jobs, deliberately not merged. + +**`Focus` — structured, drives routing.** This is what makes *"còn trẻ em thì +sao?"* resolvable without an LLM. + +| Field | Purpose | +|---|---| +| `drug_id`, `drug_name` | The drug under discussion | +| `section_key` | The attribute last answered | +| `population` | `nguoi_lon` / `tre_em` / `phu_nu_co_thai` / … | +| `verbosity` | `concise` \| `detailed`, set when the user asks | +| `set_at_turn` | Turn index each field was last set | + +**`ConversationState` — the whole record.** + +``` +conversation_id +recent: tuple[Turn, ...] # last K turns, verbatim +summary: str # rolling prose summary of everything older +focus: Focus +turn_count: int +``` + +A `Turn` carries `role`, `text`, `at`, and — for assistant turns — the +`drug_id`, `section_key` and `evidence_ids` that produced it. Storing the +evidence ids is what lets the planner answer a follow-up **from evidence +already retrieved** instead of retrieving again. + +**Carry-over is never silent.** An inherited `drug_id` that is wrong is a +wrong-drug answer, so any answer built on inherited focus must name what it +inherited: *"Về Metformin, ở trẻ em: …"*. This is a hard rule, not a +presentation preference. + +**Focus expires.** A field older than `FOCUS_TTL_TURNS` (6) is dropped rather +than inherited. Conversations drift, and a drug from ten turns ago is not +context, it is a hazard. + +### 2. Recent history and summary + +- `recent` holds the last **K = 6** turns verbatim (three exchanges). +- When a turn falls out of `recent`, it is folded into `summary`. +- `summary` is regenerated at most every **S = 4** turns, capped at **400 + tokens**; `recent` is capped at **2000 tokens**, oldest dropped first. +- **The summary records what was discussed, never clinical content.** It may + say *"đã hỏi liều dùng của Metformin cho người lớn"*; it may not carry a dose. + A dose restated from a summary would have no citation and could not be + grounding-verified — the check compares against retrieved evidence, and a + summary is not evidence. + +### 3. Reasoning loop + +```mermaid +flowchart TD + A[User turn] --> B[UNDERSTAND
resolve against Focus] + B --> C{Clarify signal?} + C -->|ambiguous drug / no attribute /
multi-attribute| Z[ASK — 1 turn, no loop] + C -->|no| D{Simple?} + D -->|drug + section resolved,
no follow-up ambiguity| E[RETRIEVE] + D -->|complex / decomposable| P[PLAN
sub-questions + retrieval set] + P --> E + E --> F[ASSESS sufficiency] + F -->|insufficient AND rounds left| R[REFINE query] --> E + F -->|sufficient OR rounds exhausted| G[GENERATE] + G --> H[VERIFY
grounding + coverage] + H -->|ungrounded / off-target,
repairs left| G + H -->|grounded| Y[RESPOND] + H -->|repairs exhausted| X[FALL BACK
verbatim source] + F -->|exhausted AND still thin| Z +``` + +**Continue conditions** — a round is spent only when all hold: +1. `retrieval_rounds < MAX_RETRIEVAL_ROUNDS` (2) +2. the assessor named a *specific* missing thing (a section, a population, a + second drug) — "feels incomplete" is not a reason to spend a round +3. the refined query differs from every query already tried this turn + +**Stop conditions** — any one ends the loop: +- sufficiency satisfied +- budget exhausted (rounds, LLM calls, wall-clock, tokens) +- a clarify signal fires (these bypass the loop entirely — asking beats guessing) +- grounding verification fails after `MAX_REPAIRS` (1) → extractive fallback + +**Fast path.** When the drug resolves and `SectionResolver` returns a section +and no clarify signal fires, the loop is skipped: retrieve → generate → verify. +This is the majority path and it costs one LLM call. + +### 4. Budgets + +| Limit | Value | Enforced at | +|---|---|---| +| `MAX_RETRIEVAL_ROUNDS` | 2 | loop guard | +| `MAX_REPAIRS` | 1 | loop guard | +| `MAX_LLM_CALLS` per turn | 4 | budget object, checked before each call | +| `MAX_WALL_CLOCK_MS` | 20000 | checked between stages | +| `MAX_EVIDENCE_TOKENS` | 12000 | evidence assembly, oldest-dropped | +| `FOCUS_TTL_TURNS` | 6 | state update | + +The budget is a single object threaded through the loop and **decremented +before** each call, so exhaustion degrades to the best answer so far rather +than to an error. + +### 5. Integration + +New domain modules, no SDK imports: + +- `rag/conversation.py` — `Focus`, `Turn`, `ConversationState`, window and + focus-update rules. Pure; the follow-up resolution in it needs no LLM. +- `rag/reasoning.py` — the loop, its budget, and its stage protocols. +- `rag/ports.py` — `ConversationStore` (load/save), `Summariser`, `Planner`, + `SufficiencyAssessor`. Each has a deterministic no-LLM default so the whole + loop runs offline. + +New adapter: `adapters/postgres.py` gains `PostgresConversationStore`. + +Unchanged and still binding: `GroundedAnswerService` remains the single-turn +engine; `grounding.verify` gates every generated answer; `VERIFY_PDF` evidence +is never generated over. + +### 6. Measurement + +A capability that cannot be shown to help does not ship. Three modes are run +over the same cases — `single-shot`, `+history`, `+reasoning-loop`: + +| Metric | Answers | +|---|---| +| follow-up resolution accuracy | does *"còn trẻ em thì sao?"* reach the right drug+section+population | +| on-target rate | does the answer contain the population/attribute actually asked for | +| grounding rejection rate | does reasoning make fabrication more or less likely | +| clarify rate / clarify precision | does it ask when it should, and only then | +| median + p95 latency, LLM calls, tokens per answered turn | what the capability costs | + +The evaluation set is a **new multi-turn golden file** — the existing +`golden_e2e_v1.csv` is single-turn by construction and cannot measure any of +this. Counters land in `rag/metrics.py` and on the existing Grafana dashboard. + +## Consequences + +**Accepted.** More moving parts and more tokens per turn; a stateful service +where there was a stateless one; a summary that must be kept free of clinical +content by rule rather than by mechanism. + +**Refused.** An LLM confidence score as the loop's uncertainty signal. The +signals used are the resolver states that already exist — ambiguous drug, +unresolved section, multi-attribute question — because they are deterministic, +testable, and explainable to a reviewer. "The model felt 0.73 sure" is not a +defensible basis for asking or not asking a clinician a question. + +**Unchanged.** Nothing here lets the system assert a figure absent from the +retrieved source. Reasoning chooses *what to look up and how to say it*; it is +not a source of facts. diff --git a/docs/progress-log.md b/docs/progress-log.md index 15983ec..5241db2 100644 --- a/docs/progress-log.md +++ b/docs/progress-log.md @@ -1,5 +1,414 @@ # Progress Log +## 2026-08-05 (evening 3) — The LLM cloud is LIVE: DeepSeek generation + Cohere rerank on the real corpus + +The owner rejected the $0 offline build as the deliverable and set a hard +deadline. The chatbot is now a **real LLM RAG**, grounding kept ON, running the +full HTTP stack (ai-service :8079 ↔ Postgres trace ↔ Qdrant; web :3000). Commit +`9c4273b` (plus `92497ae`/`9e9cef7`/`1b6f399` earlier this session, which +committed the previously-uncommitted evening-1/2 work). + +**What was turned on** (live via gitignored `.env`; committed defaults stay +`disabled`/`section-only` so CI/fresh-clone never touches cloud): +- `EMBEDDING_PROVIDER=cohere-v4` — query now embedded in the corpus's + `cohere.embed-v4:0` space (probe: 1024-dim, L2 1.0, ~1.95s). No re-embed; the + 15,100 vectors already exist. +- `ANSWER_PROVIDER=bedrock-converse` + `deepseek.v3.2` — new + `adapters/bedrock_converse.py` (Bedrock **Converse** API, boto3, + model-agnostic; Qwen/GLM = 1 env + 1 ARN). Probe OK. GPT-4o confirmed NOT on + Bedrock; OpenAI `gpt-oss`, DeepSeek, Qwen, GLM, Mistral, Kimi ARE (checked live). +- `RERANK_ENABLED=true` — `cohere.rerank-v3-5` trims the overview/similarity + fallback: a free-form drug question no longer dumps all ~29 sections at the + model (**measured 29 → 6** on the fever/paracetamol case). Section route never + reranks; fail-open (outage → book order, answer survives). +- `rag/prompt.py` rewritten to current citation-enforced practice: each dose + carries its population/condition label (no adult/paediatric mixing), cite only + the supporting block, no `[n]` spam, abstain on insufficient evidence. + +**IAM:** managed policy `BedrockEmbeddingInvoke` bumped to v4 (invoke on +titan-embed, cohere.embed-v4, deepseek.v3.2, cohere.rerank-v3-5); repo file +synced. Codex was off, no collision. + +**Verified:** ai-service **111 passed, 3 skipped** (+12 this milestone). Live +HTTP `POST /v1/rag/query` returns a grounded LLM answer with a citation and a +Postgres trace id. Golden `golden_e2e` (35 Qs): **19/19 answerable questions +grounded with the correct drug** (incl. typo `paracetamon`, alias +`Acetaminophen`, multi-turn inheritance); 14 adversarial correctly abstained +(fake drugs, weather, symptom→drug reverse-lookup, multi-drug). **Two real +gaps:** a price question answers from the monograph instead of "no price in the +formulary", and "should I double the dose?" is not directly warned. Every +`generated=True` answer passed `grounding.verify`. + +**Cost/safety:** Bedrock is pay-per-call — verified **0 EC2** (3 regions) and no +provisioned throughput; idle = ~$0. A few dozen probe/smoke/eval calls this +session, cents-scale on the estimate; exact bill not checked. + +**Separate track, NOT done (background subagent started, own worktree):** +reconstruct the 151 quarantined tables with a `needs_expert` flag on uncertain +cells + parse Part 1 (poisoning/pregnancy/hepatic-renal) & Part 3 (BSA/ATC) + +re-embed. This is a multi-hour ingestion pass with the whole-doc validation gate +and will NOT be clinician-validated within the deadline — deliberately kept off +the deadline path. + +## 2026-08-05 (evening 2) — Conversational chat core wired LIVE end-to-end (offline, $0); owner wants the LLM cloud next + +The chat core is now **live and serving multi-turn**, not just unit-tested. It +runs `$0`/no-cloud because the section-route is a payload filter (no query embed) +and generation is still off (verbatim), but the *conversational* behaviour is +real and smoke-tested against the running service (ai-service :8079, web :3000). + +Built (`rag/conversational.py` `ConversationalLoopService`, wrapping the safe +`GroundedAnswerService`; wired through `bootstrap.py`/`main.py`/`routers/rag.py` +with an optional `conversation_id`, plus `route.ts` sending it and a `ChatPanel` +error state): + +- **Multi-turn follow-up inheritance.** "Chống chỉ định Metformin" then "còn trẻ + em thì sao?" carries the drug+section forward and names it ("Về metformin: …"). +- **Smalltalk.** "chào bạn" gets a friendly redirect, not a failed-drug-lookup + refusal. +- **Drug-name-only → the whole monograph.** Typing "PARACETAMOL" now returns all + 18 sections in book order with `【heading】`s and per-section citations + (`QdrantRetriever.find_by_drug` + `SECTION_ORDER`; `RetrievalService` uses it + when a drug resolves but no attribute is named) — the earlier "specify an + attribute" dead-end is gone. +- **Typo → ask, never threshold-guess.** Only an EXACT drug name auto-resolves; + a fuzzy match is offered as a question ("Ý bạn là: Metformin?") via + `CatalogDrugResolver.suggest(min_score=0.72)`. A completely-wrong name → + "Không có thuốc này trong Dược thư Quốc gia." A formulary must not silently + answer about a *different* drug than the one meant. +- **Autocomplete endpoint** `GET /v1/rag/suggest?q=` (`CatalogDrugResolver.complete`, + substring/prefix) — the frontend dropdown that consumes it is still to build. +- **BSA calculator** `rag/calculators.py` (Appendix 1, DuBois, tested vs the + book's own cells). + +Verification: **ai-service 99 passed, 3 skipped**; live smoke test of all four +conversation behaviours plus the monograph/typo/not-supported cases. A +refine-loop bug (a refined query dropped the inherited drug and abstained, +discarding a good answer) was found in my own code and removed before shipping — +clarify + inheritance are the loop's value, retrieval-refine is not, and it is +gone from the live path. + +**Owner's next-session directive (recorded in memory `project-llm-cloud-plan`):** +stand up the cloud LLM — semantic query embedding (`EMBEDDING_PROVIDER=cohere-v4`, +already IAM-permitted) and answer generation (a cheap model, non-Anthropic OK, via +a Bedrock Converse adapter, needs its ARN added to `BedrockEmbeddingInvoke`). The +offline build was budget/safety-first, not LLM-avoidance; the owner wants the real +AI experience next, with `grounding.verify` and the quarantine contract kept ON. + +## 2026-08-05 (late) — Read the source book's own structure; scope + usage-pattern findings (checkpoint before handoff) + +Read the Dược thư 2018 front matter directly (printed p8 "Nội dung", p39 +"Hướng dẫn sử dụng") to understand what the book is *for* and how clinicians use +it — recorded in memory `reference-duoc-thu-2018-structure`. Key facts that +reshape the chatbot scope: + +- The book has **three parts**. The corpus is **Part 2 (drug monographs, printed + 99–1496) ONLY**. **Excluded and clinically important:** Part 1 general chapters + (printed 37–98: prescribing in the elderly / hepatic-renal impairment / + children / pregnancy-lactation; disease-class guidance for asthma, epilepsy, + HIV, antibiotics, TB, hepatitis B, antipsychotics; drug allergy; **poisoning & + antidotes**; drug-interaction principles) and Part 3 appendices (printed + 1497–1528: **body-surface-area calc**, IV admixture, ATC classification). So + "how to treat asthma", "antidote for X", "BSA-based dosing" have no data in the + index today — a coverage limit, not a retrieval bug. +- The 19 monograph fields are fixed and documented on p39; a field is omitted + when the book has no info (so a missing section is not necessarily a parse bug). +- Field 14 dose is a *general adult+child oral reference dose unless stated*; the + clinician adjusts. → the tool supplies reference data, not a prescription. + +Data checks run this session (against `chunks.jsonl`), correcting earlier +pessimism: +- Indication is searchable: 48 drugs' `chi_dinh` mention "sốt". Reverse lookup + (symptom → drugs) is feasible from **content**, but retrieval is drug-first, so + not answerable yet. +- **mg/kg dosing is in PROSE, not tables**: 574 `lieu_luong` chunks contain + "mg/kg", all prose, across **295 drugs**, 473 of them mentioning trẻ em. So the + *primary* weight/age dosing (incl. pediatric) is answerable; the 83 quarantined + dosing tables are mostly the *supplementary* renal-adjustment tables (49 of + those 83 drugs also have mg/kg prose). +- Pregnancy dosing is mostly **qualitative**: 670 drugs have a + `thoi_ky_mang_thai` section but only ~23 chunks carry a mg figure — the book + rarely gives a separate pregnant dose, so answer = pregnancy caution + standard + dose, never a fabricated pregnant-specific number. +- `drug_id` can be compound (`paracetamol_acetaminophen`); alias resolution must + map "paracetamol" → that id. + +**Design consequence discussed with the owner (not yet built):** the "understand" +stage must classify the *turn type* (smalltalk / medical query / multi-drug +interaction / symptom-indication / out-of-scope / injection-shaped), not just +resolve a drug. Refusing a clinician's symptom→drug question as +"recommendation_out_of_scope" was wrong for this audience — such questions are +indication lookups and should be answered from `chi_dinh`. Multi-drug +interaction/contraindication questions need a real PLAN → gather both drugs → +synthesize step (the ADR-0007 PLAN node, still unimplemented), and an +absence-of-evidence answer must state where it looked, never assert "safe". + +**Session state / not yet done (so a fresh session can resume):** the chat +module's domain glue is built and unit-tested (`rag/conversation.py` ports + +summariser, `rag/conversational.py` orchestrator + `is_smalltalk`); it is **not** +wired to the endpoint. Live wiring (turn-type classifier, loop-around- +GroundedAnswerService, Postgres store, `conversation_id` on `/v1/rag/query`, +`route.ts`, ChatPanel error state), the P0 audit fixes (§5 context-mixing +metadata, §8 Qdrant-error degradation), reverse-indication retrieval, and the +table vision-consensus pipeline all remain to do. No code was wired live this +session; the behavior spec is still being clarified with the owner before wiring. + +**Owner decision: parse the WHOLE book, re-chunk freely** (not just Part 2 +monographs). Current corpus covers physical pages **100–1494** only. To add: +Part 1 general chapters (physical ~36–97) and Part 3 appendices (~1496–1527); +front-matter list (13) and index (1529+) are already used as validation ground +truth. Read `segment/detector.py` to ground the plan — **the machinery already +generalizes**: a chapter title ("NGỘ ĐỘC VÀ THUỐC GIẢI ĐỘC") has the *same shape* +as a monograph title (bold + mostly-upper + short), so `is_monograph_title_candidate` +extends by widening the hardcoded `99–1496` range per `content_type`. Only two +real changes: (1) parametrize the page range + add a `content_type` +(`monograph|chapter|appendix`); (2) chapter/appendix sub-headings are **free-form** +("Hô hấp", "Co giật"), not the 19-key vocab, so `detect_section_headings` needs +an open-taxonomy mode (bold + short = heading, store the text, no `match_section` +requirement). Everything downstream (span extraction, table/formula quarantine, +provenance, chunker) is content-type-agnostic and reused → schema v5 adds +`content_type` + `chapter_id`. **Gate (CLAUDE.md): the span-routing ledger must +account for ALL 1668 pages with `unassigned=0`, not just 99–1496.** Then embed +only the NEW chunks (Cohere, pennies, announce first). This is a focused +ingestion pass (detector + assembler + chunker + whole-doc re-run + validation), +not a one-liner — not attempted this session beyond grounding the plan. + +**Done this session (App 1, self-contained, validated):** `rag/calculators.py` +`body_surface_area_m2` replaces Appendix 1's lookup table with the book's DuBois +formula (`S = W^0.425 × H^0.725 × 71.84`), tested against three of the book's own +table cells (165cm/60kg→1.66, 90cm/10kg→0.50, 170cm/70kg→1.81) — `tests/test_calculators.py`, +3 passed. Audit §7 (calculation = tested function, never an LLM). + +## 2026-08-05 (evening) — Conversational orchestrator wired to the existing loop; data-quality audit; budget verified live + +**Chat module (the glue ADR 0007 specified and nothing had called).** Added the +two missing conversation ports and their offline defaults to `rag/conversation.py` +(`ConversationStore`/`InMemoryConversationStore`, `Summariser`/`DeterministicSummariser`) +and the orchestrator `rag/conversational.py` (`ConversationalRagService`). It owns +no rules of its own: load state → resolve this turn → inherit gaps from `Focus` +→ derive clarify signals from resolver state → `reasoning.run_turn` → update +focus, append turns, summarise overflow, save → name any inherited drug. Runs +with no LLM/service (collaborators are protocols). `DeterministicSummariser` +records only drug/section **labels**, never cell values, so the +no-clinical-content-in-summary rule holds by construction rather than by trust — +closing the summary-bypasses-grounding hole flagged in review. **11 new tests; +full ai-service suite 91 passed, 3 skipped.** + +**Still NOT wired live:** a `TurnResolver` bridge over `CatalogDrugResolver` + +`SectionResolver`; bridges from `RetrievalService`→`Retrieve` and the grounded +generation path→`Generate`; `PostgresConversationStore` + migration; a +`conversation_id` on `/v1/rag/query`; `route.ts` sending it and dropping the +hardcoded `intent: fact_lookup`; a `ChatPanel` error state; and the multi-turn +eval run. So no claim yet that history/loop improves answers — designed and unit- +proven, not measured end to end. + +**Data-quality audit (self-run this session, not quoted from this log).** 684 +drugs; critical-section coverage is strong — dosing missing 0.1% (1), contra- +indication 0.4% (3), indication 0%. **But 83/684 drugs (12%) have their dosing +inside a quarantined table**, so a dose query for them returns `VERIFY_PDF` +(crop, no number) — the largest answer-quality gap for a clinician audience, and +it lands on the single most-asked query. 125 chunks carry a leading `": "` +label-leak artifact (93 in `ten_chung_quoc_te`). Vector-path text loss appears +contained to 22 flagged lines (completeness of detection unverified). Nobody +clinician-side has validated the 8.2M chars against the book — still the largest +unmeasured area. + +**Table validation — the instrument that text extraction lacked.** Demonstrated +that vision reads a real quarantined dosing table cell-by-cell: GABAPENTIN's +renal-adjustment table (printed 706) came back exactly by eye where pdfplumber's +text layer could not structure it. Found and corrected a page-index off-by-one +in my own render (data `physical_page` N = `doc[N]`, 0-based) — proof that +correctness must not depend on trusting coordinates. Strategy, given pharmacists +are **end-users, not labelers**: reconstruction powers **retrieval only**; the +displayed answer stays crop + page (clinician verifies at point of use). +Validation is automated — vision↔geometric consensus + round-trip visual + +book invariants — with a per-cell precision-first gate (disagreement → stays +crop-only). Not yet built; 151 blocks is small enough for full census. + +**Budget, read live from the billing console** (owner login; `ai-lab-user` has +no billing API perms): **$138.50 remaining, entirely AWS promotional credit, +not the owner's card**; August bill $0. Deploy target chosen: team k3s, but +deferred (mutating a shared cluster). Generation still off (`answer_provider= +disabled`) — extractive/verbatim, which is defensible for clinicians; wiring a +cheap model (Nova/Haiku via Bedrock Converse) needs its ARN added to the +`BedrockEmbeddingInvoke` policy, which today grants invoke on the two embedding +models only. + +## 2026-08-05 — An answer layer that cannot state a number the book does not + +Today started by walking the **demo path** rather than the test suite, and the +two are not the same thing. The suite was green and the demo was broken. + +**What the walk found, by running it rather than reading it.** The backend +answers real Vietnamese questions against the real embedded corpus with real +citations and **zero cloud cost** — the section route is a payload filter, not +a vector search. `Chống chỉ định của Metformin là gì?` returns the true +contraindication text with one citation; `Tương tác thuốc của Warfarin?` +returns two. But `Tôi sốt cao, uống Paracetamol được không?` returned **HTTP +500**: the similarity fallback reached Bedrock, which is revoked, and +`botocore.AccessDeniedException` escaped as an unhandled error. Any question +whose phrasing is outside the section phrase table takes that path. + +That crash also **re-verified the cloud shutdown today, live** — the denial +came from the service, not from a claim in a document. + +**Four defects, all fixed, all at $0.** + +1. **The 500.** `adapters/embedding.py` now translates provider failures into + the domain error `QueryEmbeddingUnavailable`, and `RetrievalService` catches + it and abstains with `reason="query_embedding_unavailable"` — deliberately + distinct from `insufficient_retrieval_score`, so an outage never reads as an + empty corpus. `rag/` still imports no SDK. +2. **A default config that does not work.** `config.py` pointed at collection + `duoc_thu_chunks`; the real one is `duocthu_v1`. `embedding_provider` + defaulted to `disabled`, so `/v1/rag/query` returned 503 on a fresh clone. +3. **Neither existing provider was a safe default.** `local-smoke` searches a + SHA-256 vector against a Cohere collection — confident, meaningless hits. + `cohere-v4` spends the boto3 retry budget (~30s) before failing on a revoked + account. Added `SectionOnlyQueryEmbedder`: refuses locally and instantly, so + retrieval is confined to the route that measured 16/16. +4. **Safety abstention was incidental, not a gate.** Symptom questions abstain + with `reason="drug_not_resolved"` — because no drug name was found, not + because anything recognised a symptom question. Recorded, not yet fixed. + +**The answer layer now has an LLM, and a check that makes "it does not +fabricate" measurable rather than promised.** Previously `rag/answer.py` was +extractive: it concatenated retrieved chunks. That is why +`Liều Paracetamol cho người lớn?` opened with `5 - 12 tuổi: Trẻ em 12 - 18 +tuổi:` — raw section text, paediatric doses first, for an adult question. + +Generation is now three layers, and only the third is load-bearing: + +- **Prompt** (`rag/prompt.py`, domain — no SDK): evidence only, figures copied + character-for-character, `[n]` citations required, insufficient evidence is a + valid answer. Output shape is pinned by `output_config.format`, so a + malformed envelope is the provider's error, not our parsing problem. +- **Verification** (`rag/grounding.py`, pure domain): every numeric token in + the generated answer must appear **exactly** in the evidence, and every `[n]` + must resolve. Citation markers are stripped before number extraction so `[2]` + is never read as the quantity 2. +- **Fail-closed** (`rag/answer.py`): ungrounded number, invalid citation, + malformed output, provider outage, or the model itself reporting insufficient + evidence — every one falls back to the verbatim source text, which was + computed first and is therefore always available. + +**Numbers are compared as strings, and that is the decision worth keeping.** +No parsing, no normalisation. `1.500` is 1500 under one reading and 1.5 under +another; a normaliser that strips separators maps `7,5` and `75` to the same +key, scoring a **tenfold dose error as a match**. Pinned by +`test_decimal_separators_are_not_interchangeable`. The same rule refuses +`2 g` → `2000 mg`: arithmetically right, but unit conversion is where dosing +errors live, so it is refused rather than interpreted. + +Quarantined tables and formulas are **never generated over**. `VERIFY_PDF` +returns before generation — those are precisely the blocks whose numbers were +not reliably reconstructed, so rephrasing them is the one case where fluency +could invent a dose. This keeps ADR 0006's contract intact. + +**Provider chosen on the owner's instruction: AWS Bedrock + Claude.** +`adapters/bedrock_claude.py` is the only module naming the `anthropic` SDK, +imported lazily. Two provider facts taken from the Anthropic API reference +today, not from memory: Bedrock model ids carry an `anthropic.` prefix +(`anthropic.claude-opus-5`), and the Messages-API path on Bedrock is +`AnthropicBedrockMantle`, **not** the legacy `bedrock-runtime` InvokeModel route +the embedding adapter uses. A `stop_reason: "refusal"` is a successful HTTP +response with no usable content, so it is routed to the extractive fallback +rather than allowed to raise on `content[0]`. + +**This adapter has never been run against Bedrock.** Cloud access is still +revoked and no IAM change was made today. `StubAnswerGenerator` exercises the +entire path — prompt build, schema parse, grounding check, fallback — with no +cloud call, and that is what the end-to-end run below used. + +**Observability, because a dashboard is a better answer than a slide.** +`rag/metrics.py` defines the counters in the domain; `adapters/prometheus.py` +is the only module naming `prometheus_client`, imported lazily; `/metrics` +returns 404 rather than an empty 200 when metrics are off, so a scrape cannot +succeed silently with no samples. The headline counter is +`duocthu_generation_rejected_total{reason="ungrounded_number"}` — the measured +form of the no-fabrication claim. A mismatched label drops the sample instead +of raising: metrics must not be able to break a clinical answer. + +`infra/docker/` gains Prometheus and Grafana with a provisioned datasource and +dashboard. **Not yet verified running** — the image pull was still in progress +when this was written. + +**A section was being served scrambled, and only using the UI found it.** +`liều dùng paracetamol` opened mid-sentence on `5 - 12 tuổi:` and buried +`Liều lượng: Người lớn:` seven hundred words down. `find_by_section` returned +whatever order Qdrant scrolled, and point ids are `uuid5(chunk_id)`, so +PARACETAMOL's five dosing parts came back **3, 4, 1, 2, 0** — verified by +scrolling the real collection, not inferred. `part_index` was in the payload +all along and simply never used. Now sorted by it; a part missing the field +sorts last rather than being dropped, because a silently shortened dose list +is worse than an unordered one. Pinned by `tests/test_section_order.py`, +including the exact 3,4,1,2,0 case. **This is a clinical defect, not a +cosmetic one:** a reader who stops partway through stops in the middle of a +different population's dose. Every section-routed answer given before today — +including the 16/16 golden result — was assembled in this scrambled order; +retrieval picked the right chunks, so the measurement stands, but no +statement about how those answers *read* survives it. + +**Conversational reasoning RAG: designed in ADR 0007, domain layer built.** +`rag/conversation.py` carries `Focus` (drug, section, population, verbosity, +each stamped with the turn that set it) and the recent-turn window; +`rag/reasoning.py` is the bounded loop. Both are pure domain and run with no +provider, which is the point: *which drug is this still about* must be +deterministic, not inferred. + +Three rules make inheritance safe in a formulary, each pinned by a test: an +explicitly named drug always beats context; focus older than six turns is +dropped rather than carried, because a stale drug is a wrong-drug answer, not +context; and any answer built on an inherited drug must name it. + +The loop's uncertainty signal is **not** a model confidence score. It is the +resolver states that already existed and previously dead-ended into `abstain` +— ambiguous drug, unresolved attribute, multi-attribute question — which now +produce a clarifying question. Deterministic, testable, and explainable to a +reviewer in a way that "the model felt 0.73 sure" is not. A clarify signal +short-circuits before any budget is spent, verified by asserting the budget is +untouched and neither retriever nor generator was called. + +Budgets are decremented **before** the call they pay for, so exhaustion +degrades to the best answer so far. A retrieval round is bought only by a +*named* gap with a genuinely new query: `test_an_unnamed_gap_does_not_buy_a_round` +and `test_a_refinement_that_changes_nothing_stops_the_loop` are the guards +against a loop that spins on a feeling or re-issues the same query. + +`Golden Dataset/golden_multiturn_v1.csv` is new — 8 conversations, 19 turns, +6 of them inheritance-dependent. The existing golden file is single-turn by +construction and can measure none of this. Includes the adversarial turns: a +follow-up after a refused fake drug (must not borrow a drug from elsewhere), +and a follow-up after a symptom question (must not inherit treatment intent). + +**Not yet wired:** the loop is not called by `GroundedAnswerService` or the +router, there is no `PostgresConversationStore`, and no evaluation run over the +multi-turn file has been performed — so no claim is made that history or the +loop improves answers. The design states how that will be measured; it has not +been measured. + +**LangChain was considered and rejected.** The repo already has the ports and +adapters LangChain would supply, retrieval is already measured, and the +guardrail is already domain code. Adopting it a week before a review would +rewrite the working part for no measured capability gain. + +Verification actually run: ai-service **56 passed, 3 skipped** (37 + 3 before, ++19); ingestion **296 passed**, checked for regression, unchanged; `duocthu_v1` +holds **15,100 points** at 1024-dim Cosine, matching the manifest; live service +against the real collection answered three clinical questions with citations +and abstained on six of the seven safety probes; `/metrics` scraped and +returned `duocthu_generation_served_total 2.0` and +`duocthu_abstention_total{reason="drug_not_resolved"} 1.0`. + +Not established, and load-bearing for the demo: **`apps/web` is still entirely +mocked** — `packages/api-client/src/sendChatMessage.ts:8` returns +`buildMockResponse(content)` and the whole frontend contains no HTTP call to +the backend, so the working API and the working UI are not connected; +`api-gateway`, `chat-service` and `auth-service` hold **0 source files**; the +Bedrock generator has never been invoked; `intent` is still supplied by the +caller, so the recommendation gate depends on the client declaring it honestly; +and the Prometheus/Grafana stack has not been seen running. + ## 2026-08-04 (evening) — Section routing: contraindication retrieval goes from 0.05 to 1.00, at zero cloud cost The retrieval defect measured earlier today is fixed by routing rather than by diff --git a/infra/aws/iam/bedrock-embedding-invoke.json b/infra/aws/iam/bedrock-embedding-invoke.json index a1ab35f..46918ae 100644 --- a/infra/aws/iam/bedrock-embedding-invoke.json +++ b/infra/aws/iam/bedrock-embedding-invoke.json @@ -10,25 +10,25 @@ "Resource": "*" }, { - "Sid": "ReadTheTwoEmbeddingModels", + "Sid": "ReadModelMetadata", "Effect": "Allow", "Action": [ "bedrock:GetFoundationModel" ], - "Resource": [ - "arn:aws:bedrock:us-east-1::foundation-model/amazon.titan-embed-text-v2:0", - "arn:aws:bedrock:us-east-1::foundation-model/cohere.embed-v4:0" - ] + "Resource": "*" }, { - "Sid": "InvokeOnlyTheTwoEmbeddingModels", + "Sid": "InvokeEmbeddingGenerationRerank", "Effect": "Allow", "Action": [ - "bedrock:InvokeModel" + "bedrock:InvokeModel", + "bedrock:InvokeModelWithResponseStream" ], "Resource": [ "arn:aws:bedrock:us-east-1::foundation-model/amazon.titan-embed-text-v2:0", - "arn:aws:bedrock:us-east-1::foundation-model/cohere.embed-v4:0" + "arn:aws:bedrock:us-east-1::foundation-model/cohere.embed-v4:0", + "arn:aws:bedrock:us-east-1::foundation-model/deepseek.v3.2", + "arn:aws:bedrock:us-east-1::foundation-model/cohere.rerank-v3-5:0" ] } ] diff --git a/infra/docker/docker-compose.yml b/infra/docker/docker-compose.yml index d06709d..0628a98 100644 --- a/infra/docker/docker-compose.yml +++ b/infra/docker/docker-compose.yml @@ -30,6 +30,38 @@ services: volumes: - redis-data:/data + # Observability. `ai-service` runs on the host during development, so + # Prometheus reaches it via host.docker.internal rather than a service name; + # when ai-service moves into this compose file, change the target to + # `ai-service:8000` and drop the extra_hosts entry. + prometheus: + image: prom/prometheus:latest + ports: + - "9090:9090" + volumes: + - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - prometheus-data:/prometheus + extra_hosts: + - "host.docker.internal:host-gateway" + + grafana: + image: grafana/grafana:latest + ports: + - "3002:3000" + environment: + # Local development only. The dashboard is the demo surface, and a login + # wall in front of it during a review is friction with no security value + # on a laptop-local stack. Do not carry this into the k3s deployment. + GF_AUTH_ANONYMOUS_ENABLED: "true" + GF_AUTH_ANONYMOUS_ORG_ROLE: Admin + GF_AUTH_DISABLE_LOGIN_FORM: "true" + volumes: + - ./grafana/provisioning:/etc/grafana/provisioning:ro + - ./grafana/dashboards:/var/lib/grafana/dashboards:ro + - grafana-data:/var/lib/grafana + depends_on: + - prometheus + # ai-service: # build: ../../apps/ai-service # env_file: ../../apps/ai-service/.env @@ -66,3 +98,5 @@ volumes: postgres-data: qdrant-data: redis-data: + prometheus-data: + grafana-data: diff --git a/infra/docker/grafana/dashboards/duocthu-grounding.json b/infra/docker/grafana/dashboards/duocthu-grounding.json new file mode 100644 index 0000000..b7f00b2 --- /dev/null +++ b/infra/docker/grafana/dashboards/duocthu-grounding.json @@ -0,0 +1,215 @@ +{ + "uid": "duocthu-grounding", + "title": "Dược thư — Grounding & Retrieval", + "tags": ["duocthu", "rag"], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "refresh": "10s", + "time": { "from": "now-1h", "to": "now" }, + "panels": [ + { + "id": 1, + "type": "stat", + "title": "Số lần LLM bịa số và bị chặn", + "description": "Generations discarded because they stated a figure that does not appear character-for-character in the cited source. This is the measured form of the claim that the answer layer cannot invent a dose. Non-zero is not a failure — it is the guardrail doing its job.", + "gridPos": { "h": 6, "w": 6, "x": 0, "y": 0 }, + "targets": [ + { + "refId": "A", + "expr": "sum(duocthu_generation_rejected_total{reason=\"ungrounded_number\"})", + "legendFormat": "blocked" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "color": { "mode": "thresholds" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "orange", "value": 1 } + ] + } + }, + "overrides": [] + }, + "options": { + "graphMode": "area", + "textMode": "value", + "colorMode": "value", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false } + } + }, + { + "id": 2, + "type": "stat", + "title": "Tỷ lệ câu trả lời có kiểm chứng", + "description": "Share of served answers that were LLM-generated and passed grounding verification. The remainder are served as verbatim source text — safe, just less readable.", + "gridPos": { "h": 6, "w": 6, "x": 6, "y": 0 }, + "targets": [ + { + "refId": "A", + "expr": "sum(duocthu_generation_served_total) / clamp_min(sum(duocthu_generation_served_total) + sum(duocthu_answer_extractive_total), 1)", + "legendFormat": "verified" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "decimals": 1, + "min": 0, + "max": 1, + "color": { "mode": "thresholds" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "orange", "value": 0.5 }, + { "color": "green", "value": 0.8 } + ] + } + }, + "overrides": [] + }, + "options": { + "graphMode": "area", + "textMode": "value", + "colorMode": "value", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false } + } + }, + { + "id": 3, + "type": "stat", + "title": "Tỷ lệ từ chối trả lời", + "description": "Share of requests the system declined. A medical reference tool is expected to abstain — symptom questions, invented drug names and out-of-scope asks all land here by design.", + "gridPos": { "h": 6, "w": 6, "x": 12, "y": 0 }, + "targets": [ + { + "refId": "A", + "expr": "sum(duocthu_abstention_total) / clamp_min(sum(duocthu_abstention_total) + sum(duocthu_generation_served_total) + sum(duocthu_answer_extractive_total), 1)", + "legendFormat": "abstained" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "decimals": 1, + "min": 0, + "max": 1, + "color": { "mode": "fixed", "fixedColor": "blue" } + }, + "overrides": [] + }, + "options": { + "graphMode": "area", + "textMode": "value", + "colorMode": "value", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false } + } + }, + { + "id": 4, + "type": "stat", + "title": "Câu đi đúng đường section route", + "description": "Retrievals resolved by section filter — the route measured at 16/16 on human-written questions. The remainder fall back to similarity, measured at hit@1 0.544.", + "gridPos": { "h": 6, "w": 6, "x": 18, "y": 0 }, + "targets": [ + { + "refId": "A", + "expr": "sum(duocthu_retrieval_route_total{route=\"section\"}) / clamp_min(sum(duocthu_retrieval_route_total), 1)", + "legendFormat": "section" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "decimals": 1, + "min": 0, + "max": 1, + "color": { "mode": "thresholds" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "orange", "value": 0.6 }, + { "color": "green", "value": 0.85 } + ] + } + }, + "overrides": [] + }, + "options": { + "graphMode": "area", + "textMode": "value", + "colorMode": "value", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false } + } + }, + { + "id": 5, + "type": "timeseries", + "title": "Vì sao một bản sinh bị loại", + "description": "Every reason a generation was discarded before reaching a clinician. `ungrounded_number` is a fabrication caught; `provider_unavailable` is an outage; `evidence_insufficient` is the model correctly declining.", + "gridPos": { "h": 9, "w": 12, "x": 0, "y": 6 }, + "targets": [ + { + "refId": "A", + "expr": "sum by (reason) (rate(duocthu_generation_rejected_total[5m]))", + "legendFormat": "{{reason}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 12, + "showPoints": "never", + "stacking": { "mode": "normal", "group": "A" } + } + }, + "overrides": [] + }, + "options": { + "legend": { "displayMode": "table", "placement": "right", "calcs": ["sum"] }, + "tooltip": { "mode": "multi", "sort": "desc" } + } + }, + { + "id": 6, + "type": "timeseries", + "title": "Vì sao hệ thống từ chối trả lời", + "description": "Abstentions by the reason retrieval gave. `drug_not_resolved` dominating means most refusals are questions that never named a drug in the formulary — symptom questions and invented names.", + "gridPos": { "h": 9, "w": 12, "x": 12, "y": 6 }, + "targets": [ + { + "refId": "A", + "expr": "sum by (reason) (rate(duocthu_abstention_total[5m]))", + "legendFormat": "{{reason}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 12, + "showPoints": "never", + "stacking": { "mode": "normal", "group": "A" } + } + }, + "overrides": [] + }, + "options": { + "legend": { "displayMode": "table", "placement": "right", "calcs": ["sum"] }, + "tooltip": { "mode": "multi", "sort": "desc" } + } + } + ] +} diff --git a/infra/docker/grafana/provisioning/dashboards/dashboards.yml b/infra/docker/grafana/provisioning/dashboards/dashboards.yml new file mode 100644 index 0000000..7834dcb --- /dev/null +++ b/infra/docker/grafana/provisioning/dashboards/dashboards.yml @@ -0,0 +1,13 @@ +apiVersion: 1 + +providers: + - name: duocthu + orgId: 1 + folder: Dược thư + type: file + disableDeletion: false + updateIntervalSeconds: 30 + allowUiUpdates: true + options: + path: /var/lib/grafana/dashboards + foldersFromFilesStructure: false diff --git a/infra/docker/grafana/provisioning/datasources/prometheus.yml b/infra/docker/grafana/provisioning/datasources/prometheus.yml new file mode 100644 index 0000000..bb009bb --- /dev/null +++ b/infra/docker/grafana/provisioning/datasources/prometheus.yml @@ -0,0 +1,9 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: false diff --git a/infra/docker/prometheus/prometheus.yml b/infra/docker/prometheus/prometheus.yml new file mode 100644 index 0000000..64fcbf5 --- /dev/null +++ b/infra/docker/prometheus/prometheus.yml @@ -0,0 +1,16 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + +scrape_configs: + - job_name: ai-service + metrics_path: /metrics + static_configs: + # ai-service runs on the host during development. `/metrics` returns 404 + # when METRICS_ENABLED is false or prometheus-client is not installed, so + # a target that is UP-but-404 means the app is running without metrics — + # not that the app is down. + - targets: ["host.docker.internal:8079"] + labels: + service: ai-service + env: local diff --git a/packages/api-client/src/mockFixtures.ts b/packages/api-client/src/mockFixtures.ts index 4865d86..507e66d 100644 --- a/packages/api-client/src/mockFixtures.ts +++ b/packages/api-client/src/mockFixtures.ts @@ -8,9 +8,13 @@ export function buildMockResponse(userContent: string): SendMessageResponse { message: { id, role: "assistant", + // Deliberately carries no dose. A fixture that states a plausible + // milligram figure is indistinguishable from a real answer in a + // screenshot, and screenshots outlive the code that produced them. content: - `(Mock) Paracetamol được chỉ định để giảm đau, hạ sốt. Liều thường dùng ở người ` + - `lớn là 500-1000mg mỗi 4-6 giờ, tối đa 4g/ngày. Đây là dữ liệu giả lập cho câu hỏi: "${userContent}".`, + `⚠️ DỮ LIỆU GIẢ LẬP — KHÔNG PHẢI NỘI DUNG DƯỢC THƯ. Đây là phản hồi mẫu ` + + `dùng khi backend chưa sẵn sàng, không chứa số liệu y khoa và không được ` + + `dùng để tra cứu. Câu hỏi nhận được: "${userContent}".`, citations: [ { drugName: "PARACETAMOL", diff --git a/packages/api-client/src/sendChatMessage.ts b/packages/api-client/src/sendChatMessage.ts index 61905ac..627c3ba 100644 --- a/packages/api-client/src/sendChatMessage.ts +++ b/packages/api-client/src/sendChatMessage.ts @@ -1,9 +1,43 @@ import type { SendMessageResponse } from "@duoc-thu/shared-types"; import { buildMockResponse } from "./mockFixtures"; -const MOCK_LATENCY_MS = 400; +/** + * Sends a question to the real RAG backend through the app's own route + * handler, which holds the service URL and maps the response. + * + * The mock is retained but is now opt-in via `NEXT_PUBLIC_USE_MOCK_CHAT`, and + * every mocked answer is labelled as such in its own text. Silently falling + * back to a plausible-looking fake answer is the exact failure this project + * cannot afford: a fabricated dose that looks like a real one. + */ +const USE_MOCK = process.env.NEXT_PUBLIC_USE_MOCK_CHAT === "true"; -export async function sendChatMessage(content: string): Promise { - await new Promise((resolve) => setTimeout(resolve, MOCK_LATENCY_MS)); - return buildMockResponse(content); +export class ChatUnavailableError extends Error { + constructor(readonly status: number) { + super(`Chat backend unavailable (${status})`); + this.name = "ChatUnavailableError"; + } +} + +export async function sendChatMessage( + content: string, + conversationId?: string +): Promise { + if (USE_MOCK) { + await new Promise((resolve) => setTimeout(resolve, 400)); + return buildMockResponse(content); + } + + const response = await fetch("/api/chat", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ content, conversationId }), + }); + + if (!response.ok) { + // Surfaced to the user as an error state, never as an answer. + throw new ChatUnavailableError(response.status); + } + + return (await response.json()) as SendMessageResponse; }