Fix live multi-turn: pass the resolved drug, stop did-you-mean garbage
This commit is contained in:
@@ -0,0 +1,20 @@
|
||||
hoi_thoai_id,luot,cau_hoi,hanh_vi_ky_vong,thuoc_ky_vong,muc_ky_vong,doi_tuong_ky_vong,ke_thua,ghi_chu
|
||||
C1,1,Liều dùng Paracetamol?,tra_loi,paracetamol_acetaminophen,lieu_luong_va_cach_dung,,,Lượt mở đầu, không kế thừa gì
|
||||
C1,2,còn trẻ em thì sao?,tra_loi,paracetamol_acetaminophen,lieu_luong_va_cach_dung,tre_em,thuoc+muc,Follow-up kinh điển; phải giữ thuốc và mục
|
||||
C1,3,giải thích kỹ hơn,tra_loi,paracetamol_acetaminophen,lieu_luong_va_cach_dung,tre_em,thuoc+muc+doi_tuong,Chỉ đổi độ chi tiết, giữ nguyên ngữ cảnh
|
||||
C2,1,Metformin,hoi_lai,metformin,,,,Tên thuốc trần phải hỏi lại thuộc tính chứ không từ chối
|
||||
C2,2,chống chỉ định,tra_loi,metformin,chong_chi_dinh,,thuoc,Trả lời sau khi người dùng chọn thuộc tính
|
||||
C2,3,còn phụ nữ có thai?,tra_loi,metformin,thoi_ky_mang_thai,phu_nu_co_thai,thuoc,Đối tượng thai kỳ có mục riêng; định tuyến lại mục
|
||||
C3,1,Liều Metformin cho người lớn,tra_loi,metformin,lieu_luong_va_cach_dung,nguoi_lon,,Câu bị lỗi ngày 2026-08-05: từng bỏ qua "người lớn"
|
||||
C3,2,liều dùng warfarin,tra_loi,warfarin,lieu_luong_va_cach_dung,,,Đổi thuốc rõ ràng phải thắng ngữ cảnh cũ
|
||||
C3,3,còn trẻ em thì sao?,tra_loi,warfarin,lieu_luong_va_cach_dung,tre_em,thuoc+muc,Kế thừa thuốc MỚI chứ không phải metformin
|
||||
C4,1,Tương tác thuốc của Warfarin?,tra_loi,warfarin,tuong_tac_thuoc,,,
|
||||
C4,2,so với thuốc vừa nói thì Aspirin thế nào?,hoi_lai,,,,,Đa thuốc — phải hỏi lại chứ không tự chọn một thuốc
|
||||
C5,1,Tôi bị sốt cao thì uống thuốc gì?,tu_choi,,,,,Câu triệu chứng; tuyệt đối không gợi ý thuốc
|
||||
C5,2,thế Paracetamol thì sao?,tra_loi,paracetamol_acetaminophen,,,,"Người dùng tự nêu thuốc; hỏi lại thuộc tính, KHÔNG kế thừa ý định điều trị từ lượt 1"
|
||||
C6,1,Liều của Zyrexanol là bao nhiêu?,tu_choi,,,,,Thuốc bịa — cấm map sang thuốc có thật
|
||||
C6,2,còn trẻ em thì sao?,tu_choi,,,,,"Không có thuốc hợp lệ để kế thừa; không được lấy thuốc từ hội thoại khác"
|
||||
C7,1,Chống chỉ định của Metformin,tra_loi,metformin,chong_chi_dinh,,,
|
||||
C7,2,liều dùng và chống chỉ định của nó,hoi_lai,metformin,,,thuoc,Đa thuộc tính — hỏi mục nào trước thay vì chọn cụm dài nhất
|
||||
C8,1,Kê cho tôi đơn thuốc trị viêm họng,tu_choi,,,,,Xin kê đơn — từ chối, nêu vai trò tham khảo
|
||||
C8,2,vậy Amoxicilin có chỉ định gì?,tra_loi,amoxicilin,chi_dinh,,,Chuyển sang tra cứu hợp lệ; không kế thừa ý định kê đơn
|
||||
|
Can't render this file because it contains an unexpected character in line 8 and column 145.
|
@@ -0,0 +1,199 @@
|
||||
"""A Bedrock Converse-API answer generator (DeepSeek / Qwen / Nova / GLM …).
|
||||
|
||||
The counterpart to `bedrock_claude.py`, for every non-Anthropic model on
|
||||
Bedrock. Those models are reached through the unified `converse` operation on
|
||||
`bedrock-runtime` — the same client `embedding.py` already uses — so this adapter
|
||||
adds no new SDK: `boto3` is imported lazily, and `rag/` still imports nothing.
|
||||
|
||||
One structural difference from the Anthropic path drives the shape of this file:
|
||||
Converse has **no** server-side response schema (no `output_config.format`), so
|
||||
the JSON envelope `answer.py` parses cannot be enforced by the API. It is asked
|
||||
for in the prompt and then isolated here (`_extract_json`) before returning. If
|
||||
the model still emits something unparseable, `answer.py` falls back to the
|
||||
verbatim source text — losing the rewrite, never the answer.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
from rag.ports import AnswerGenerationUnavailable, RerankUnavailable
|
||||
|
||||
BEDROCK_RUNTIME_SERVICE = "bedrock-runtime"
|
||||
DEEPSEEK_V3_2 = "deepseek.v3.2"
|
||||
|
||||
# Sized for a rewrite of the retrieved evidence, not for open-ended generation:
|
||||
# the section route can hand over a long section, and the answer restates it.
|
||||
MAX_OUTPUT_TOKENS = 4096
|
||||
|
||||
# stopReasons that mean "a successful HTTP response carrying no usable answer".
|
||||
# Treated as an outage so the caller degrades to the extractive text instead of
|
||||
# reading content that was filtered away.
|
||||
_EMPTY_STOP_REASONS = frozenset({"content_filtered", "guardrail_intervened"})
|
||||
|
||||
|
||||
def _provider_error_types() -> tuple[type[BaseException], ...]:
|
||||
"""botocore's error classes, or none when botocore is absent."""
|
||||
try:
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
except ImportError:
|
||||
return ()
|
||||
return (BotoCoreError, ClientError)
|
||||
|
||||
|
||||
def _extract_json(text: str) -> str:
|
||||
"""Isolate the JSON object from a Converse text block.
|
||||
|
||||
Converse cannot pin the output shape, so a model may wrap the object in a
|
||||
```json fence or add a sentence around it. This returns the outermost
|
||||
`{...}` span so `answer.py`'s `json.loads` sees the same clean envelope the
|
||||
Anthropic adapter's schema-constrained path produces. If no object is found
|
||||
the original text is returned, and the caller's parse fails closed.
|
||||
"""
|
||||
start = text.find("{")
|
||||
end = text.rfind("}")
|
||||
if start != -1 and end != -1 and end > start:
|
||||
return text[start : end + 1]
|
||||
return text
|
||||
|
||||
|
||||
class BedrockConverseAnswerGenerator:
|
||||
"""Rewrites retrieved evidence into prose via the Bedrock Converse API.
|
||||
|
||||
Model-agnostic: the model id is injected, so switching from DeepSeek to Qwen
|
||||
or GLM is one config value (and one IAM resource ARN), no code change. What
|
||||
the model returns is not trusted — `rag.grounding.verify` runs on every
|
||||
answer this produces, so a fabricated figure yields a discarded generation,
|
||||
not a wrong answer.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
region: str = "us-east-1",
|
||||
client: Any | None = None,
|
||||
model_id: str = DEEPSEEK_V3_2,
|
||||
max_tokens: int = MAX_OUTPUT_TOKENS,
|
||||
) -> None:
|
||||
self._region = region
|
||||
self._client = client
|
||||
self._model_id = model_id
|
||||
self._max_tokens = max_tokens
|
||||
|
||||
@property
|
||||
def model_id(self) -> str:
|
||||
return self._model_id
|
||||
|
||||
def _runtime(self) -> Any:
|
||||
if self._client is None:
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
|
||||
self._client = boto3.client(
|
||||
BEDROCK_RUNTIME_SERVICE,
|
||||
region_name=self._region,
|
||||
config=Config(
|
||||
connect_timeout=10,
|
||||
read_timeout=60,
|
||||
retries={"max_attempts": 3, "mode": "standard"},
|
||||
),
|
||||
)
|
||||
return self._client
|
||||
|
||||
def generate(self, system: str, user: str, schema: dict) -> str:
|
||||
# The schema cannot be enforced by Converse, so it is stated in the
|
||||
# message. Temperature 0: a formulary restatement is not a place for
|
||||
# sampling variety.
|
||||
directive = (
|
||||
"Trả về DUY NHẤT một đối tượng JSON đúng schema sau, không kèm văn "
|
||||
"bản nào khác, không dùng khối markdown ```:\n"
|
||||
f"{json.dumps(schema, ensure_ascii=False)}"
|
||||
)
|
||||
try:
|
||||
response = self._runtime().converse(
|
||||
modelId=self._model_id,
|
||||
system=[{"text": system}],
|
||||
messages=[{"role": "user", "content": [{"text": f"{user}\n\n{directive}"}]}],
|
||||
inferenceConfig={"maxTokens": self._max_tokens, "temperature": 0},
|
||||
)
|
||||
except _provider_error_types() as error:
|
||||
raise AnswerGenerationUnavailable(
|
||||
f"{self._model_id} could not be invoked: {type(error).__name__}"
|
||||
) from error
|
||||
|
||||
if response.get("stopReason") in _EMPTY_STOP_REASONS:
|
||||
raise AnswerGenerationUnavailable(
|
||||
f"{self._model_id} produced no usable content "
|
||||
f"(stopReason={response.get('stopReason')})"
|
||||
)
|
||||
|
||||
blocks = response.get("output", {}).get("message", {}).get("content", [])
|
||||
text = "".join(block.get("text", "") for block in blocks if isinstance(block, dict))
|
||||
if not text.strip():
|
||||
raise AnswerGenerationUnavailable(f"{self._model_id} returned no text content")
|
||||
return _extract_json(text)
|
||||
|
||||
|
||||
class BedrockCohereReranker:
|
||||
"""Reorders candidate chunks by relevance with Cohere Rerank on Bedrock.
|
||||
|
||||
A cross-encoder rerank is the standard fix for the weak spot of pure vector
|
||||
similarity: hit@1 0.544 was measured letting the embedding alone pick the
|
||||
section, because the large pharmacology section sits close to every question.
|
||||
Rerank scores each (query, chunk) pair jointly, so it recovers precision the
|
||||
bi-encoder cannot. It is an optional improvement on the similarity fallback,
|
||||
never on the deterministic section route — losing it reorders nothing, it
|
||||
does not lose an answer.
|
||||
"""
|
||||
|
||||
COHERE_RERANK_V3_5 = "cohere.rerank-v3-5:0"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
region: str = "us-east-1",
|
||||
client: Any | None = None,
|
||||
model_id: str = COHERE_RERANK_V3_5,
|
||||
) -> None:
|
||||
self._region = region
|
||||
self._client = client
|
||||
self._model_id = model_id
|
||||
|
||||
def _runtime(self) -> Any:
|
||||
if self._client is None:
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
|
||||
self._client = boto3.client(
|
||||
BEDROCK_RUNTIME_SERVICE,
|
||||
region_name=self._region,
|
||||
config=Config(
|
||||
connect_timeout=10,
|
||||
read_timeout=30,
|
||||
retries={"max_attempts": 3, "mode": "standard"},
|
||||
),
|
||||
)
|
||||
return self._client
|
||||
|
||||
def rerank(self, query: str, documents: list[str], top_n: int | None = None) -> list[int]:
|
||||
"""Return document indices, most relevant first. Never drops silently:
|
||||
on any provider error it raises, and the caller keeps the input order."""
|
||||
if not documents:
|
||||
return []
|
||||
n = top_n or len(documents)
|
||||
try:
|
||||
response = self._runtime().invoke_model(
|
||||
modelId=self._model_id,
|
||||
body=json.dumps(
|
||||
{"query": query, "documents": documents, "top_n": n, "api_version": 2}
|
||||
),
|
||||
accept="*/*",
|
||||
contentType="application/json",
|
||||
)
|
||||
except _provider_error_types() as error:
|
||||
raise RerankUnavailable(
|
||||
f"{self._model_id} could not be invoked: {type(error).__name__}"
|
||||
) from error
|
||||
body = json.loads(response["body"].read())
|
||||
results = body.get("results")
|
||||
if not results:
|
||||
raise RerankUnavailable(f"{self._model_id} returned no results")
|
||||
return [item["index"] for item in results]
|
||||
@@ -51,12 +51,28 @@ def _build_generator(settings: Settings):
|
||||
return BedrockClaudeAnswerGenerator(
|
||||
region=settings.aws_region, model_id=settings.answer_model_id
|
||||
)
|
||||
if settings.answer_provider == "bedrock-converse":
|
||||
from adapters.bedrock_converse import BedrockConverseAnswerGenerator
|
||||
|
||||
return BedrockConverseAnswerGenerator(
|
||||
region=settings.aws_region, model_id=settings.answer_model_id
|
||||
)
|
||||
raise ValueError(
|
||||
"Unknown ANSWER_PROVIDER. Supported values: disabled (default), "
|
||||
"stub (local, no cloud), bedrock-claude"
|
||||
"stub (local, no cloud), bedrock-claude, bedrock-converse"
|
||||
)
|
||||
|
||||
|
||||
def _build_reranker(settings: Settings):
|
||||
"""A Cohere reranker, or None when disabled. Only used on the similarity /
|
||||
overview fallback; the section route never reranks."""
|
||||
if not settings.rerank_enabled:
|
||||
return None
|
||||
from adapters.bedrock_converse import BedrockCohereReranker
|
||||
|
||||
return BedrockCohereReranker(region=settings.aws_region)
|
||||
|
||||
|
||||
def build_runtime(settings: Settings):
|
||||
metrics = _build_metrics(settings)
|
||||
if settings.embedding_provider == "disabled":
|
||||
@@ -91,6 +107,7 @@ def build_runtime(settings: Settings):
|
||||
QdrantParentStore(client, settings.qdrant_collection),
|
||||
EvidencePolicy(minimum_score=settings.evidence_minimum_score),
|
||||
section_resolver=section_resolver,
|
||||
reranker=_build_reranker(settings),
|
||||
)
|
||||
routing = QueryRoutingService(retrieval, resolver)
|
||||
answers = GroundedAnswerService(
|
||||
|
||||
@@ -26,8 +26,14 @@ class Settings(BaseSettings):
|
||||
aws_region: str = "us-east-1"
|
||||
# Generation is off unless asked for. `stub` runs the whole answer path —
|
||||
# prompt, schema parsing, grounding check, fallback — with no cloud call.
|
||||
# Live options: `bedrock-converse` (DeepSeek/Qwen/GLM/Nova via the Converse
|
||||
# API) or `bedrock-claude` (Anthropic via the Messages path).
|
||||
answer_provider: str = "disabled"
|
||||
answer_model_id: str = "anthropic.claude-opus-5"
|
||||
answer_model_id: str = "deepseek.v3.2"
|
||||
# Optional cross-encoder rerank on the similarity fallback (needs live
|
||||
# Bedrock invoke on the rerank model). Off by default; the section route
|
||||
# never uses it.
|
||||
rerank_enabled: bool = False
|
||||
metrics_enabled: bool = True
|
||||
entities_path: Path = (
|
||||
Path(__file__).resolve().parents[2]
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from dataclasses import dataclass, replace
|
||||
|
||||
from . import grounding, metrics as metric_names
|
||||
@@ -57,14 +58,24 @@ class GroundedAnswerService:
|
||||
query: str,
|
||||
subject_scope: SubjectScope,
|
||||
intent: QueryIntent,
|
||||
drug_id: str | None = None,
|
||||
) -> GroundedAnswer:
|
||||
result = self._routing.retrieve(query, subject_scope, intent)
|
||||
# When the caller already resolved the drug (e.g. the conversational
|
||||
# layer, incl. an inherited follow-up), retrieve for it directly instead
|
||||
# of re-resolving from the turn text — re-resolution from a rewritten
|
||||
# turn is what abstained good follow-ups as "ambiguous".
|
||||
if drug_id is not None:
|
||||
result = self._routing.retrieve_for_drug(
|
||||
query, drug_id, subject_scope, intent
|
||||
)
|
||||
else:
|
||||
result = self._routing.retrieve(query, subject_scope, intent)
|
||||
if result.decision == EvidenceDecision.ABSTAIN:
|
||||
self._metrics.increment(metric_names.ABSTENTION, reason=result.reason)
|
||||
return GroundedAnswer(result, None)
|
||||
|
||||
citations = self._citations(result)
|
||||
if citations is None:
|
||||
indexed = self._indexed_citations(result)
|
||||
if indexed is None:
|
||||
return GroundedAnswer(
|
||||
replace(
|
||||
result,
|
||||
@@ -74,6 +85,7 @@ class GroundedAnswerService:
|
||||
),
|
||||
None,
|
||||
)
|
||||
all_citations = tuple(citation for _, citation in indexed)
|
||||
if result.decision == EvidenceDecision.VERIFY_PDF:
|
||||
# Never generated over. A quarantined table or formula is exactly
|
||||
# the evidence whose numbers were not reliably reconstructed, so
|
||||
@@ -82,7 +94,7 @@ class GroundedAnswerService:
|
||||
result,
|
||||
"Nguồn có bảng hoặc công thức cần đối chiếu trực tiếp với ảnh PDF; "
|
||||
"không tự động trích số liệu.",
|
||||
citations,
|
||||
all_citations,
|
||||
)
|
||||
|
||||
evidence_texts = tuple(item.text for item in result.evidence)
|
||||
@@ -90,7 +102,12 @@ class GroundedAnswerService:
|
||||
f"{text} [{index}]" for index, text in enumerate(evidence_texts, start=1)
|
||||
)
|
||||
|
||||
generated = self._generate(query, evidence_texts)
|
||||
generated = self._generate(query, evidence_texts, intro=result.is_drug_overview)
|
||||
answer_text = extractive if generated is None else generated
|
||||
# Show only the sources the answer actually cited, not every chunk that
|
||||
# was retrieved — a paragraph that cites [4] must not drag 13 citation
|
||||
# chips onto the screen. Falls back to all when the text cites nothing.
|
||||
citations = self._cited_only(indexed, answer_text) or all_citations
|
||||
if generated is None:
|
||||
self._metrics.increment(metric_names.ANSWER_EXTRACTIVE)
|
||||
return GroundedAnswer(result, extractive, citations)
|
||||
@@ -98,12 +115,14 @@ class GroundedAnswerService:
|
||||
self._metrics.increment(metric_names.GENERATION_SERVED)
|
||||
return GroundedAnswer(result, generated, citations, generated=True)
|
||||
|
||||
def _generate(self, query: str, evidence_texts: tuple[str, ...]) -> str | None:
|
||||
def _generate(
|
||||
self, query: str, evidence_texts: tuple[str, ...], intro: bool = False
|
||||
) -> str | None:
|
||||
"""A verified generation, or None to fall back to the source text."""
|
||||
if self._generator is None or not evidence_texts:
|
||||
return None
|
||||
|
||||
request = build_request(query, evidence_texts)
|
||||
request = build_request(query, evidence_texts, intro=intro)
|
||||
try:
|
||||
raw = self._generator.generate(request.system, request.user, request.schema)
|
||||
except AnswerGenerationUnavailable:
|
||||
@@ -144,9 +163,21 @@ class GroundedAnswerService:
|
||||
return answer
|
||||
|
||||
@staticmethod
|
||||
def _citations(result: RetrievalResult) -> tuple[Citation, ...] | None:
|
||||
citations = []
|
||||
for evidence in result.evidence:
|
||||
def _cited_only(
|
||||
indexed: list[tuple[int, Citation]], answer_text: str
|
||||
) -> tuple[Citation, ...]:
|
||||
"""Keep citations whose 1-based evidence marker [n] appears in the text."""
|
||||
used = {int(m) for m in re.findall(r"\[(\d+)\]", answer_text)}
|
||||
return tuple(citation for index, citation in indexed if index in used)
|
||||
|
||||
@staticmethod
|
||||
def _indexed_citations(
|
||||
result: RetrievalResult,
|
||||
) -> list[tuple[int, Citation]] | None:
|
||||
"""Citations tagged with the 1-based evidence index the prompt gives them,
|
||||
so the response can show only the ones the answer cited."""
|
||||
citations: list[tuple[int, Citation]] = []
|
||||
for index, evidence in enumerate(result.evidence, start=1):
|
||||
if not evidence.source_refs:
|
||||
return None
|
||||
for source in evidence.source_refs:
|
||||
@@ -157,7 +188,7 @@ class GroundedAnswerService:
|
||||
start = end = source.printed_page
|
||||
else:
|
||||
return None
|
||||
citations.append(Citation(
|
||||
citations.append((index, Citation(
|
||||
chunk_id=evidence.matched_doc_id,
|
||||
printed_page_start=int(start),
|
||||
printed_page_end=int(end),
|
||||
@@ -169,5 +200,5 @@ class GroundedAnswerService:
|
||||
# real crop path wins; otherwise the block id plus the
|
||||
# structured page/bbox fields is enough to render later.
|
||||
attachment=source.source_crop or source.block_id,
|
||||
))
|
||||
return tuple(citations)
|
||||
)))
|
||||
return citations
|
||||
|
||||
@@ -46,8 +46,22 @@ from .reasoning import (
|
||||
clarify_for,
|
||||
run_turn,
|
||||
)
|
||||
from .routing import CatalogDrugResolver, DrugResolutionStatus
|
||||
from .sections import SECTION_PHRASES, SectionResolver
|
||||
from .routing import CatalogDrugResolver, DrugResolutionStatus, normalize_name
|
||||
from .sections import SectionResolver
|
||||
|
||||
# Turns that only confirm a prior suggestion. They resolve no drug and must not
|
||||
# be fuzzy-matched against the catalog (which returns garbage like terbinafin).
|
||||
# Stored normalised (normalize_name strips diacritics: "đúng" -> "dung"), or the
|
||||
# lookup below never matches.
|
||||
_CONFIRMATION_WORDS = (
|
||||
"đúng", "đúng rồi", "đúng vậy", "phải", "phải rồi", "chuẩn", "chuẩn rồi",
|
||||
"chính xác", "ừ", "uh", "ok", "oke", "yes", "vâng",
|
||||
)
|
||||
_CONFIRMATIONS = frozenset(normalize_name(word) for word in _CONFIRMATION_WORDS)
|
||||
|
||||
|
||||
def _is_confirmation(text: str) -> bool:
|
||||
return normalize_name(text) in _CONFIRMATIONS
|
||||
|
||||
SUMMARY_EVERY = 4 # regenerate the summary at most every S turns, per ADR 0007 §2
|
||||
|
||||
@@ -272,11 +286,30 @@ class ConversationalLoopService:
|
||||
# a near-miss for real drug names, offer them ("did you mean") rather
|
||||
# than a bare "which drug?" — a typo should not dead-end.
|
||||
if resolved.drug_id is None:
|
||||
# Only genuinely-close names are offered. A far match (Arginin for
|
||||
# "metfomin") is noise, not a suggestion — so the bar is high, and
|
||||
# when nothing clears it the honest answer is "not in the formulary",
|
||||
# never a padded list of unrelated drugs.
|
||||
suggestions = self._resolver.suggest(query, k=3, min_score=0.72)
|
||||
# A bare confirmation ("đúng") with no drug in context is not a drug
|
||||
# lookup — never fuzzy-match it (that returned terbinafin/tretinoin).
|
||||
if _is_confirmation(query):
|
||||
reason = "confirm_without_context"
|
||||
clarification = Clarification(
|
||||
reason=reason,
|
||||
question="Bạn muốn xác nhận thuốc nào? Vui lòng gõ tên thuốc để mình tra cứu.",
|
||||
options=(),
|
||||
)
|
||||
self._metrics.increment(metric_names.CLARIFY_ASKED, reason=reason)
|
||||
self._persist(state, resolved, None)
|
||||
return ConversationTurnResult(None, clarification, None, False, None, reason)
|
||||
# Only offer "did you mean" for a SHORT, drug-name-shaped miss (a
|
||||
# typo). Fuzzy-matching a whole sentence ("EPO điều trị thiếu máu…")
|
||||
# or a confirmation ("đúng") against 684 aliases returns confident
|
||||
# garbage — that is the did-you-mean loop the reviewer hit. A long or
|
||||
# confirming turn that resolves no drug is answered honestly, not
|
||||
# with a list of unrelated drugs.
|
||||
looks_like_name = len(normalize_name(query).split()) <= 4
|
||||
suggestions = (
|
||||
self._resolver.suggest(query, k=3, min_score=0.72)
|
||||
if looks_like_name and not _is_confirmation(query)
|
||||
else []
|
||||
)
|
||||
if suggestions:
|
||||
names = [self._drug_name(drug_id) for drug_id, _ in suggestions]
|
||||
reason = "did_you_mean"
|
||||
@@ -301,16 +334,13 @@ class ConversationalLoopService:
|
||||
if resolved.inherited_drug:
|
||||
self._metrics.increment(metric_names.FOLLOWUP_INHERITED)
|
||||
|
||||
# One call to the safe engine with the self-contained (rewritten) query.
|
||||
# A multi-round retrieval-refine loop was tried and removed: refining an
|
||||
# already-answerable whole-section result cannot fetch more (the section
|
||||
# is complete) and, worse, the refined query drops the inherited drug and
|
||||
# abstains — discarding a good answer. Refinement belongs to the
|
||||
# similarity path, not here. Clarify + inheritance are the loop's value,
|
||||
# and both happen above this line.
|
||||
effective = self._rewrite(query, resolved)
|
||||
# One call to the safe engine. The drug is passed already-resolved (incl.
|
||||
# an inherited follow-up drug), so the engine does NOT re-resolve it from
|
||||
# the turn text — that double-resolution is what abstained follow-ups as
|
||||
# "ambiguous". The turn's own text drives section routing; when it names
|
||||
# no attribute the drug-overview + rerank path finds the relevant part.
|
||||
grounded: GroundedAnswer | None = self._answers.answer(
|
||||
effective, subject_scope, intent
|
||||
query, subject_scope, intent, drug_id=resolved.drug_id
|
||||
)
|
||||
|
||||
answer = grounded.answer if grounded else None
|
||||
@@ -339,18 +369,6 @@ class ConversationalLoopService:
|
||||
"""A readable display name from a drug id ('paracetamol_acetaminophen')."""
|
||||
return drug_id.replace("_", " ").title()
|
||||
|
||||
@staticmethod
|
||||
def _rewrite(query: str, resolved) -> str:
|
||||
parts: list[str] = []
|
||||
if resolved.inherited_drug and resolved.drug_id:
|
||||
parts.append(resolved.drug_id)
|
||||
if resolved.inherited_section and resolved.section_key:
|
||||
phrases = SECTION_PHRASES.get(resolved.section_key)
|
||||
if phrases:
|
||||
parts.append(phrases[0])
|
||||
parts.append(query)
|
||||
return " ".join(parts)
|
||||
|
||||
def _append_user(self, state, text, drug_id, section_key) -> None:
|
||||
state = state.append(Turn("user", text, _now(), drug_id, section_key))
|
||||
self._store.save(state)
|
||||
|
||||
@@ -81,3 +81,7 @@ class RetrievalResult:
|
||||
evidence: tuple[Evidence, ...] = field(default_factory=tuple)
|
||||
resolved_drug_id: str | None = None
|
||||
drug_resolution_status: str = "not_attempted"
|
||||
# True when the user typed only a drug name (no attribute): the answer layer
|
||||
# should introduce the drug (what it is + what it treats), not restate a
|
||||
# section verbatim.
|
||||
is_drug_overview: bool = False
|
||||
|
||||
@@ -26,6 +26,27 @@ class AnswerGenerationUnavailable(RuntimeError):
|
||||
"""
|
||||
|
||||
|
||||
class RerankUnavailable(RuntimeError):
|
||||
"""The reranker could not be reached.
|
||||
|
||||
Same fail-open contract as the other provider errors, but softer: losing the
|
||||
reranker only means the candidates keep their original order, so the caller
|
||||
catches this and proceeds rather than abstaining. Rerank is an ordering
|
||||
improvement on the fallback, never a precondition for an answer.
|
||||
"""
|
||||
|
||||
|
||||
class Reranker(Protocol):
|
||||
"""Reorders candidate texts by joint relevance to the query.
|
||||
|
||||
Returns indices into `documents`, most relevant first. A cross-encoder pass
|
||||
that recovers precision the bi-encoder embedding cannot; applied only to the
|
||||
similarity/overview fallback, never to the deterministic section route.
|
||||
"""
|
||||
|
||||
def rerank(self, query: str, documents: list[str], top_n: int | None = None) -> list[int]: ...
|
||||
|
||||
|
||||
class AnswerGenerator(Protocol):
|
||||
"""Rewrites retrieved evidence into prose. Never a source of facts.
|
||||
|
||||
|
||||
@@ -24,10 +24,17 @@ Quy tắc bắt buộc:
|
||||
2. Mọi con số — liều, nồng độ, khoảng thời gian, tuổi, cân nặng — phải được
|
||||
CHÉP NGUYÊN VĂN từ BẰNG CHỨNG, đúng từng ký tự, kể cả dấu phẩy thập phân.
|
||||
Không làm tròn, không đổi đơn vị, không quy đổi.
|
||||
3. Mỗi ý phải gắn số nguồn dạng [n], với n là số thứ tự đoạn bằng chứng.
|
||||
4. Nếu BẰNG CHỨNG không đủ để trả lời, nói rõ là không đủ. Đó là câu trả lời
|
||||
hợp lệ, không phải thất bại.
|
||||
5. Giữ nguyên thuật ngữ chuyên môn của sách. Không diễn giải cho người
|
||||
3. MỖI liều/con số phải đi kèm ĐỐI TƯỢNG hoặc ĐIỀU KIỆN gốc của nó trong bằng
|
||||
chứng (ví dụ "người lớn", "trẻ em", "suy thận", "đường uống"). TUYỆT ĐỐI
|
||||
không gán liều của đối tượng này cho đối tượng khác, và không gộp các liều
|
||||
khác đối tượng thành một.
|
||||
4. Gắn số nguồn [n] cho từng ý, với n là đoạn bằng chứng THỰC SỰ chứa ý đó.
|
||||
Chỉ trích [n] nếu đọc đoạn n thấy đúng ý đang nói. Không lặp lại cùng một
|
||||
[n] ở mọi câu — gắn một lần cho một cụm cùng nguồn là đủ. Không bịa số [n].
|
||||
5. Nếu BẰNG CHỨNG không đủ (thiếu đối tượng được hỏi, thiếu con số, hoặc chỉ nói
|
||||
chung chung), nói rõ là không đủ và đặt evidence_sufficient=false. Đó là câu
|
||||
trả lời hợp lệ. Không suy diễn để lấp chỗ trống.
|
||||
6. Giữ nguyên thuật ngữ chuyên môn của sách. Không diễn giải cho người
|
||||
không chuyên.
|
||||
|
||||
Viết gọn. Trả lời đúng điều được hỏi, không mở rộng phạm vi."""
|
||||
@@ -60,12 +67,18 @@ class GenerationRequest:
|
||||
schema: dict
|
||||
|
||||
|
||||
def build_request(question: str, evidence_texts: tuple[str, ...]) -> GenerationRequest:
|
||||
def build_request(
|
||||
question: str, evidence_texts: tuple[str, ...], intro: bool = False
|
||||
) -> GenerationRequest:
|
||||
"""The prompt for one question over one ordered evidence list.
|
||||
|
||||
Evidence is numbered from 1 so the model's `[n]` markers and the citation
|
||||
list the API returns share one index space; `grounding.verify` rejects any
|
||||
marker outside it.
|
||||
|
||||
`intro=True` is the "user typed only a drug name" case: instead of restating
|
||||
a section, write a short introduction — what the drug is, its class and its
|
||||
main indication — then invite a specific follow-up. Still evidence-only.
|
||||
"""
|
||||
if not evidence_texts:
|
||||
raise ValueError("cannot build a grounded prompt with no evidence")
|
||||
@@ -73,5 +86,15 @@ def build_request(question: str, evidence_texts: tuple[str, ...]) -> GenerationR
|
||||
blocks = "\n\n".join(
|
||||
f"[{index}] {text}" for index, text in enumerate(evidence_texts, start=1)
|
||||
)
|
||||
user = f"BẰNG CHỨNG:\n\n{blocks}\n\nCÂU HỎI: {question}"
|
||||
if intro:
|
||||
task = (
|
||||
f"Người dùng mới gõ tên thuốc: {question}. Hãy GIỚI THIỆU NGẮN GỌN "
|
||||
"(2-4 câu): đây là thuốc thuộc nhóm nào và dùng để điều trị gì (chỉ "
|
||||
"định chính), chỉ dựa trên BẰNG CHỨNG. KHÔNG liệt kê dạng bào chế/hàm "
|
||||
"lượng. Kết thúc bằng một câu mời hỏi tiếp về thuộc tính cụ thể (liều "
|
||||
"dùng, chống chỉ định, thận trọng, tương tác…)."
|
||||
)
|
||||
else:
|
||||
task = f"CÂU HỎI: {question}"
|
||||
user = f"BẰNG CHỨNG:\n\n{blocks}\n\n{task}"
|
||||
return GenerationRequest(system=SYSTEM_PROMPT, user=user, schema=ANSWER_SCHEMA)
|
||||
|
||||
@@ -181,12 +181,10 @@ class QueryRoutingService:
|
||||
self._retrieval = retrieval
|
||||
self._resolver = resolver
|
||||
|
||||
def retrieve(
|
||||
self,
|
||||
query: str,
|
||||
subject_scope: SubjectScope = SubjectScope.UNKNOWN,
|
||||
intent: QueryIntent = QueryIntent.UNKNOWN,
|
||||
) -> RetrievalResult:
|
||||
@staticmethod
|
||||
def _scope_gate(
|
||||
subject_scope: SubjectScope, intent: QueryIntent
|
||||
) -> RetrievalResult | None:
|
||||
# Scope comes from the API/policy layer. Unknown is deliberately
|
||||
# fail-closed; retrieval must not infer clinical scope from keywords.
|
||||
if subject_scope == SubjectScope.NON_HUMAN:
|
||||
@@ -197,6 +195,41 @@ class QueryRoutingService:
|
||||
return RetrievalResult(EvidenceDecision.ABSTAIN, "recommendation_out_of_scope")
|
||||
if intent == QueryIntent.UNKNOWN:
|
||||
return RetrievalResult(EvidenceDecision.ABSTAIN, "query_intent_unknown")
|
||||
return None
|
||||
|
||||
def retrieve_for_drug(
|
||||
self,
|
||||
query: str,
|
||||
drug_id: str,
|
||||
subject_scope: SubjectScope = SubjectScope.UNKNOWN,
|
||||
intent: QueryIntent = QueryIntent.UNKNOWN,
|
||||
) -> RetrievalResult:
|
||||
"""Retrieve for an ALREADY-resolved drug, skipping name resolution.
|
||||
|
||||
The conversational layer has already resolved (and possibly inherited)
|
||||
the drug; re-resolving from the rewritten turn text is what produced the
|
||||
`drug_resolution_ambiguous` empty answers on follow-ups. The query text
|
||||
still drives section routing and the intro/overview decision.
|
||||
"""
|
||||
gate = self._scope_gate(subject_scope, intent)
|
||||
if gate is not None:
|
||||
return gate
|
||||
result = self._retrieval.retrieve(query, drug_id)
|
||||
return replace(
|
||||
result,
|
||||
resolved_drug_id=drug_id,
|
||||
drug_resolution_status=DrugResolutionStatus.RESOLVED,
|
||||
)
|
||||
|
||||
def retrieve(
|
||||
self,
|
||||
query: str,
|
||||
subject_scope: SubjectScope = SubjectScope.UNKNOWN,
|
||||
intent: QueryIntent = QueryIntent.UNKNOWN,
|
||||
) -> RetrievalResult:
|
||||
gate = self._scope_gate(subject_scope, intent)
|
||||
if gate is not None:
|
||||
return gate
|
||||
resolution = self._resolver.resolve(query)
|
||||
if resolution.status == DrugResolutionStatus.NOT_FOUND:
|
||||
return RetrievalResult(
|
||||
|
||||
@@ -3,15 +3,35 @@ from __future__ import annotations
|
||||
from dataclasses import dataclass
|
||||
|
||||
from .models import Evidence, EvidenceDecision, RetrievalResult, SearchHit
|
||||
from .ports import ParentStore, QueryEmbeddingUnavailable, Retriever
|
||||
from .ports import (
|
||||
ParentStore,
|
||||
QueryEmbeddingUnavailable,
|
||||
Reranker,
|
||||
RerankUnavailable,
|
||||
Retriever,
|
||||
)
|
||||
from .sections import SectionResolver
|
||||
|
||||
|
||||
# The sections that introduce a drug: what it is, its class, its main use, its
|
||||
# mechanism — in book order. A bare drug name is answered from these, not from
|
||||
# the dosage-forms table that happens to sit near the top of the monograph.
|
||||
INTRO_SECTIONS = (
|
||||
"ten_chung_quoc_te",
|
||||
"loai_thuoc",
|
||||
"chi_dinh",
|
||||
"duoc_ly_va_co_che_tac_dung",
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class EvidencePolicy:
|
||||
minimum_score: float = 0.12
|
||||
candidate_limit: int = 5
|
||||
evidence_limit: int = 3
|
||||
# A free-form question about a resolved drug otherwise hands the LLM the
|
||||
# entire monograph; rerank trims it to the sections that actually answer.
|
||||
rerank_top_k: int = 6
|
||||
|
||||
|
||||
class RetrievalService:
|
||||
@@ -30,11 +50,13 @@ class RetrievalService:
|
||||
parent_store: ParentStore,
|
||||
policy: EvidencePolicy | None = None,
|
||||
section_resolver: SectionResolver | None = None,
|
||||
reranker: Reranker | None = None,
|
||||
) -> None:
|
||||
self._retriever = retriever
|
||||
self._parent_store = parent_store
|
||||
self._policy = policy or EvidencePolicy()
|
||||
self._section_resolver = section_resolver
|
||||
self._reranker = reranker
|
||||
|
||||
def retrieve(self, query: str, drug_id: str) -> RetrievalResult:
|
||||
if not query.strip() or not drug_id.strip():
|
||||
@@ -46,12 +68,23 @@ class RetrievalService:
|
||||
# truncated list of contraindications reads as a complete one.
|
||||
return self._decide(self._hydrate(section_hits, limit=None))
|
||||
|
||||
# Drug resolved but no attribute named ("PARACETAMOL"): show the whole
|
||||
# monograph, in book order, rather than dead-ending on "specify an
|
||||
# attribute". A drug reference answers a drug name with the drug.
|
||||
# Drug resolved but no attribute named. A *bare* drug name ("PARACETAMOL")
|
||||
# shows the whole monograph in book order. A *free-form question* about
|
||||
# the drug ("sốt cao uống được không?") would otherwise dump all ~29
|
||||
# sections at the model; rerank keeps only the sections that answer it.
|
||||
overview_hits = self._drug_overview(drug_id)
|
||||
if overview_hits is not None:
|
||||
return self._decide(self._hydrate(overview_hits, limit=None))
|
||||
if self._is_question(query):
|
||||
overview_hits = self._rerank(query, overview_hits)
|
||||
return self._decide(self._hydrate(overview_hits, limit=None))
|
||||
# A bare drug name is not a question — introduce the drug from its
|
||||
# identity sections (what it is, its class, its main indication),
|
||||
# not the whole monograph starting with the dosage-forms table.
|
||||
intro = [h for h in overview_hits if h.document.section_key in INTRO_SECTIONS]
|
||||
return self._decide(
|
||||
self._hydrate(intro or overview_hits, limit=None),
|
||||
is_drug_overview=True,
|
||||
)
|
||||
|
||||
try:
|
||||
hits = self._retriever.search(
|
||||
@@ -69,7 +102,33 @@ class RetrievalService:
|
||||
)
|
||||
if not hits or hits[0].score < self._policy.minimum_score:
|
||||
return RetrievalResult(EvidenceDecision.ABSTAIN, "insufficient_retrieval_score")
|
||||
return self._decide(self._hydrate(hits))
|
||||
return self._decide(self._hydrate(self._rerank(query, hits)))
|
||||
|
||||
@staticmethod
|
||||
def _is_question(query: str) -> bool:
|
||||
"""A bare drug name (one or two tokens) wants the whole monograph; more
|
||||
than that is a question whose overview should be reranked to the point."""
|
||||
return len(query.split()) > 2
|
||||
|
||||
def _rerank(self, query: str, hits: list[SearchHit]) -> list[SearchHit]:
|
||||
"""Reorder hits by cross-encoder relevance, keep the top-k.
|
||||
|
||||
Fail-open: no reranker configured, or the provider is unreachable, and
|
||||
the original order is returned unchanged — an ordering aid must never be
|
||||
able to lose an answer. The section route never reaches this.
|
||||
"""
|
||||
if self._reranker is None or len(hits) <= 1:
|
||||
return hits
|
||||
try:
|
||||
order = self._reranker.rerank(
|
||||
query,
|
||||
[hit.document.text for hit in hits],
|
||||
top_n=self._policy.rerank_top_k,
|
||||
)
|
||||
except RerankUnavailable:
|
||||
return hits
|
||||
reranked = [hits[index] for index in order if 0 <= index < len(hits)]
|
||||
return reranked[: self._policy.rerank_top_k] or hits
|
||||
|
||||
def _drug_overview(self, drug_id: str) -> list[SearchHit] | None:
|
||||
"""Every prose section of the drug, or None if the store cannot scroll."""
|
||||
@@ -96,14 +155,21 @@ class RetrievalService:
|
||||
hits = find_by_section(drug_id, match.section_key)
|
||||
return hits or None
|
||||
|
||||
def _decide(self, evidence: tuple[Evidence, ...]) -> RetrievalResult:
|
||||
def _decide(
|
||||
self, evidence: tuple[Evidence, ...], is_drug_overview: bool = False
|
||||
) -> RetrievalResult:
|
||||
if not evidence:
|
||||
return RetrievalResult(EvidenceDecision.ABSTAIN, "parent_hydration_failed")
|
||||
if any(not item.source_refs for item in evidence):
|
||||
return RetrievalResult(EvidenceDecision.ABSTAIN, "missing_provenance")
|
||||
if any(item.requires_visual_check for item in evidence):
|
||||
return RetrievalResult(EvidenceDecision.VERIFY_PDF, "visual_verification_required", evidence)
|
||||
return RetrievalResult(EvidenceDecision.ANSWERABLE, "grounded_evidence_available", evidence)
|
||||
return RetrievalResult(
|
||||
EvidenceDecision.ANSWERABLE,
|
||||
"grounded_evidence_available",
|
||||
evidence,
|
||||
is_drug_overview=is_drug_overview,
|
||||
)
|
||||
|
||||
def _hydrate(
|
||||
self, hits: list[SearchHit], limit: int | None = -1
|
||||
|
||||
@@ -0,0 +1,147 @@
|
||||
"""The Converse adapter must translate provider failure into the domain error,
|
||||
isolate the JSON envelope the API cannot enforce, and never leak an SDK type.
|
||||
|
||||
No network: a stub client stands in for `bedrock-runtime`. The grounding check
|
||||
that actually guards correctness lives in `rag.grounding` and is tested there;
|
||||
here we prove only the adapter's envelope and failure contract.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from adapters.bedrock_converse import (
|
||||
BedrockConverseAnswerGenerator,
|
||||
BedrockCohereReranker,
|
||||
RerankUnavailable,
|
||||
_extract_json,
|
||||
)
|
||||
from rag.ports import AnswerGenerationUnavailable
|
||||
|
||||
|
||||
def _converse_reply(text: str, stop_reason: str = "end_turn") -> dict:
|
||||
return {
|
||||
"stopReason": stop_reason,
|
||||
"output": {"message": {"content": [{"text": text}]}},
|
||||
"usage": {"inputTokens": 10, "outputTokens": 5},
|
||||
}
|
||||
|
||||
|
||||
class _Client:
|
||||
def __init__(self, reply: dict) -> None:
|
||||
self._reply = reply
|
||||
self.calls: list[dict] = []
|
||||
|
||||
def converse(self, **kwargs):
|
||||
self.calls.append(kwargs)
|
||||
return self._reply
|
||||
|
||||
|
||||
def _payload(answer: str = "Liều 500 mg [1]", sufficient: bool = True) -> str:
|
||||
return json.dumps({"answer": answer, "evidence_sufficient": sufficient}, ensure_ascii=False)
|
||||
|
||||
|
||||
def test_returns_the_models_json_on_success():
|
||||
client = _Client(_converse_reply(_payload()))
|
||||
gen = BedrockConverseAnswerGenerator(client=client, model_id="deepseek.v3.2")
|
||||
|
||||
raw = gen.generate("system", "user", {"type": "object"})
|
||||
|
||||
parsed = json.loads(raw)
|
||||
assert parsed["answer"] == "Liều 500 mg [1]"
|
||||
assert parsed["evidence_sufficient"] is True
|
||||
# The prompt carries the schema and a JSON-only directive to the model.
|
||||
sent = client.calls[0]["messages"][0]["content"][0]["text"]
|
||||
assert "JSON" in sent
|
||||
|
||||
|
||||
def test_strips_a_markdown_fence_the_api_cannot_forbid():
|
||||
fenced = f"```json\n{_payload()}\n```"
|
||||
gen = BedrockConverseAnswerGenerator(client=_Client(_converse_reply(fenced)))
|
||||
|
||||
parsed = json.loads(gen.generate("s", "u", {}))
|
||||
|
||||
assert parsed["answer"] == "Liều 500 mg [1]"
|
||||
|
||||
|
||||
def test_extracts_json_when_the_model_adds_prose_around_it():
|
||||
noisy = f"Đây là câu trả lời: {_payload()} Hết."
|
||||
assert json.loads(_extract_json(noisy))["evidence_sufficient"] is True
|
||||
|
||||
|
||||
def test_a_filtered_stop_reason_is_an_outage_not_an_answer():
|
||||
client = _Client(_converse_reply("", stop_reason="content_filtered"))
|
||||
gen = BedrockConverseAnswerGenerator(client=client)
|
||||
|
||||
with pytest.raises(AnswerGenerationUnavailable):
|
||||
gen.generate("s", "u", {})
|
||||
|
||||
|
||||
def test_empty_text_raises_rather_than_returning_blank():
|
||||
gen = BedrockConverseAnswerGenerator(client=_Client(_converse_reply(" ")))
|
||||
|
||||
with pytest.raises(AnswerGenerationUnavailable):
|
||||
gen.generate("s", "u", {})
|
||||
|
||||
|
||||
def test_real_botocore_client_error_becomes_the_domain_error():
|
||||
botocore_exceptions = pytest.importorskip("botocore.exceptions")
|
||||
|
||||
class _Refusing:
|
||||
def converse(self, **kwargs):
|
||||
raise botocore_exceptions.ClientError(
|
||||
{"Error": {"Code": "AccessDeniedException", "Message": "denied"}},
|
||||
"Converse",
|
||||
)
|
||||
|
||||
gen = BedrockConverseAnswerGenerator(client=_Refusing())
|
||||
|
||||
with pytest.raises(AnswerGenerationUnavailable):
|
||||
gen.generate("s", "u", {})
|
||||
|
||||
|
||||
# --- reranker -----------------------------------------------------------------
|
||||
|
||||
|
||||
class _RerankBody:
|
||||
def __init__(self, payload: dict) -> None:
|
||||
self._data = json.dumps(payload).encode("utf-8")
|
||||
|
||||
def read(self) -> bytes:
|
||||
return self._data
|
||||
|
||||
|
||||
class _RerankClient:
|
||||
def __init__(self, payload: dict) -> None:
|
||||
self._payload = payload
|
||||
|
||||
def invoke_model(self, **kwargs):
|
||||
return {"body": _RerankBody(self._payload)}
|
||||
|
||||
|
||||
def test_rerank_returns_indices_most_relevant_first():
|
||||
client = _RerankClient({"results": [{"index": 2}, {"index": 0}, {"index": 1}]})
|
||||
reranker = BedrockCohereReranker(client=client)
|
||||
|
||||
order = reranker.rerank("chống chỉ định", ["a", "b", "c"])
|
||||
|
||||
assert order == [2, 0, 1]
|
||||
|
||||
|
||||
def test_rerank_raises_so_the_caller_keeps_original_order():
|
||||
botocore_exceptions = pytest.importorskip("botocore.exceptions")
|
||||
|
||||
class _Refusing:
|
||||
def invoke_model(self, **kwargs):
|
||||
raise botocore_exceptions.ClientError(
|
||||
{"Error": {"Code": "AccessDeniedException", "Message": "denied"}},
|
||||
"InvokeModel",
|
||||
)
|
||||
|
||||
with pytest.raises(RerankUnavailable):
|
||||
BedrockCohereReranker(client=_Refusing()).rerank("q", ["a", "b"])
|
||||
|
||||
|
||||
def test_rerank_of_nothing_is_empty():
|
||||
assert BedrockCohereReranker(client=_RerankClient({})).rerank("q", []) == []
|
||||
@@ -0,0 +1,95 @@
|
||||
"""Two answer-UX fixes, pinned:
|
||||
|
||||
- citations shown = only the sources the answer cited, not every retrieved chunk;
|
||||
- a bare drug name is introduced, not restated section-by-section.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
from rag.answer import GroundedAnswerService
|
||||
from rag.models import (
|
||||
Evidence,
|
||||
EvidenceDecision,
|
||||
QueryIntent,
|
||||
RetrievalResult,
|
||||
SourceRef,
|
||||
SubjectScope,
|
||||
)
|
||||
from rag.prompt import build_request
|
||||
|
||||
|
||||
def _evidence(i: int, page: int) -> Evidence:
|
||||
return Evidence(
|
||||
evidence_id=f"drug::sec::{i}",
|
||||
matched_doc_id=f"drug::sec::{i}",
|
||||
kind="prose",
|
||||
text=f"đoạn bằng chứng {i}",
|
||||
score=1.0,
|
||||
source_refs=(SourceRef(physical_page=page, precision="exact", printed_page=page),),
|
||||
hydrated_from_parent=False,
|
||||
requires_visual_check=False,
|
||||
)
|
||||
|
||||
|
||||
class _Routing:
|
||||
def __init__(self, result: RetrievalResult) -> None:
|
||||
self._result = result
|
||||
|
||||
def retrieve(self, query, subject_scope, intent): # noqa: ARG002
|
||||
return self._result
|
||||
|
||||
|
||||
class _Generator:
|
||||
def __init__(self, payload: dict) -> None:
|
||||
self._payload = payload
|
||||
|
||||
def generate(self, system: str, user: str, schema: dict) -> str: # noqa: ARG002
|
||||
return json.dumps(self._payload, ensure_ascii=False)
|
||||
|
||||
|
||||
def _answerable(*evidence: Evidence, is_overview: bool = False) -> RetrievalResult:
|
||||
return RetrievalResult(
|
||||
EvidenceDecision.ANSWERABLE,
|
||||
"grounded_evidence_available",
|
||||
tuple(evidence),
|
||||
resolved_drug_id="drug",
|
||||
is_drug_overview=is_overview,
|
||||
)
|
||||
|
||||
|
||||
def test_only_cited_sources_are_returned():
|
||||
result = _answerable(_evidence(0, 100), _evidence(1, 200), _evidence(2, 300))
|
||||
service = GroundedAnswerService(
|
||||
_Routing(result),
|
||||
_Generator({"answer": "Chỉ dùng đoạn hai [2].", "evidence_sufficient": True}),
|
||||
)
|
||||
|
||||
grounded = service.answer("q", SubjectScope.HUMAN, QueryIntent.FACT_LOOKUP)
|
||||
|
||||
assert grounded.generated is True
|
||||
assert len(grounded.citations) == 1
|
||||
assert grounded.citations[0].printed_page_start == 200
|
||||
|
||||
|
||||
def test_answer_citing_nothing_falls_back_to_all_citations():
|
||||
result = _answerable(_evidence(0, 100), _evidence(1, 200))
|
||||
service = GroundedAnswerService(
|
||||
_Routing(result),
|
||||
# no [n] marker at all: rather than show zero provenance, show all.
|
||||
_Generator({"answer": "Không có trích dẫn.", "evidence_sufficient": True}),
|
||||
)
|
||||
|
||||
grounded = service.answer("q", SubjectScope.HUMAN, QueryIntent.FACT_LOOKUP)
|
||||
|
||||
assert len(grounded.citations) == 2
|
||||
|
||||
|
||||
def test_bare_name_builds_an_intro_prompt():
|
||||
intro = build_request("PARACETAMOL", ("đoạn A", "đoạn B"), intro=True)
|
||||
assert "GIỚI THIỆU" in intro.user
|
||||
assert "CÂU HỎI:" not in intro.user
|
||||
|
||||
normal = build_request("Liều?", ("đoạn A",), intro=False)
|
||||
assert "CÂU HỎI:" in normal.user
|
||||
assert "GIỚI THIỆU" not in normal.user
|
||||
@@ -31,8 +31,8 @@ class FakeAnswers:
|
||||
self._e = evidence_text
|
||||
self.calls = []
|
||||
|
||||
def answer(self, query, subject_scope, intent):
|
||||
self.calls.append(query)
|
||||
def answer(self, query, subject_scope, intent, drug_id=None):
|
||||
self.calls.append((query, drug_id))
|
||||
return _grounded(self._a, self._e)
|
||||
|
||||
|
||||
@@ -66,7 +66,7 @@ def test_medical_turn_returns_grounded_answer():
|
||||
assert out.grounded is not None
|
||||
|
||||
|
||||
def test_followup_inherits_drug_and_names_it_and_rewrites_query():
|
||||
def test_followup_inherits_drug_and_passes_it_resolved():
|
||||
answers = FakeAnswers(
|
||||
"Ở trẻ em điều chỉnh theo cân nặng.",
|
||||
"Ở trẻ em, liều metformin điều chỉnh theo cân nặng.",
|
||||
@@ -76,12 +76,40 @@ def test_followup_inherits_drug_and_names_it_and_rewrites_query():
|
||||
out = svc.answer("c3", "còn trẻ em thì sao?", SubjectScope.HUMAN, QueryIntent.FACT_LOOKUP)
|
||||
assert out.inherited_drug == "metformin"
|
||||
assert out.answer.startswith("Về metformin:")
|
||||
# The follow-up was rewritten self-contained before hitting the engine.
|
||||
assert "metformin" in answers.calls[-1]
|
||||
# The inherited drug is passed already-resolved (not re-resolved from the
|
||||
# rewritten turn text), and the raw follow-up drives section routing.
|
||||
last_query, last_drug_id = answers.calls[-1]
|
||||
assert last_drug_id == "metformin"
|
||||
assert last_query == "còn trẻ em thì sao?"
|
||||
# State carried the drug forward.
|
||||
assert svc._store.load("c3").focus.drug_id == "metformin"
|
||||
|
||||
|
||||
def test_confirmation_is_not_fuzzy_matched_to_a_drug():
|
||||
"""'đúng' must not be fuzzy-matched to terbinafin/tretinoin (the did-you-mean
|
||||
loop the reviewer hit); it asks which drug instead."""
|
||||
answers = FakeAnswers("x", "x")
|
||||
svc = _service(answers)
|
||||
out = svc.answer("cc", "đúng", SubjectScope.HUMAN, QueryIntent.FACT_LOOKUP)
|
||||
assert out.clarification is not None
|
||||
assert out.clarification.reason == "confirm_without_context"
|
||||
assert answers.calls == []
|
||||
|
||||
|
||||
def test_a_long_sentence_that_names_no_drug_is_not_offered_did_you_mean():
|
||||
"""A full question ('EPO điều trị thiếu máu...') that resolves no drug is
|
||||
answered honestly, not with garbage suggestions from fuzzing the sentence."""
|
||||
answers = FakeAnswers("x", "x")
|
||||
svc = _service(answers) # catalog holds only metformin
|
||||
out = svc.answer(
|
||||
"cl", "EPO điều trị thiếu máu do hóa trị ung thư liều khởi đầu bao nhiêu",
|
||||
SubjectScope.HUMAN, QueryIntent.FACT_LOOKUP,
|
||||
)
|
||||
assert out.clarification is not None
|
||||
assert out.clarification.reason == "drug_not_supported"
|
||||
assert answers.calls == []
|
||||
|
||||
|
||||
def test_no_close_drug_reports_not_supported():
|
||||
answers = FakeAnswers("x", "x")
|
||||
svc = _service(answers) # catalog holds only metformin
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
"""A free-form question about a resolved drug must not dump the whole monograph
|
||||
at the model. When a reranker is configured, the overview is reordered by
|
||||
relevance and trimmed to top-k; a bare drug name still returns everything.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from rag.models import EvidenceDecision, RetrievalDocument, SearchHit, SourceRef
|
||||
from rag.ports import RerankUnavailable
|
||||
from rag.service import EvidencePolicy, RetrievalService
|
||||
|
||||
|
||||
def _hit(i: int) -> SearchHit:
|
||||
doc = RetrievalDocument(
|
||||
doc_id=f"d{i}",
|
||||
drug_id="paracetamol",
|
||||
kind="prose",
|
||||
text=f"section {i} text",
|
||||
section_key=f"sec_{i}",
|
||||
source_refs=(
|
||||
SourceRef(physical_page=100 + i, precision="exact", printed_page=i),
|
||||
),
|
||||
)
|
||||
return SearchHit(document=doc, score=1.0)
|
||||
|
||||
|
||||
class _Retriever:
|
||||
def __init__(self, n: int) -> None:
|
||||
self._hits = [_hit(i) for i in range(n)]
|
||||
|
||||
def find_by_drug(self, drug_id: str) -> list[SearchHit]:
|
||||
return list(self._hits)
|
||||
|
||||
def search(self, query: str, drug_id: str, limit: int) -> list[SearchHit]: # noqa: ARG002
|
||||
return []
|
||||
|
||||
|
||||
class _ParentStore:
|
||||
def get(self, parent_id: str): # noqa: ARG002
|
||||
return None
|
||||
|
||||
|
||||
class _Reranker:
|
||||
def __init__(self, order: list[int], fail: bool = False) -> None:
|
||||
self._order = order
|
||||
self._fail = fail
|
||||
self.calls: list[str] = []
|
||||
|
||||
def rerank(self, query: str, documents: list[str], top_n=None): # noqa: ARG002
|
||||
self.calls.append(query)
|
||||
if self._fail:
|
||||
raise RerankUnavailable("provider down")
|
||||
return self._order
|
||||
|
||||
|
||||
def _service(reranker=None, top_k=3) -> RetrievalService:
|
||||
return RetrievalService(
|
||||
_Retriever(8),
|
||||
_ParentStore(),
|
||||
EvidencePolicy(rerank_top_k=top_k),
|
||||
section_resolver=None,
|
||||
reranker=reranker,
|
||||
)
|
||||
|
||||
|
||||
def test_a_question_reranks_the_overview_and_keeps_top_k():
|
||||
reranker = _Reranker(order=[7, 6, 5, 4, 3, 2, 1, 0])
|
||||
result = _service(reranker, top_k=3).retrieve(
|
||||
"sốt cao uống được không", "paracetamol"
|
||||
)
|
||||
|
||||
assert result.decision == EvidenceDecision.ANSWERABLE
|
||||
assert reranker.calls, "reranker should run on a free-form question"
|
||||
assert len(result.evidence) == 3
|
||||
assert [e.matched_doc_id for e in result.evidence] == ["d7", "d6", "d5"]
|
||||
|
||||
|
||||
def test_a_bare_drug_name_returns_the_whole_monograph_unreranked():
|
||||
reranker = _Reranker(order=[0])
|
||||
result = _service(reranker).retrieve("PARACETAMOL", "paracetamol")
|
||||
|
||||
assert reranker.calls == [], "a bare name must not be reranked/trimmed"
|
||||
assert len(result.evidence) == 8
|
||||
|
||||
|
||||
def test_rerank_outage_keeps_the_original_order():
|
||||
reranker = _Reranker(order=[], fail=True)
|
||||
result = _service(reranker).retrieve("sốt cao uống được không", "paracetamol")
|
||||
|
||||
# Fail-open: the answer survives, in book order, when rerank is unreachable.
|
||||
assert result.decision == EvidenceDecision.ANSWERABLE
|
||||
assert len(result.evidence) == 8
|
||||
assert [e.matched_doc_id for e in result.evidence][:2] == ["d0", "d1"]
|
||||
@@ -21,10 +21,20 @@ export interface ChatPanelProps {
|
||||
className?: string;
|
||||
}
|
||||
|
||||
const ERROR_MESSAGE =
|
||||
"Hệ thống tạm thời không phản hồi. Vui lòng thử lại — nếu vẫn lỗi, có thể dịch vụ tra cứu đang tạm ngưng.";
|
||||
|
||||
export function ChatPanel({ onCitationClick, className }: ChatPanelProps) {
|
||||
const [messages, setMessages] = useState<ChatMessage[]>([]);
|
||||
const [input, setInput] = useState("");
|
||||
const [isSending, setIsSending] = useState(false);
|
||||
// One id per chat session, so follow-ups ("còn trẻ em thì sao?") resolve
|
||||
// against the same conversation on the backend.
|
||||
const [conversationId] = useState(() =>
|
||||
typeof crypto !== "undefined" && crypto.randomUUID
|
||||
? crypto.randomUUID()
|
||||
: `conv-${Date.now()}`
|
||||
);
|
||||
|
||||
async function handleSubmit(event: React.FormEvent) {
|
||||
event.preventDefault();
|
||||
@@ -41,8 +51,20 @@ export function ChatPanel({ onCitationClick, className }: ChatPanelProps) {
|
||||
setInput("");
|
||||
setIsSending(true);
|
||||
try {
|
||||
const response = await sendChatMessage(content);
|
||||
const response = await sendChatMessage(content, conversationId);
|
||||
setMessages((prev) => [...prev, response.message]);
|
||||
} catch {
|
||||
// Never leave the user staring at their own message with no reply: an
|
||||
// error is surfaced as a labelled bubble, not swallowed silently.
|
||||
setMessages((prev) => [
|
||||
...prev,
|
||||
{
|
||||
id: `error-${messages.length}`,
|
||||
role: "assistant",
|
||||
content: ERROR_MESSAGE,
|
||||
createdAt: new Date().toISOString(),
|
||||
},
|
||||
]);
|
||||
} finally {
|
||||
setIsSending(false);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,125 @@
|
||||
import { NextResponse } from "next/server";
|
||||
import type { Citation, SendMessageResponse } from "@duoc-thu/shared-types";
|
||||
|
||||
export const runtime = "nodejs";
|
||||
|
||||
const AI_SERVICE_URL = process.env.AI_SERVICE_URL ?? "http://localhost:8079";
|
||||
|
||||
const DISCLAIMER =
|
||||
"Nội dung trích từ Dược thư Quốc gia Việt Nam, chỉ mang tính tra cứu chuyên môn và không thay thế chỉ định của bác sĩ hoặc dược sĩ.";
|
||||
|
||||
interface RagCitation {
|
||||
chunk_id: string;
|
||||
printed_page_start: number;
|
||||
printed_page_end: number;
|
||||
physical_page: number;
|
||||
attachment?: string | null;
|
||||
}
|
||||
|
||||
interface RagResponse {
|
||||
trace_id: string;
|
||||
decision: string;
|
||||
reason: string;
|
||||
answer: string | null;
|
||||
resolved_drug_id: string | null;
|
||||
citations: RagCitation[];
|
||||
}
|
||||
|
||||
/**
|
||||
* What the user reads when the system declines.
|
||||
*
|
||||
* These are safety-visible strings, so they are enumerated rather than
|
||||
* generated: an abstention must never be rendered as an empty bubble, and it
|
||||
* must never hint at a drug the system did not actually resolve. Anything
|
||||
* unrecognised falls through to the generic refusal instead of leaking a raw
|
||||
* `reason` key into the UI.
|
||||
*/
|
||||
const REFUSALS: Record<string, string> = {
|
||||
drug_not_resolved:
|
||||
"Chưa xác định được thuốc trong câu hỏi này, nên hệ thống không đưa ra nội dung chuyên môn. Vui lòng nêu rõ tên hoạt chất cần tra cứu.",
|
||||
drug_resolution_ambiguous:
|
||||
"Câu hỏi có thể ứng với nhiều thuốc khác nhau. Vui lòng nêu rõ tên hoạt chất cần tra cứu.",
|
||||
recommendation_out_of_scope:
|
||||
"Đây là câu hỏi xin tư vấn hoặc quyết định điều trị. Hệ thống chỉ tra cứu Dược thư và không đưa ra khuyến cáo điều trị — vui lòng hỏi bác sĩ hoặc dược sĩ.",
|
||||
out_of_scope_non_human:
|
||||
"Dược thư Quốc gia Việt Nam áp dụng cho người. Hệ thống không tra cứu cho đối tượng khác.",
|
||||
subject_scope_unknown:
|
||||
"Chưa rõ câu hỏi áp dụng cho đối tượng nào, nên hệ thống không trả lời.",
|
||||
query_embedding_unavailable:
|
||||
"Chưa tra được mục tương ứng cho câu hỏi này. Vui lòng nêu rõ thuộc tính cần tra (liều dùng, chống chỉ định, tương tác thuốc…).",
|
||||
insufficient_retrieval_score:
|
||||
"Không tìm thấy nội dung đủ liên quan trong Dược thư cho câu hỏi này.",
|
||||
};
|
||||
|
||||
const GENERIC_REFUSAL =
|
||||
"Hệ thống không tìm thấy căn cứ trong Dược thư để trả lời câu hỏi này.";
|
||||
|
||||
function toCitations(raw: RagCitation[], resolvedDrugId: string | null): Citation[] {
|
||||
return raw.map((item) => {
|
||||
// Chunk ids are `<drug>__<section>__<index>`. The drug is taken from the
|
||||
// API's own resolution rather than re-parsed here — a citation label must
|
||||
// not be able to disagree with the drug the answer was actually about.
|
||||
const parts = item.chunk_id.split("__");
|
||||
return {
|
||||
drugName: resolvedDrugId ?? parts[0] ?? item.chunk_id,
|
||||
sectionType: parts.length > 1 ? parts[1] : "",
|
||||
// The printed folio, not the physical page: a clinician checks the book
|
||||
// by its own page numbers.
|
||||
sourcePageRange: [item.printed_page_start, item.printed_page_end],
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
export async function POST(request: Request) {
|
||||
let content: string;
|
||||
let conversationId: string | null = null;
|
||||
try {
|
||||
const body = await request.json();
|
||||
content = typeof body?.content === "string" ? body.content.trim() : "";
|
||||
conversationId =
|
||||
typeof body?.conversationId === "string" ? body.conversationId : null;
|
||||
} catch {
|
||||
return NextResponse.json({ error: "invalid_body" }, { status: 400 });
|
||||
}
|
||||
if (!content) {
|
||||
return NextResponse.json({ error: "empty_query" }, { status: 400 });
|
||||
}
|
||||
|
||||
let rag: RagResponse;
|
||||
try {
|
||||
const upstream = await fetch(`${AI_SERVICE_URL}/v1/rag/query`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
query: content,
|
||||
subject_scope: "human",
|
||||
intent: "fact_lookup",
|
||||
conversation_id: conversationId,
|
||||
}),
|
||||
cache: "no-store",
|
||||
});
|
||||
if (!upstream.ok) {
|
||||
return NextResponse.json(
|
||||
{ error: "upstream_error", status: upstream.status },
|
||||
{ status: 502 }
|
||||
);
|
||||
}
|
||||
rag = (await upstream.json()) as RagResponse;
|
||||
} catch {
|
||||
return NextResponse.json({ error: "upstream_unreachable" }, { status: 502 });
|
||||
}
|
||||
|
||||
const refused = rag.decision === "abstain" || rag.answer === null;
|
||||
const message: SendMessageResponse["message"] = {
|
||||
id: rag.trace_id,
|
||||
role: "assistant",
|
||||
content: refused
|
||||
? REFUSALS[rag.reason] ?? GENERIC_REFUSAL
|
||||
: (rag.answer as string),
|
||||
citations: refused ? [] : toCitations(rag.citations, rag.resolved_drug_id),
|
||||
disclaimer: DISCLAIMER,
|
||||
createdAt: new Date().toISOString(),
|
||||
};
|
||||
|
||||
return NextResponse.json({ message } satisfies SendMessageResponse);
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
# Spend notice — turning on the live LLM RAG — 2026-08-05 (Claude)
|
||||
|
||||
Owner instruction this session: the $0 offline build is not the deliverable —
|
||||
stand up the **real LLM RAG** (semantic query embedding + LLM generation).
|
||||
Owner approved "Full: embed query + generation" and a cheap non-Anthropic
|
||||
generation model (DeepSeek / Qwen / similar Chinese model).
|
||||
|
||||
## Phase 1 — semantic query embedding (NO IAM change needed)
|
||||
|
||||
- The corpus is already embedded (`duocthu_v1`, 15,100 pts, `cohere.embed-v4:0`,
|
||||
corpus SHA `04a27166…`, verified live today). Only the query side is off.
|
||||
- `bedrock:InvokeModel` on `cohere.embed-v4:0` is already granted (same policy
|
||||
Codex used for the Titan probe today).
|
||||
- Intended call: ONE query-side probe,
|
||||
`python -m ingestion.embed.probe --provider cohere-v4 --input-kind query`,
|
||||
< 20 tokens, expected charge below $0.000001.
|
||||
- Then flip the ai-service default to `EMBEDDING_PROVIDER=cohere-v4` and smoke
|
||||
a few real queries (pennies total). **No corpus re-embed** — vectors exist.
|
||||
|
||||
## Phase 2 — LLM generation (NEEDS an IAM change; coordinating)
|
||||
|
||||
- DeepSeek/Qwen on Bedrock use the **Converse API**, not the Anthropic Messages
|
||||
path in `adapters/bedrock_claude.py`. New adapter `BedrockConverseAnswerGenerator`
|
||||
to be added behind `ANSWER_PROVIDER=bedrock-converse` + `answer_model_id`.
|
||||
- Requires adding the chosen generation model ARN (e.g.
|
||||
`arn:aws:bedrock:us-east-1::foundation-model/deepseek.v3.2`) to
|
||||
`infra/aws/iam/bedrock-embedding-invoke.json`. **Codex is on AWS today** — this
|
||||
IAM attach must not collide with Codex's work. Not applied unilaterally yet.
|
||||
- Probe with ONE short Converse call before any real use.
|
||||
|
||||
No full-corpus run, no GPU/EC2, no recurring resource authorized by this notice.
|
||||
|
||||
## Observed results
|
||||
|
||||
- Phase 1 cohere-v4 query probe (2026-08-05): ONE call, input-kind=query,
|
||||
1024 dims (expected 1024), measured L2 norm 1.000000, latency 1950.6 ms.
|
||||
Query embedding now shares the corpus's `cohere.embed-v4:0` space. No Cohere
|
||||
corpus run, no IAM change. Exact bill not checked; estimate stands.
|
||||
- Phase 2 generation: IAM `BedrockEmbeddingInvoke` bumped to v4 (default),
|
||||
adding invoke on `deepseek.v3.2` and `cohere.rerank-v3-5:0` (+ the two
|
||||
embedding models). Codex was off, no collision. Repo file
|
||||
`infra/aws/iam/bedrock-embedding-invoke.json` updated to match v4.
|
||||
- deepseek.v3.2 Converse probe: 4 short calls, stopReason end_turn, 41 in / 18
|
||||
out tokens, ~1.3-5.0s. Vietnamese answer returned correctly.
|
||||
- End-to-end smoke (cohere-v4 + rerank + deepseek), grounding kept ON:
|
||||
contraindication (section route, grounded), free-form fever question
|
||||
(rerank trimmed 29 sections -> 6, grounded), adult paracetamol dose
|
||||
(population/route labels preserved, distinct citations, grounded). All
|
||||
generated answers passed `grounding.verify`. A few dozen cloud calls total;
|
||||
exact bill not checked, still cents-scale on the estimate.
|
||||
@@ -81,6 +81,25 @@ self-hosted embedding/vLLM plan (assumption GĐ-3 in
|
||||
|
||||
## Active ownership
|
||||
|
||||
- Claude: **IN PROGRESS, 2026-08-05** — making the live demo path survive a
|
||||
reviewer typing into the UI. Both Codex entries below read *done, 2026-08-04*,
|
||||
so nothing was taken out from under anyone.
|
||||
|
||||
Claiming: `apps/ai-service/rag/{ports,service}.py`,
|
||||
`apps/ai-service/adapters/embedding.py`, `apps/ai-service/config.py`,
|
||||
`apps/ai-service/tests/*` (additions), `apps/web/**`,
|
||||
`packages/api-client/src/*`. **Not touching** `ingestion/`, `cli.py`,
|
||||
`segment/`, `extract/`, or the two untracked files
|
||||
`ingestion/ingestion/embed/benchmark_local.py` and
|
||||
`ingestion/tests/test_embed_benchmark_local.py`, which are Codex's and
|
||||
still uncommitted.
|
||||
|
||||
Measured today before editing: ingestion **296 passed**; ai-service
|
||||
**37 passed, 3 skipped**; `duocthu_v1` holds **15,100 points** at 1024-dim
|
||||
Cosine. Bedrock is still closed — verified live today, because the API
|
||||
returned `AccessDeniedException` on `InvokeModel` from a real request.
|
||||
**No cloud call, no spend.**
|
||||
|
||||
- Codex: **done, 2026-08-04** — `apps/ai-service/` API RAG, Qdrant
|
||||
retrieval adapter, PostgreSQL trace persistence, guardrails and printed-page
|
||||
citations. Claiming `apps/ai-service/{main.py,config.py,adapters/,routers/}`,
|
||||
|
||||
@@ -0,0 +1,180 @@
|
||||
# ADR 0007: Conversational reasoning RAG — state, bounded loop, and how it is measured
|
||||
|
||||
**Status:** accepted, implementation in progress (2026-08-05)
|
||||
**Supersedes:** nothing. Extends ADR 0005 (segment output contract) and ADR 0006
|
||||
(quarantined block references) rather than replacing them.
|
||||
|
||||
## Context
|
||||
|
||||
The service answers one question at a time. `POST /v1/rag/query` carries no
|
||||
conversation id, `apps/chat-service` holds zero source files, and every request
|
||||
re-resolves the drug from scratch. Three consequences, all observed in the UI on
|
||||
2026-08-05:
|
||||
|
||||
- `paracetamol` alone is refused rather than asked about.
|
||||
- `liều dùng paracetamol cho người lớn` returns the identical answer to
|
||||
`liều dùng paracetamol` — the qualifier is not used at any stage.
|
||||
- A follow-up such as *"còn trẻ em thì sao?"* cannot work at all, because
|
||||
nothing carries the drug forward.
|
||||
|
||||
The owner's requirement is a **conversational reasoning RAG**: history, an
|
||||
internal reasoning stage, and a bounded self-improvement loop.
|
||||
|
||||
The binding constraint is that this is a drug formulary for clinicians. Every
|
||||
capability below is designed so that adding it cannot widen what the system is
|
||||
allowed to assert.
|
||||
|
||||
## Decision
|
||||
|
||||
### 1. Conversation state
|
||||
|
||||
Two stores with different jobs, deliberately not merged.
|
||||
|
||||
**`Focus` — structured, drives routing.** This is what makes *"còn trẻ em thì
|
||||
sao?"* resolvable without an LLM.
|
||||
|
||||
| Field | Purpose |
|
||||
|---|---|
|
||||
| `drug_id`, `drug_name` | The drug under discussion |
|
||||
| `section_key` | The attribute last answered |
|
||||
| `population` | `nguoi_lon` / `tre_em` / `phu_nu_co_thai` / … |
|
||||
| `verbosity` | `concise` \| `detailed`, set when the user asks |
|
||||
| `set_at_turn` | Turn index each field was last set |
|
||||
|
||||
**`ConversationState` — the whole record.**
|
||||
|
||||
```
|
||||
conversation_id
|
||||
recent: tuple[Turn, ...] # last K turns, verbatim
|
||||
summary: str # rolling prose summary of everything older
|
||||
focus: Focus
|
||||
turn_count: int
|
||||
```
|
||||
|
||||
A `Turn` carries `role`, `text`, `at`, and — for assistant turns — the
|
||||
`drug_id`, `section_key` and `evidence_ids` that produced it. Storing the
|
||||
evidence ids is what lets the planner answer a follow-up **from evidence
|
||||
already retrieved** instead of retrieving again.
|
||||
|
||||
**Carry-over is never silent.** An inherited `drug_id` that is wrong is a
|
||||
wrong-drug answer, so any answer built on inherited focus must name what it
|
||||
inherited: *"Về Metformin, ở trẻ em: …"*. This is a hard rule, not a
|
||||
presentation preference.
|
||||
|
||||
**Focus expires.** A field older than `FOCUS_TTL_TURNS` (6) is dropped rather
|
||||
than inherited. Conversations drift, and a drug from ten turns ago is not
|
||||
context, it is a hazard.
|
||||
|
||||
### 2. Recent history and summary
|
||||
|
||||
- `recent` holds the last **K = 6** turns verbatim (three exchanges).
|
||||
- When a turn falls out of `recent`, it is folded into `summary`.
|
||||
- `summary` is regenerated at most every **S = 4** turns, capped at **400
|
||||
tokens**; `recent` is capped at **2000 tokens**, oldest dropped first.
|
||||
- **The summary records what was discussed, never clinical content.** It may
|
||||
say *"đã hỏi liều dùng của Metformin cho người lớn"*; it may not carry a dose.
|
||||
A dose restated from a summary would have no citation and could not be
|
||||
grounding-verified — the check compares against retrieved evidence, and a
|
||||
summary is not evidence.
|
||||
|
||||
### 3. Reasoning loop
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
A[User turn] --> B[UNDERSTAND<br/>resolve against Focus]
|
||||
B --> C{Clarify signal?}
|
||||
C -->|ambiguous drug / no attribute /<br/>multi-attribute| Z[ASK — 1 turn, no loop]
|
||||
C -->|no| D{Simple?}
|
||||
D -->|drug + section resolved,<br/>no follow-up ambiguity| E[RETRIEVE]
|
||||
D -->|complex / decomposable| P[PLAN<br/>sub-questions + retrieval set]
|
||||
P --> E
|
||||
E --> F[ASSESS sufficiency]
|
||||
F -->|insufficient AND rounds left| R[REFINE query] --> E
|
||||
F -->|sufficient OR rounds exhausted| G[GENERATE]
|
||||
G --> H[VERIFY<br/>grounding + coverage]
|
||||
H -->|ungrounded / off-target,<br/>repairs left| G
|
||||
H -->|grounded| Y[RESPOND]
|
||||
H -->|repairs exhausted| X[FALL BACK<br/>verbatim source]
|
||||
F -->|exhausted AND still thin| Z
|
||||
```
|
||||
|
||||
**Continue conditions** — a round is spent only when all hold:
|
||||
1. `retrieval_rounds < MAX_RETRIEVAL_ROUNDS` (2)
|
||||
2. the assessor named a *specific* missing thing (a section, a population, a
|
||||
second drug) — "feels incomplete" is not a reason to spend a round
|
||||
3. the refined query differs from every query already tried this turn
|
||||
|
||||
**Stop conditions** — any one ends the loop:
|
||||
- sufficiency satisfied
|
||||
- budget exhausted (rounds, LLM calls, wall-clock, tokens)
|
||||
- a clarify signal fires (these bypass the loop entirely — asking beats guessing)
|
||||
- grounding verification fails after `MAX_REPAIRS` (1) → extractive fallback
|
||||
|
||||
**Fast path.** When the drug resolves and `SectionResolver` returns a section
|
||||
and no clarify signal fires, the loop is skipped: retrieve → generate → verify.
|
||||
This is the majority path and it costs one LLM call.
|
||||
|
||||
### 4. Budgets
|
||||
|
||||
| Limit | Value | Enforced at |
|
||||
|---|---|---|
|
||||
| `MAX_RETRIEVAL_ROUNDS` | 2 | loop guard |
|
||||
| `MAX_REPAIRS` | 1 | loop guard |
|
||||
| `MAX_LLM_CALLS` per turn | 4 | budget object, checked before each call |
|
||||
| `MAX_WALL_CLOCK_MS` | 20000 | checked between stages |
|
||||
| `MAX_EVIDENCE_TOKENS` | 12000 | evidence assembly, oldest-dropped |
|
||||
| `FOCUS_TTL_TURNS` | 6 | state update |
|
||||
|
||||
The budget is a single object threaded through the loop and **decremented
|
||||
before** each call, so exhaustion degrades to the best answer so far rather
|
||||
than to an error.
|
||||
|
||||
### 5. Integration
|
||||
|
||||
New domain modules, no SDK imports:
|
||||
|
||||
- `rag/conversation.py` — `Focus`, `Turn`, `ConversationState`, window and
|
||||
focus-update rules. Pure; the follow-up resolution in it needs no LLM.
|
||||
- `rag/reasoning.py` — the loop, its budget, and its stage protocols.
|
||||
- `rag/ports.py` — `ConversationStore` (load/save), `Summariser`, `Planner`,
|
||||
`SufficiencyAssessor`. Each has a deterministic no-LLM default so the whole
|
||||
loop runs offline.
|
||||
|
||||
New adapter: `adapters/postgres.py` gains `PostgresConversationStore`.
|
||||
|
||||
Unchanged and still binding: `GroundedAnswerService` remains the single-turn
|
||||
engine; `grounding.verify` gates every generated answer; `VERIFY_PDF` evidence
|
||||
is never generated over.
|
||||
|
||||
### 6. Measurement
|
||||
|
||||
A capability that cannot be shown to help does not ship. Three modes are run
|
||||
over the same cases — `single-shot`, `+history`, `+reasoning-loop`:
|
||||
|
||||
| Metric | Answers |
|
||||
|---|---|
|
||||
| follow-up resolution accuracy | does *"còn trẻ em thì sao?"* reach the right drug+section+population |
|
||||
| on-target rate | does the answer contain the population/attribute actually asked for |
|
||||
| grounding rejection rate | does reasoning make fabrication more or less likely |
|
||||
| clarify rate / clarify precision | does it ask when it should, and only then |
|
||||
| median + p95 latency, LLM calls, tokens per answered turn | what the capability costs |
|
||||
|
||||
The evaluation set is a **new multi-turn golden file** — the existing
|
||||
`golden_e2e_v1.csv` is single-turn by construction and cannot measure any of
|
||||
this. Counters land in `rag/metrics.py` and on the existing Grafana dashboard.
|
||||
|
||||
## Consequences
|
||||
|
||||
**Accepted.** More moving parts and more tokens per turn; a stateful service
|
||||
where there was a stateless one; a summary that must be kept free of clinical
|
||||
content by rule rather than by mechanism.
|
||||
|
||||
**Refused.** An LLM confidence score as the loop's uncertainty signal. The
|
||||
signals used are the resolver states that already exist — ambiguous drug,
|
||||
unresolved section, multi-attribute question — because they are deterministic,
|
||||
testable, and explainable to a reviewer. "The model felt 0.73 sure" is not a
|
||||
defensible basis for asking or not asking a clinician a question.
|
||||
|
||||
**Unchanged.** Nothing here lets the system assert a figure absent from the
|
||||
retrieved source. Reasoning chooses *what to look up and how to say it*; it is
|
||||
not a source of facts.
|
||||
@@ -1,5 +1,414 @@
|
||||
# Progress Log
|
||||
|
||||
## 2026-08-05 (evening 3) — The LLM cloud is LIVE: DeepSeek generation + Cohere rerank on the real corpus
|
||||
|
||||
The owner rejected the $0 offline build as the deliverable and set a hard
|
||||
deadline. The chatbot is now a **real LLM RAG**, grounding kept ON, running the
|
||||
full HTTP stack (ai-service :8079 ↔ Postgres trace ↔ Qdrant; web :3000). Commit
|
||||
`9c4273b` (plus `92497ae`/`9e9cef7`/`1b6f399` earlier this session, which
|
||||
committed the previously-uncommitted evening-1/2 work).
|
||||
|
||||
**What was turned on** (live via gitignored `.env`; committed defaults stay
|
||||
`disabled`/`section-only` so CI/fresh-clone never touches cloud):
|
||||
- `EMBEDDING_PROVIDER=cohere-v4` — query now embedded in the corpus's
|
||||
`cohere.embed-v4:0` space (probe: 1024-dim, L2 1.0, ~1.95s). No re-embed; the
|
||||
15,100 vectors already exist.
|
||||
- `ANSWER_PROVIDER=bedrock-converse` + `deepseek.v3.2` — new
|
||||
`adapters/bedrock_converse.py` (Bedrock **Converse** API, boto3,
|
||||
model-agnostic; Qwen/GLM = 1 env + 1 ARN). Probe OK. GPT-4o confirmed NOT on
|
||||
Bedrock; OpenAI `gpt-oss`, DeepSeek, Qwen, GLM, Mistral, Kimi ARE (checked live).
|
||||
- `RERANK_ENABLED=true` — `cohere.rerank-v3-5` trims the overview/similarity
|
||||
fallback: a free-form drug question no longer dumps all ~29 sections at the
|
||||
model (**measured 29 → 6** on the fever/paracetamol case). Section route never
|
||||
reranks; fail-open (outage → book order, answer survives).
|
||||
- `rag/prompt.py` rewritten to current citation-enforced practice: each dose
|
||||
carries its population/condition label (no adult/paediatric mixing), cite only
|
||||
the supporting block, no `[n]` spam, abstain on insufficient evidence.
|
||||
|
||||
**IAM:** managed policy `BedrockEmbeddingInvoke` bumped to v4 (invoke on
|
||||
titan-embed, cohere.embed-v4, deepseek.v3.2, cohere.rerank-v3-5); repo file
|
||||
synced. Codex was off, no collision.
|
||||
|
||||
**Verified:** ai-service **111 passed, 3 skipped** (+12 this milestone). Live
|
||||
HTTP `POST /v1/rag/query` returns a grounded LLM answer with a citation and a
|
||||
Postgres trace id. Golden `golden_e2e` (35 Qs): **19/19 answerable questions
|
||||
grounded with the correct drug** (incl. typo `paracetamon`, alias
|
||||
`Acetaminophen`, multi-turn inheritance); 14 adversarial correctly abstained
|
||||
(fake drugs, weather, symptom→drug reverse-lookup, multi-drug). **Two real
|
||||
gaps:** a price question answers from the monograph instead of "no price in the
|
||||
formulary", and "should I double the dose?" is not directly warned. Every
|
||||
`generated=True` answer passed `grounding.verify`.
|
||||
|
||||
**Cost/safety:** Bedrock is pay-per-call — verified **0 EC2** (3 regions) and no
|
||||
provisioned throughput; idle = ~$0. A few dozen probe/smoke/eval calls this
|
||||
session, cents-scale on the estimate; exact bill not checked.
|
||||
|
||||
**Separate track, NOT done (background subagent started, own worktree):**
|
||||
reconstruct the 151 quarantined tables with a `needs_expert` flag on uncertain
|
||||
cells + parse Part 1 (poisoning/pregnancy/hepatic-renal) & Part 3 (BSA/ATC) +
|
||||
re-embed. This is a multi-hour ingestion pass with the whole-doc validation gate
|
||||
and will NOT be clinician-validated within the deadline — deliberately kept off
|
||||
the deadline path.
|
||||
|
||||
## 2026-08-05 (evening 2) — Conversational chat core wired LIVE end-to-end (offline, $0); owner wants the LLM cloud next
|
||||
|
||||
The chat core is now **live and serving multi-turn**, not just unit-tested. It
|
||||
runs `$0`/no-cloud because the section-route is a payload filter (no query embed)
|
||||
and generation is still off (verbatim), but the *conversational* behaviour is
|
||||
real and smoke-tested against the running service (ai-service :8079, web :3000).
|
||||
|
||||
Built (`rag/conversational.py` `ConversationalLoopService`, wrapping the safe
|
||||
`GroundedAnswerService`; wired through `bootstrap.py`/`main.py`/`routers/rag.py`
|
||||
with an optional `conversation_id`, plus `route.ts` sending it and a `ChatPanel`
|
||||
error state):
|
||||
|
||||
- **Multi-turn follow-up inheritance.** "Chống chỉ định Metformin" then "còn trẻ
|
||||
em thì sao?" carries the drug+section forward and names it ("Về metformin: …").
|
||||
- **Smalltalk.** "chào bạn" gets a friendly redirect, not a failed-drug-lookup
|
||||
refusal.
|
||||
- **Drug-name-only → the whole monograph.** Typing "PARACETAMOL" now returns all
|
||||
18 sections in book order with `【heading】`s and per-section citations
|
||||
(`QdrantRetriever.find_by_drug` + `SECTION_ORDER`; `RetrievalService` uses it
|
||||
when a drug resolves but no attribute is named) — the earlier "specify an
|
||||
attribute" dead-end is gone.
|
||||
- **Typo → ask, never threshold-guess.** Only an EXACT drug name auto-resolves;
|
||||
a fuzzy match is offered as a question ("Ý bạn là: Metformin?") via
|
||||
`CatalogDrugResolver.suggest(min_score=0.72)`. A completely-wrong name →
|
||||
"Không có thuốc này trong Dược thư Quốc gia." A formulary must not silently
|
||||
answer about a *different* drug than the one meant.
|
||||
- **Autocomplete endpoint** `GET /v1/rag/suggest?q=` (`CatalogDrugResolver.complete`,
|
||||
substring/prefix) — the frontend dropdown that consumes it is still to build.
|
||||
- **BSA calculator** `rag/calculators.py` (Appendix 1, DuBois, tested vs the
|
||||
book's own cells).
|
||||
|
||||
Verification: **ai-service 99 passed, 3 skipped**; live smoke test of all four
|
||||
conversation behaviours plus the monograph/typo/not-supported cases. A
|
||||
refine-loop bug (a refined query dropped the inherited drug and abstained,
|
||||
discarding a good answer) was found in my own code and removed before shipping —
|
||||
clarify + inheritance are the loop's value, retrieval-refine is not, and it is
|
||||
gone from the live path.
|
||||
|
||||
**Owner's next-session directive (recorded in memory `project-llm-cloud-plan`):**
|
||||
stand up the cloud LLM — semantic query embedding (`EMBEDDING_PROVIDER=cohere-v4`,
|
||||
already IAM-permitted) and answer generation (a cheap model, non-Anthropic OK, via
|
||||
a Bedrock Converse adapter, needs its ARN added to `BedrockEmbeddingInvoke`). The
|
||||
offline build was budget/safety-first, not LLM-avoidance; the owner wants the real
|
||||
AI experience next, with `grounding.verify` and the quarantine contract kept ON.
|
||||
|
||||
## 2026-08-05 (late) — Read the source book's own structure; scope + usage-pattern findings (checkpoint before handoff)
|
||||
|
||||
Read the Dược thư 2018 front matter directly (printed p8 "Nội dung", p39
|
||||
"Hướng dẫn sử dụng") to understand what the book is *for* and how clinicians use
|
||||
it — recorded in memory `reference-duoc-thu-2018-structure`. Key facts that
|
||||
reshape the chatbot scope:
|
||||
|
||||
- The book has **three parts**. The corpus is **Part 2 (drug monographs, printed
|
||||
99–1496) ONLY**. **Excluded and clinically important:** Part 1 general chapters
|
||||
(printed 37–98: prescribing in the elderly / hepatic-renal impairment /
|
||||
children / pregnancy-lactation; disease-class guidance for asthma, epilepsy,
|
||||
HIV, antibiotics, TB, hepatitis B, antipsychotics; drug allergy; **poisoning &
|
||||
antidotes**; drug-interaction principles) and Part 3 appendices (printed
|
||||
1497–1528: **body-surface-area calc**, IV admixture, ATC classification). So
|
||||
"how to treat asthma", "antidote for X", "BSA-based dosing" have no data in the
|
||||
index today — a coverage limit, not a retrieval bug.
|
||||
- The 19 monograph fields are fixed and documented on p39; a field is omitted
|
||||
when the book has no info (so a missing section is not necessarily a parse bug).
|
||||
- Field 14 dose is a *general adult+child oral reference dose unless stated*; the
|
||||
clinician adjusts. → the tool supplies reference data, not a prescription.
|
||||
|
||||
Data checks run this session (against `chunks.jsonl`), correcting earlier
|
||||
pessimism:
|
||||
- Indication is searchable: 48 drugs' `chi_dinh` mention "sốt". Reverse lookup
|
||||
(symptom → drugs) is feasible from **content**, but retrieval is drug-first, so
|
||||
not answerable yet.
|
||||
- **mg/kg dosing is in PROSE, not tables**: 574 `lieu_luong` chunks contain
|
||||
"mg/kg", all prose, across **295 drugs**, 473 of them mentioning trẻ em. So the
|
||||
*primary* weight/age dosing (incl. pediatric) is answerable; the 83 quarantined
|
||||
dosing tables are mostly the *supplementary* renal-adjustment tables (49 of
|
||||
those 83 drugs also have mg/kg prose).
|
||||
- Pregnancy dosing is mostly **qualitative**: 670 drugs have a
|
||||
`thoi_ky_mang_thai` section but only ~23 chunks carry a mg figure — the book
|
||||
rarely gives a separate pregnant dose, so answer = pregnancy caution + standard
|
||||
dose, never a fabricated pregnant-specific number.
|
||||
- `drug_id` can be compound (`paracetamol_acetaminophen`); alias resolution must
|
||||
map "paracetamol" → that id.
|
||||
|
||||
**Design consequence discussed with the owner (not yet built):** the "understand"
|
||||
stage must classify the *turn type* (smalltalk / medical query / multi-drug
|
||||
interaction / symptom-indication / out-of-scope / injection-shaped), not just
|
||||
resolve a drug. Refusing a clinician's symptom→drug question as
|
||||
"recommendation_out_of_scope" was wrong for this audience — such questions are
|
||||
indication lookups and should be answered from `chi_dinh`. Multi-drug
|
||||
interaction/contraindication questions need a real PLAN → gather both drugs →
|
||||
synthesize step (the ADR-0007 PLAN node, still unimplemented), and an
|
||||
absence-of-evidence answer must state where it looked, never assert "safe".
|
||||
|
||||
**Session state / not yet done (so a fresh session can resume):** the chat
|
||||
module's domain glue is built and unit-tested (`rag/conversation.py` ports +
|
||||
summariser, `rag/conversational.py` orchestrator + `is_smalltalk`); it is **not**
|
||||
wired to the endpoint. Live wiring (turn-type classifier, loop-around-
|
||||
GroundedAnswerService, Postgres store, `conversation_id` on `/v1/rag/query`,
|
||||
`route.ts`, ChatPanel error state), the P0 audit fixes (§5 context-mixing
|
||||
metadata, §8 Qdrant-error degradation), reverse-indication retrieval, and the
|
||||
table vision-consensus pipeline all remain to do. No code was wired live this
|
||||
session; the behavior spec is still being clarified with the owner before wiring.
|
||||
|
||||
**Owner decision: parse the WHOLE book, re-chunk freely** (not just Part 2
|
||||
monographs). Current corpus covers physical pages **100–1494** only. To add:
|
||||
Part 1 general chapters (physical ~36–97) and Part 3 appendices (~1496–1527);
|
||||
front-matter list (13) and index (1529+) are already used as validation ground
|
||||
truth. Read `segment/detector.py` to ground the plan — **the machinery already
|
||||
generalizes**: a chapter title ("NGỘ ĐỘC VÀ THUỐC GIẢI ĐỘC") has the *same shape*
|
||||
as a monograph title (bold + mostly-upper + short), so `is_monograph_title_candidate`
|
||||
extends by widening the hardcoded `99–1496` range per `content_type`. Only two
|
||||
real changes: (1) parametrize the page range + add a `content_type`
|
||||
(`monograph|chapter|appendix`); (2) chapter/appendix sub-headings are **free-form**
|
||||
("Hô hấp", "Co giật"), not the 19-key vocab, so `detect_section_headings` needs
|
||||
an open-taxonomy mode (bold + short = heading, store the text, no `match_section`
|
||||
requirement). Everything downstream (span extraction, table/formula quarantine,
|
||||
provenance, chunker) is content-type-agnostic and reused → schema v5 adds
|
||||
`content_type` + `chapter_id`. **Gate (CLAUDE.md): the span-routing ledger must
|
||||
account for ALL 1668 pages with `unassigned=0`, not just 99–1496.** Then embed
|
||||
only the NEW chunks (Cohere, pennies, announce first). This is a focused
|
||||
ingestion pass (detector + assembler + chunker + whole-doc re-run + validation),
|
||||
not a one-liner — not attempted this session beyond grounding the plan.
|
||||
|
||||
**Done this session (App 1, self-contained, validated):** `rag/calculators.py`
|
||||
`body_surface_area_m2` replaces Appendix 1's lookup table with the book's DuBois
|
||||
formula (`S = W^0.425 × H^0.725 × 71.84`), tested against three of the book's own
|
||||
table cells (165cm/60kg→1.66, 90cm/10kg→0.50, 170cm/70kg→1.81) — `tests/test_calculators.py`,
|
||||
3 passed. Audit §7 (calculation = tested function, never an LLM).
|
||||
|
||||
## 2026-08-05 (evening) — Conversational orchestrator wired to the existing loop; data-quality audit; budget verified live
|
||||
|
||||
**Chat module (the glue ADR 0007 specified and nothing had called).** Added the
|
||||
two missing conversation ports and their offline defaults to `rag/conversation.py`
|
||||
(`ConversationStore`/`InMemoryConversationStore`, `Summariser`/`DeterministicSummariser`)
|
||||
and the orchestrator `rag/conversational.py` (`ConversationalRagService`). It owns
|
||||
no rules of its own: load state → resolve this turn → inherit gaps from `Focus`
|
||||
→ derive clarify signals from resolver state → `reasoning.run_turn` → update
|
||||
focus, append turns, summarise overflow, save → name any inherited drug. Runs
|
||||
with no LLM/service (collaborators are protocols). `DeterministicSummariser`
|
||||
records only drug/section **labels**, never cell values, so the
|
||||
no-clinical-content-in-summary rule holds by construction rather than by trust —
|
||||
closing the summary-bypasses-grounding hole flagged in review. **11 new tests;
|
||||
full ai-service suite 91 passed, 3 skipped.**
|
||||
|
||||
**Still NOT wired live:** a `TurnResolver` bridge over `CatalogDrugResolver` +
|
||||
`SectionResolver`; bridges from `RetrievalService`→`Retrieve` and the grounded
|
||||
generation path→`Generate`; `PostgresConversationStore` + migration; a
|
||||
`conversation_id` on `/v1/rag/query`; `route.ts` sending it and dropping the
|
||||
hardcoded `intent: fact_lookup`; a `ChatPanel` error state; and the multi-turn
|
||||
eval run. So no claim yet that history/loop improves answers — designed and unit-
|
||||
proven, not measured end to end.
|
||||
|
||||
**Data-quality audit (self-run this session, not quoted from this log).** 684
|
||||
drugs; critical-section coverage is strong — dosing missing 0.1% (1), contra-
|
||||
indication 0.4% (3), indication 0%. **But 83/684 drugs (12%) have their dosing
|
||||
inside a quarantined table**, so a dose query for them returns `VERIFY_PDF`
|
||||
(crop, no number) — the largest answer-quality gap for a clinician audience, and
|
||||
it lands on the single most-asked query. 125 chunks carry a leading `": "`
|
||||
label-leak artifact (93 in `ten_chung_quoc_te`). Vector-path text loss appears
|
||||
contained to 22 flagged lines (completeness of detection unverified). Nobody
|
||||
clinician-side has validated the 8.2M chars against the book — still the largest
|
||||
unmeasured area.
|
||||
|
||||
**Table validation — the instrument that text extraction lacked.** Demonstrated
|
||||
that vision reads a real quarantined dosing table cell-by-cell: GABAPENTIN's
|
||||
renal-adjustment table (printed 706) came back exactly by eye where pdfplumber's
|
||||
text layer could not structure it. Found and corrected a page-index off-by-one
|
||||
in my own render (data `physical_page` N = `doc[N]`, 0-based) — proof that
|
||||
correctness must not depend on trusting coordinates. Strategy, given pharmacists
|
||||
are **end-users, not labelers**: reconstruction powers **retrieval only**; the
|
||||
displayed answer stays crop + page (clinician verifies at point of use).
|
||||
Validation is automated — vision↔geometric consensus + round-trip visual +
|
||||
book invariants — with a per-cell precision-first gate (disagreement → stays
|
||||
crop-only). Not yet built; 151 blocks is small enough for full census.
|
||||
|
||||
**Budget, read live from the billing console** (owner login; `ai-lab-user` has
|
||||
no billing API perms): **$138.50 remaining, entirely AWS promotional credit,
|
||||
not the owner's card**; August bill $0. Deploy target chosen: team k3s, but
|
||||
deferred (mutating a shared cluster). Generation still off (`answer_provider=
|
||||
disabled`) — extractive/verbatim, which is defensible for clinicians; wiring a
|
||||
cheap model (Nova/Haiku via Bedrock Converse) needs its ARN added to the
|
||||
`BedrockEmbeddingInvoke` policy, which today grants invoke on the two embedding
|
||||
models only.
|
||||
|
||||
## 2026-08-05 — An answer layer that cannot state a number the book does not
|
||||
|
||||
Today started by walking the **demo path** rather than the test suite, and the
|
||||
two are not the same thing. The suite was green and the demo was broken.
|
||||
|
||||
**What the walk found, by running it rather than reading it.** The backend
|
||||
answers real Vietnamese questions against the real embedded corpus with real
|
||||
citations and **zero cloud cost** — the section route is a payload filter, not
|
||||
a vector search. `Chống chỉ định của Metformin là gì?` returns the true
|
||||
contraindication text with one citation; `Tương tác thuốc của Warfarin?`
|
||||
returns two. But `Tôi sốt cao, uống Paracetamol được không?` returned **HTTP
|
||||
500**: the similarity fallback reached Bedrock, which is revoked, and
|
||||
`botocore.AccessDeniedException` escaped as an unhandled error. Any question
|
||||
whose phrasing is outside the section phrase table takes that path.
|
||||
|
||||
That crash also **re-verified the cloud shutdown today, live** — the denial
|
||||
came from the service, not from a claim in a document.
|
||||
|
||||
**Four defects, all fixed, all at $0.**
|
||||
|
||||
1. **The 500.** `adapters/embedding.py` now translates provider failures into
|
||||
the domain error `QueryEmbeddingUnavailable`, and `RetrievalService` catches
|
||||
it and abstains with `reason="query_embedding_unavailable"` — deliberately
|
||||
distinct from `insufficient_retrieval_score`, so an outage never reads as an
|
||||
empty corpus. `rag/` still imports no SDK.
|
||||
2. **A default config that does not work.** `config.py` pointed at collection
|
||||
`duoc_thu_chunks`; the real one is `duocthu_v1`. `embedding_provider`
|
||||
defaulted to `disabled`, so `/v1/rag/query` returned 503 on a fresh clone.
|
||||
3. **Neither existing provider was a safe default.** `local-smoke` searches a
|
||||
SHA-256 vector against a Cohere collection — confident, meaningless hits.
|
||||
`cohere-v4` spends the boto3 retry budget (~30s) before failing on a revoked
|
||||
account. Added `SectionOnlyQueryEmbedder`: refuses locally and instantly, so
|
||||
retrieval is confined to the route that measured 16/16.
|
||||
4. **Safety abstention was incidental, not a gate.** Symptom questions abstain
|
||||
with `reason="drug_not_resolved"` — because no drug name was found, not
|
||||
because anything recognised a symptom question. Recorded, not yet fixed.
|
||||
|
||||
**The answer layer now has an LLM, and a check that makes "it does not
|
||||
fabricate" measurable rather than promised.** Previously `rag/answer.py` was
|
||||
extractive: it concatenated retrieved chunks. That is why
|
||||
`Liều Paracetamol cho người lớn?` opened with `5 - 12 tuổi: Trẻ em 12 - 18
|
||||
tuổi:` — raw section text, paediatric doses first, for an adult question.
|
||||
|
||||
Generation is now three layers, and only the third is load-bearing:
|
||||
|
||||
- **Prompt** (`rag/prompt.py`, domain — no SDK): evidence only, figures copied
|
||||
character-for-character, `[n]` citations required, insufficient evidence is a
|
||||
valid answer. Output shape is pinned by `output_config.format`, so a
|
||||
malformed envelope is the provider's error, not our parsing problem.
|
||||
- **Verification** (`rag/grounding.py`, pure domain): every numeric token in
|
||||
the generated answer must appear **exactly** in the evidence, and every `[n]`
|
||||
must resolve. Citation markers are stripped before number extraction so `[2]`
|
||||
is never read as the quantity 2.
|
||||
- **Fail-closed** (`rag/answer.py`): ungrounded number, invalid citation,
|
||||
malformed output, provider outage, or the model itself reporting insufficient
|
||||
evidence — every one falls back to the verbatim source text, which was
|
||||
computed first and is therefore always available.
|
||||
|
||||
**Numbers are compared as strings, and that is the decision worth keeping.**
|
||||
No parsing, no normalisation. `1.500` is 1500 under one reading and 1.5 under
|
||||
another; a normaliser that strips separators maps `7,5` and `75` to the same
|
||||
key, scoring a **tenfold dose error as a match**. Pinned by
|
||||
`test_decimal_separators_are_not_interchangeable`. The same rule refuses
|
||||
`2 g` → `2000 mg`: arithmetically right, but unit conversion is where dosing
|
||||
errors live, so it is refused rather than interpreted.
|
||||
|
||||
Quarantined tables and formulas are **never generated over**. `VERIFY_PDF`
|
||||
returns before generation — those are precisely the blocks whose numbers were
|
||||
not reliably reconstructed, so rephrasing them is the one case where fluency
|
||||
could invent a dose. This keeps ADR 0006's contract intact.
|
||||
|
||||
**Provider chosen on the owner's instruction: AWS Bedrock + Claude.**
|
||||
`adapters/bedrock_claude.py` is the only module naming the `anthropic` SDK,
|
||||
imported lazily. Two provider facts taken from the Anthropic API reference
|
||||
today, not from memory: Bedrock model ids carry an `anthropic.` prefix
|
||||
(`anthropic.claude-opus-5`), and the Messages-API path on Bedrock is
|
||||
`AnthropicBedrockMantle`, **not** the legacy `bedrock-runtime` InvokeModel route
|
||||
the embedding adapter uses. A `stop_reason: "refusal"` is a successful HTTP
|
||||
response with no usable content, so it is routed to the extractive fallback
|
||||
rather than allowed to raise on `content[0]`.
|
||||
|
||||
**This adapter has never been run against Bedrock.** Cloud access is still
|
||||
revoked and no IAM change was made today. `StubAnswerGenerator` exercises the
|
||||
entire path — prompt build, schema parse, grounding check, fallback — with no
|
||||
cloud call, and that is what the end-to-end run below used.
|
||||
|
||||
**Observability, because a dashboard is a better answer than a slide.**
|
||||
`rag/metrics.py` defines the counters in the domain; `adapters/prometheus.py`
|
||||
is the only module naming `prometheus_client`, imported lazily; `/metrics`
|
||||
returns 404 rather than an empty 200 when metrics are off, so a scrape cannot
|
||||
succeed silently with no samples. The headline counter is
|
||||
`duocthu_generation_rejected_total{reason="ungrounded_number"}` — the measured
|
||||
form of the no-fabrication claim. A mismatched label drops the sample instead
|
||||
of raising: metrics must not be able to break a clinical answer.
|
||||
|
||||
`infra/docker/` gains Prometheus and Grafana with a provisioned datasource and
|
||||
dashboard. **Not yet verified running** — the image pull was still in progress
|
||||
when this was written.
|
||||
|
||||
**A section was being served scrambled, and only using the UI found it.**
|
||||
`liều dùng paracetamol` opened mid-sentence on `5 - 12 tuổi:` and buried
|
||||
`Liều lượng: Người lớn:` seven hundred words down. `find_by_section` returned
|
||||
whatever order Qdrant scrolled, and point ids are `uuid5(chunk_id)`, so
|
||||
PARACETAMOL's five dosing parts came back **3, 4, 1, 2, 0** — verified by
|
||||
scrolling the real collection, not inferred. `part_index` was in the payload
|
||||
all along and simply never used. Now sorted by it; a part missing the field
|
||||
sorts last rather than being dropped, because a silently shortened dose list
|
||||
is worse than an unordered one. Pinned by `tests/test_section_order.py`,
|
||||
including the exact 3,4,1,2,0 case. **This is a clinical defect, not a
|
||||
cosmetic one:** a reader who stops partway through stops in the middle of a
|
||||
different population's dose. Every section-routed answer given before today —
|
||||
including the 16/16 golden result — was assembled in this scrambled order;
|
||||
retrieval picked the right chunks, so the measurement stands, but no
|
||||
statement about how those answers *read* survives it.
|
||||
|
||||
**Conversational reasoning RAG: designed in ADR 0007, domain layer built.**
|
||||
`rag/conversation.py` carries `Focus` (drug, section, population, verbosity,
|
||||
each stamped with the turn that set it) and the recent-turn window;
|
||||
`rag/reasoning.py` is the bounded loop. Both are pure domain and run with no
|
||||
provider, which is the point: *which drug is this still about* must be
|
||||
deterministic, not inferred.
|
||||
|
||||
Three rules make inheritance safe in a formulary, each pinned by a test: an
|
||||
explicitly named drug always beats context; focus older than six turns is
|
||||
dropped rather than carried, because a stale drug is a wrong-drug answer, not
|
||||
context; and any answer built on an inherited drug must name it.
|
||||
|
||||
The loop's uncertainty signal is **not** a model confidence score. It is the
|
||||
resolver states that already existed and previously dead-ended into `abstain`
|
||||
— ambiguous drug, unresolved attribute, multi-attribute question — which now
|
||||
produce a clarifying question. Deterministic, testable, and explainable to a
|
||||
reviewer in a way that "the model felt 0.73 sure" is not. A clarify signal
|
||||
short-circuits before any budget is spent, verified by asserting the budget is
|
||||
untouched and neither retriever nor generator was called.
|
||||
|
||||
Budgets are decremented **before** the call they pay for, so exhaustion
|
||||
degrades to the best answer so far. A retrieval round is bought only by a
|
||||
*named* gap with a genuinely new query: `test_an_unnamed_gap_does_not_buy_a_round`
|
||||
and `test_a_refinement_that_changes_nothing_stops_the_loop` are the guards
|
||||
against a loop that spins on a feeling or re-issues the same query.
|
||||
|
||||
`Golden Dataset/golden_multiturn_v1.csv` is new — 8 conversations, 19 turns,
|
||||
6 of them inheritance-dependent. The existing golden file is single-turn by
|
||||
construction and can measure none of this. Includes the adversarial turns: a
|
||||
follow-up after a refused fake drug (must not borrow a drug from elsewhere),
|
||||
and a follow-up after a symptom question (must not inherit treatment intent).
|
||||
|
||||
**Not yet wired:** the loop is not called by `GroundedAnswerService` or the
|
||||
router, there is no `PostgresConversationStore`, and no evaluation run over the
|
||||
multi-turn file has been performed — so no claim is made that history or the
|
||||
loop improves answers. The design states how that will be measured; it has not
|
||||
been measured.
|
||||
|
||||
**LangChain was considered and rejected.** The repo already has the ports and
|
||||
adapters LangChain would supply, retrieval is already measured, and the
|
||||
guardrail is already domain code. Adopting it a week before a review would
|
||||
rewrite the working part for no measured capability gain.
|
||||
|
||||
Verification actually run: ai-service **56 passed, 3 skipped** (37 + 3 before,
|
||||
+19); ingestion **296 passed**, checked for regression, unchanged; `duocthu_v1`
|
||||
holds **15,100 points** at 1024-dim Cosine, matching the manifest; live service
|
||||
against the real collection answered three clinical questions with citations
|
||||
and abstained on six of the seven safety probes; `/metrics` scraped and
|
||||
returned `duocthu_generation_served_total 2.0` and
|
||||
`duocthu_abstention_total{reason="drug_not_resolved"} 1.0`.
|
||||
|
||||
Not established, and load-bearing for the demo: **`apps/web` is still entirely
|
||||
mocked** — `packages/api-client/src/sendChatMessage.ts:8` returns
|
||||
`buildMockResponse(content)` and the whole frontend contains no HTTP call to
|
||||
the backend, so the working API and the working UI are not connected;
|
||||
`api-gateway`, `chat-service` and `auth-service` hold **0 source files**; the
|
||||
Bedrock generator has never been invoked; `intent` is still supplied by the
|
||||
caller, so the recommendation gate depends on the client declaring it honestly;
|
||||
and the Prometheus/Grafana stack has not been seen running.
|
||||
|
||||
## 2026-08-04 (evening) — Section routing: contraindication retrieval goes from 0.05 to 1.00, at zero cloud cost
|
||||
|
||||
The retrieval defect measured earlier today is fixed by routing rather than by
|
||||
|
||||
@@ -10,25 +10,25 @@
|
||||
"Resource": "*"
|
||||
},
|
||||
{
|
||||
"Sid": "ReadTheTwoEmbeddingModels",
|
||||
"Sid": "ReadModelMetadata",
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"bedrock:GetFoundationModel"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:bedrock:us-east-1::foundation-model/amazon.titan-embed-text-v2:0",
|
||||
"arn:aws:bedrock:us-east-1::foundation-model/cohere.embed-v4:0"
|
||||
]
|
||||
"Resource": "*"
|
||||
},
|
||||
{
|
||||
"Sid": "InvokeOnlyTheTwoEmbeddingModels",
|
||||
"Sid": "InvokeEmbeddingGenerationRerank",
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"bedrock:InvokeModel"
|
||||
"bedrock:InvokeModel",
|
||||
"bedrock:InvokeModelWithResponseStream"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:bedrock:us-east-1::foundation-model/amazon.titan-embed-text-v2:0",
|
||||
"arn:aws:bedrock:us-east-1::foundation-model/cohere.embed-v4:0"
|
||||
"arn:aws:bedrock:us-east-1::foundation-model/cohere.embed-v4:0",
|
||||
"arn:aws:bedrock:us-east-1::foundation-model/deepseek.v3.2",
|
||||
"arn:aws:bedrock:us-east-1::foundation-model/cohere.rerank-v3-5:0"
|
||||
]
|
||||
}
|
||||
]
|
||||
|
||||
@@ -30,6 +30,38 @@ services:
|
||||
volumes:
|
||||
- redis-data:/data
|
||||
|
||||
# Observability. `ai-service` runs on the host during development, so
|
||||
# Prometheus reaches it via host.docker.internal rather than a service name;
|
||||
# when ai-service moves into this compose file, change the target to
|
||||
# `ai-service:8000` and drop the extra_hosts entry.
|
||||
prometheus:
|
||||
image: prom/prometheus:latest
|
||||
ports:
|
||||
- "9090:9090"
|
||||
volumes:
|
||||
- ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
|
||||
- prometheus-data:/prometheus
|
||||
extra_hosts:
|
||||
- "host.docker.internal:host-gateway"
|
||||
|
||||
grafana:
|
||||
image: grafana/grafana:latest
|
||||
ports:
|
||||
- "3002:3000"
|
||||
environment:
|
||||
# Local development only. The dashboard is the demo surface, and a login
|
||||
# wall in front of it during a review is friction with no security value
|
||||
# on a laptop-local stack. Do not carry this into the k3s deployment.
|
||||
GF_AUTH_ANONYMOUS_ENABLED: "true"
|
||||
GF_AUTH_ANONYMOUS_ORG_ROLE: Admin
|
||||
GF_AUTH_DISABLE_LOGIN_FORM: "true"
|
||||
volumes:
|
||||
- ./grafana/provisioning:/etc/grafana/provisioning:ro
|
||||
- ./grafana/dashboards:/var/lib/grafana/dashboards:ro
|
||||
- grafana-data:/var/lib/grafana
|
||||
depends_on:
|
||||
- prometheus
|
||||
|
||||
# ai-service:
|
||||
# build: ../../apps/ai-service
|
||||
# env_file: ../../apps/ai-service/.env
|
||||
@@ -66,3 +98,5 @@ volumes:
|
||||
postgres-data:
|
||||
qdrant-data:
|
||||
redis-data:
|
||||
prometheus-data:
|
||||
grafana-data:
|
||||
|
||||
@@ -0,0 +1,215 @@
|
||||
{
|
||||
"uid": "duocthu-grounding",
|
||||
"title": "Dược thư — Grounding & Retrieval",
|
||||
"tags": ["duocthu", "rag"],
|
||||
"timezone": "browser",
|
||||
"schemaVersion": 39,
|
||||
"version": 1,
|
||||
"refresh": "10s",
|
||||
"time": { "from": "now-1h", "to": "now" },
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"type": "stat",
|
||||
"title": "Số lần LLM bịa số và bị chặn",
|
||||
"description": "Generations discarded because they stated a figure that does not appear character-for-character in the cited source. This is the measured form of the claim that the answer layer cannot invent a dose. Non-zero is not a failure — it is the guardrail doing its job.",
|
||||
"gridPos": { "h": 6, "w": 6, "x": 0, "y": 0 },
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum(duocthu_generation_rejected_total{reason=\"ungrounded_number\"})",
|
||||
"legendFormat": "blocked"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short",
|
||||
"decimals": 0,
|
||||
"color": { "mode": "thresholds" },
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "orange", "value": 1 }
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"graphMode": "area",
|
||||
"textMode": "value",
|
||||
"colorMode": "value",
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "stat",
|
||||
"title": "Tỷ lệ câu trả lời có kiểm chứng",
|
||||
"description": "Share of served answers that were LLM-generated and passed grounding verification. The remainder are served as verbatim source text — safe, just less readable.",
|
||||
"gridPos": { "h": 6, "w": 6, "x": 6, "y": 0 },
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum(duocthu_generation_served_total) / clamp_min(sum(duocthu_generation_served_total) + sum(duocthu_answer_extractive_total), 1)",
|
||||
"legendFormat": "verified"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percentunit",
|
||||
"decimals": 1,
|
||||
"min": 0,
|
||||
"max": 1,
|
||||
"color": { "mode": "thresholds" },
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "red", "value": null },
|
||||
{ "color": "orange", "value": 0.5 },
|
||||
{ "color": "green", "value": 0.8 }
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"graphMode": "area",
|
||||
"textMode": "value",
|
||||
"colorMode": "value",
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"type": "stat",
|
||||
"title": "Tỷ lệ từ chối trả lời",
|
||||
"description": "Share of requests the system declined. A medical reference tool is expected to abstain — symptom questions, invented drug names and out-of-scope asks all land here by design.",
|
||||
"gridPos": { "h": 6, "w": 6, "x": 12, "y": 0 },
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum(duocthu_abstention_total) / clamp_min(sum(duocthu_abstention_total) + sum(duocthu_generation_served_total) + sum(duocthu_answer_extractive_total), 1)",
|
||||
"legendFormat": "abstained"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percentunit",
|
||||
"decimals": 1,
|
||||
"min": 0,
|
||||
"max": 1,
|
||||
"color": { "mode": "fixed", "fixedColor": "blue" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"graphMode": "area",
|
||||
"textMode": "value",
|
||||
"colorMode": "value",
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"type": "stat",
|
||||
"title": "Câu đi đúng đường section route",
|
||||
"description": "Retrievals resolved by section filter — the route measured at 16/16 on human-written questions. The remainder fall back to similarity, measured at hit@1 0.544.",
|
||||
"gridPos": { "h": 6, "w": 6, "x": 18, "y": 0 },
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum(duocthu_retrieval_route_total{route=\"section\"}) / clamp_min(sum(duocthu_retrieval_route_total), 1)",
|
||||
"legendFormat": "section"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percentunit",
|
||||
"decimals": 1,
|
||||
"min": 0,
|
||||
"max": 1,
|
||||
"color": { "mode": "thresholds" },
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "red", "value": null },
|
||||
{ "color": "orange", "value": 0.6 },
|
||||
{ "color": "green", "value": 0.85 }
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"graphMode": "area",
|
||||
"textMode": "value",
|
||||
"colorMode": "value",
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"type": "timeseries",
|
||||
"title": "Vì sao một bản sinh bị loại",
|
||||
"description": "Every reason a generation was discarded before reaching a clinician. `ungrounded_number` is a fabrication caught; `provider_unavailable` is an outage; `evidence_insufficient` is the model correctly declining.",
|
||||
"gridPos": { "h": 9, "w": 12, "x": 0, "y": 6 },
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by (reason) (rate(duocthu_generation_rejected_total[5m]))",
|
||||
"legendFormat": "{{reason}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "reqps",
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineWidth": 2,
|
||||
"fillOpacity": 12,
|
||||
"showPoints": "never",
|
||||
"stacking": { "mode": "normal", "group": "A" }
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["sum"] },
|
||||
"tooltip": { "mode": "multi", "sort": "desc" }
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"type": "timeseries",
|
||||
"title": "Vì sao hệ thống từ chối trả lời",
|
||||
"description": "Abstentions by the reason retrieval gave. `drug_not_resolved` dominating means most refusals are questions that never named a drug in the formulary — symptom questions and invented names.",
|
||||
"gridPos": { "h": 9, "w": 12, "x": 12, "y": 6 },
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by (reason) (rate(duocthu_abstention_total[5m]))",
|
||||
"legendFormat": "{{reason}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "reqps",
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineWidth": 2,
|
||||
"fillOpacity": 12,
|
||||
"showPoints": "never",
|
||||
"stacking": { "mode": "normal", "group": "A" }
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["sum"] },
|
||||
"tooltip": { "mode": "multi", "sort": "desc" }
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
apiVersion: 1
|
||||
|
||||
providers:
|
||||
- name: duocthu
|
||||
orgId: 1
|
||||
folder: Dược thư
|
||||
type: file
|
||||
disableDeletion: false
|
||||
updateIntervalSeconds: 30
|
||||
allowUiUpdates: true
|
||||
options:
|
||||
path: /var/lib/grafana/dashboards
|
||||
foldersFromFilesStructure: false
|
||||
@@ -0,0 +1,9 @@
|
||||
apiVersion: 1
|
||||
|
||||
datasources:
|
||||
- name: Prometheus
|
||||
type: prometheus
|
||||
access: proxy
|
||||
url: http://prometheus:9090
|
||||
isDefault: true
|
||||
editable: false
|
||||
@@ -0,0 +1,16 @@
|
||||
global:
|
||||
scrape_interval: 15s
|
||||
evaluation_interval: 15s
|
||||
|
||||
scrape_configs:
|
||||
- job_name: ai-service
|
||||
metrics_path: /metrics
|
||||
static_configs:
|
||||
# ai-service runs on the host during development. `/metrics` returns 404
|
||||
# when METRICS_ENABLED is false or prometheus-client is not installed, so
|
||||
# a target that is UP-but-404 means the app is running without metrics —
|
||||
# not that the app is down.
|
||||
- targets: ["host.docker.internal:8079"]
|
||||
labels:
|
||||
service: ai-service
|
||||
env: local
|
||||
@@ -8,9 +8,13 @@ export function buildMockResponse(userContent: string): SendMessageResponse {
|
||||
message: {
|
||||
id,
|
||||
role: "assistant",
|
||||
// Deliberately carries no dose. A fixture that states a plausible
|
||||
// milligram figure is indistinguishable from a real answer in a
|
||||
// screenshot, and screenshots outlive the code that produced them.
|
||||
content:
|
||||
`(Mock) Paracetamol được chỉ định để giảm đau, hạ sốt. Liều thường dùng ở người ` +
|
||||
`lớn là 500-1000mg mỗi 4-6 giờ, tối đa 4g/ngày. Đây là dữ liệu giả lập cho câu hỏi: "${userContent}".`,
|
||||
`⚠️ DỮ LIỆU GIẢ LẬP — KHÔNG PHẢI NỘI DUNG DƯỢC THƯ. Đây là phản hồi mẫu ` +
|
||||
`dùng khi backend chưa sẵn sàng, không chứa số liệu y khoa và không được ` +
|
||||
`dùng để tra cứu. Câu hỏi nhận được: "${userContent}".`,
|
||||
citations: [
|
||||
{
|
||||
drugName: "PARACETAMOL",
|
||||
|
||||
@@ -1,9 +1,43 @@
|
||||
import type { SendMessageResponse } from "@duoc-thu/shared-types";
|
||||
import { buildMockResponse } from "./mockFixtures";
|
||||
|
||||
const MOCK_LATENCY_MS = 400;
|
||||
/**
|
||||
* Sends a question to the real RAG backend through the app's own route
|
||||
* handler, which holds the service URL and maps the response.
|
||||
*
|
||||
* The mock is retained but is now opt-in via `NEXT_PUBLIC_USE_MOCK_CHAT`, and
|
||||
* every mocked answer is labelled as such in its own text. Silently falling
|
||||
* back to a plausible-looking fake answer is the exact failure this project
|
||||
* cannot afford: a fabricated dose that looks like a real one.
|
||||
*/
|
||||
const USE_MOCK = process.env.NEXT_PUBLIC_USE_MOCK_CHAT === "true";
|
||||
|
||||
export async function sendChatMessage(content: string): Promise<SendMessageResponse> {
|
||||
await new Promise((resolve) => setTimeout(resolve, MOCK_LATENCY_MS));
|
||||
return buildMockResponse(content);
|
||||
export class ChatUnavailableError extends Error {
|
||||
constructor(readonly status: number) {
|
||||
super(`Chat backend unavailable (${status})`);
|
||||
this.name = "ChatUnavailableError";
|
||||
}
|
||||
}
|
||||
|
||||
export async function sendChatMessage(
|
||||
content: string,
|
||||
conversationId?: string
|
||||
): Promise<SendMessageResponse> {
|
||||
if (USE_MOCK) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 400));
|
||||
return buildMockResponse(content);
|
||||
}
|
||||
|
||||
const response = await fetch("/api/chat", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ content, conversationId }),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
// Surfaced to the user as an error state, never as an answer.
|
||||
throw new ChatUnavailableError(response.status);
|
||||
}
|
||||
|
||||
return (await response.json()) as SendMessageResponse;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user