Fix ai-service Dockerfile: bake in drug_entities.json, override its path
This commit is contained in:
@@ -0,0 +1,34 @@
|
||||
FROM python:3.12-slim
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends gcc \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY apps/ai-service/ ./
|
||||
# config.py's `entities_path` default assumes a full monorepo checkout
|
||||
# (parents[2] from apps/ai-service/config.py = repo root); this image only
|
||||
# has apps/ai-service flattened into /app, so the file is baked in here and
|
||||
# ENTITIES_PATH in .env.prod points at it instead.
|
||||
COPY ingestion/data/verified/drug_entities.json ./ingestion_data/drug_entities.json
|
||||
|
||||
# Flat module layout (rag/, adapters/, routers/...), not an installable
|
||||
# package — setuptools' auto-discovery rejects "multiple top-level packages"
|
||||
# for `pip install .`, so the runtime deps are listed directly instead,
|
||||
# mirroring pyproject.toml's [project.dependencies] + the metrics/generation
|
||||
# extras + boto3 (used for Bedrock, not declared in pyproject.toml).
|
||||
RUN pip install --no-cache-dir \
|
||||
"fastapi>=0.115,<1" \
|
||||
"httpx>=0.27,<1" \
|
||||
"psycopg[binary]>=3.2,<4" \
|
||||
"pydantic-settings>=2.6,<3" \
|
||||
"qdrant-client>=1.7,<2" \
|
||||
"uvicorn[standard]>=0.30,<1" \
|
||||
"prometheus-client>=0.20,<1" \
|
||||
"anthropic>=0.112,<1" \
|
||||
"boto3"
|
||||
|
||||
EXPOSE 8000
|
||||
|
||||
CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"]
|
||||
@@ -72,7 +72,12 @@ class BedrockClaudeAnswerGenerator:
|
||||
if self._client is None:
|
||||
from anthropic import AnthropicBedrockMantle
|
||||
|
||||
self._client = AnthropicBedrockMantle(aws_region=self._region)
|
||||
# max_retries: the SDK's own default (2) is lower than the retry
|
||||
# budget given the other Bedrock adapters (bedrock_converse.py,
|
||||
# embedding.py) after a real throttling burst measured live
|
||||
# 2026-08-07 — matched here for consistency, in case this
|
||||
# provider is ever selected instead of bedrock-converse.
|
||||
self._client = AnthropicBedrockMantle(aws_region=self._region, max_retries=4)
|
||||
return self._client
|
||||
|
||||
def generate(self, system: str, user: str, schema: dict) -> str:
|
||||
|
||||
@@ -94,7 +94,15 @@ class BedrockConverseAnswerGenerator:
|
||||
config=Config(
|
||||
connect_timeout=10,
|
||||
read_timeout=60,
|
||||
retries={"max_attempts": 3, "mode": "standard"},
|
||||
# "adaptive" was tried 2026-08-07 and reverted same day: its
|
||||
# client-side rate limiter remembers "throttled" across
|
||||
# requests and paces even unrelated, otherwise-healthy calls
|
||||
# down after a burst — turned a single answerable turn's
|
||||
# baseline ~9s into 1-5 MINUTES following this session's own
|
||||
# heavy adversarial test traffic. "standard" retries each
|
||||
# call independently, no shared state to get stuck in a bad
|
||||
# regime. max_attempts alone (3->4) is kept.
|
||||
retries={"max_attempts": 4, "mode": "standard"},
|
||||
),
|
||||
)
|
||||
return self._client
|
||||
@@ -168,7 +176,15 @@ class BedrockCohereReranker:
|
||||
config=Config(
|
||||
connect_timeout=10,
|
||||
read_timeout=30,
|
||||
retries={"max_attempts": 3, "mode": "standard"},
|
||||
# "adaptive" was tried 2026-08-07 and reverted same day: its
|
||||
# client-side rate limiter remembers "throttled" across
|
||||
# requests and paces even unrelated, otherwise-healthy calls
|
||||
# down after a burst — turned a single answerable turn's
|
||||
# baseline ~9s into 1-5 MINUTES following this session's own
|
||||
# heavy adversarial test traffic. "standard" retries each
|
||||
# call independently, no shared state to get stuck in a bad
|
||||
# regime. max_attempts alone (3->4) is kept.
|
||||
retries={"max_attempts": 4, "mode": "standard"},
|
||||
),
|
||||
)
|
||||
return self._client
|
||||
|
||||
@@ -74,7 +74,10 @@ class BedrockCohereQueryEmbedder:
|
||||
config=Config(
|
||||
connect_timeout=10,
|
||||
read_timeout=30,
|
||||
retries={"max_attempts": 3, "mode": "standard"},
|
||||
# "adaptive" was tried 2026-08-07 and reverted same day — see
|
||||
# the matching comment in adapters/bedrock_converse.py for the
|
||||
# measured 1-5-minute regression it caused. "standard" kept.
|
||||
retries={"max_attempts": 4, "mode": "standard"},
|
||||
),
|
||||
)
|
||||
return self._client
|
||||
|
||||
@@ -93,3 +93,59 @@ class PostgresTraceRepository:
|
||||
decision=row[4], reason=row[5], resolved_drug_id=row[6],
|
||||
citations=tuple(row[7]), created_at=row[8],
|
||||
)
|
||||
|
||||
|
||||
class PostgresConversationStore:
|
||||
"""Durable, cross-worker replacement for `RagAgent`'s in-process
|
||||
`dict[str, list[str]]` history — the F-08 gap named in ADR 0008: history
|
||||
was lost on restart and not shared across workers.
|
||||
|
||||
Append-only, one connection per call — the same tradeoffs as
|
||||
`PostgresTraceRepository` (F-09: real pooling is a further improvement
|
||||
not made here) and the same `connect_timeout=5` for the same reason
|
||||
(an unreachable-but-not-refusing host hangs on the OS TCP timeout
|
||||
otherwise, defeating a caller's fail-open try/except just as completely
|
||||
as no try/except at all).
|
||||
|
||||
Raises on any error rather than swallowing it — `RagAgent` is the
|
||||
caller that decides fail-open (lose this turn's memory, not the
|
||||
response), matching how `routers/rag.py` already wraps
|
||||
`PostgresTraceRepository.save()` rather than the repository hiding its
|
||||
own failures.
|
||||
"""
|
||||
|
||||
def __init__(self, dsn: str) -> None:
|
||||
self._dsn = dsn
|
||||
|
||||
def migrate(self, migration_path: Path) -> None:
|
||||
import psycopg
|
||||
|
||||
statement = migration_path.read_text(encoding="utf-8")
|
||||
with psycopg.connect(self._dsn, connect_timeout=5) as connection:
|
||||
connection.execute(statement)
|
||||
|
||||
def recent(self, conversation_id: str, limit: int) -> list[str]:
|
||||
"""The last `limit` lines, oldest first — matches the ordering the
|
||||
understanding LLM prompt already expects ("LỊCH SỬ HỘI THOẠI (cũ ->
|
||||
mới)")."""
|
||||
import psycopg
|
||||
|
||||
with psycopg.connect(self._dsn, connect_timeout=5) as connection:
|
||||
rows = connection.execute(
|
||||
"""
|
||||
SELECT line FROM rag_conversation_turn
|
||||
WHERE conversation_id = %s ORDER BY id DESC LIMIT %s
|
||||
""",
|
||||
(conversation_id, limit),
|
||||
).fetchall()
|
||||
return [row[0] for row in reversed(rows)]
|
||||
|
||||
def append(self, conversation_id: str, line: str) -> None:
|
||||
import psycopg
|
||||
|
||||
with psycopg.connect(self._dsn, connect_timeout=5) as connection:
|
||||
connection.execute(
|
||||
"INSERT INTO rag_conversation_turn (conversation_id, line) "
|
||||
"VALUES (%s, %s)",
|
||||
(conversation_id, line),
|
||||
)
|
||||
|
||||
@@ -233,6 +233,118 @@ class QdrantRetriever:
|
||||
hits.append(SearchHit(_document(payload), 1.0))
|
||||
return hits
|
||||
|
||||
def find_by_indication(self, indication_text: str, limit: int) -> list[SearchHit]:
|
||||
"""Reverse lookup: every drug whose `chi_dinh` text mentions the given
|
||||
symptom/indication, keyword-matched. Deterministic, no fabrication
|
||||
risk — the same "exact match wins, no-match-means-None" philosophy
|
||||
`find_by_section` already uses, applied across drugs instead of
|
||||
within one. `limit` caps how many DRUGS are returned (one hit per
|
||||
drug, first match wins), not how many chunks are scanned — a common
|
||||
symptom can match far more drugs than is useful to show.
|
||||
|
||||
Prose only: a `block_descriptor` chunk carries no real `chi_dinh`
|
||||
text (its text is built only from metadata per the quarantine
|
||||
contract), so keyword-matching it would be meaningless.
|
||||
|
||||
`indication_text`, normalized, must appear as a CONTIGUOUS,
|
||||
word-boundary-anchored phrase in the chunk's text — not a plain
|
||||
substring (risks a false positive inside an unrelated longer word
|
||||
after diacritic-stripping) and not a scattered bag-of-words match
|
||||
either. Found live 2026-08-07: a token-SUBSET match (every word
|
||||
present *somewhere*, any order) let a long nonsense phrase built
|
||||
from common filler words ("bệnh chưa từng ghi nhận trong sách…")
|
||||
false-positive against real chi_dinh text, since words that common
|
||||
appear scattered through nearly everything — it reached generation
|
||||
before being caught, instead of failing here where it's cheap. A
|
||||
genuine paraphrase that doesn't share the book's exact wording is
|
||||
`search_indication`'s job (semantic), not this one's (lexical).
|
||||
"""
|
||||
import re
|
||||
|
||||
from qdrant_client.models import FieldCondition, Filter, MatchValue
|
||||
|
||||
from rag.text import normalize_name
|
||||
|
||||
needle = normalize_name(indication_text)
|
||||
if not needle:
|
||||
return []
|
||||
needle_pattern = re.compile(rf"(?:^| ){re.escape(needle)}(?:$| )")
|
||||
|
||||
scroll_filter = Filter(
|
||||
must=[
|
||||
FieldCondition(key="section_key", match=MatchValue(value="chi_dinh")),
|
||||
FieldCondition(key="chunk_kind", match=MatchValue(value="prose")),
|
||||
]
|
||||
)
|
||||
hits: list[SearchHit] = []
|
||||
seen_drugs: set[str] = set()
|
||||
offset = None
|
||||
while True:
|
||||
points, offset = self._client.scroll(
|
||||
collection_name=self._collection_name,
|
||||
scroll_filter=scroll_filter,
|
||||
limit=256,
|
||||
offset=offset,
|
||||
with_payload=True,
|
||||
)
|
||||
for point in points:
|
||||
payload = dict(point.payload or {})
|
||||
drug_id = payload.get("drug_id")
|
||||
if drug_id in seen_drugs:
|
||||
continue
|
||||
text = normalize_name(payload.get("text", ""))
|
||||
if not needle_pattern.search(f" {text} "):
|
||||
continue
|
||||
seen_drugs.add(drug_id)
|
||||
hits.append(SearchHit(_document(payload), 1.0))
|
||||
if len(hits) >= limit:
|
||||
return hits
|
||||
if offset is None:
|
||||
break
|
||||
return hits
|
||||
|
||||
def search_indication(self, query: str, limit: int) -> list[SearchHit]:
|
||||
"""Dense-vector fallback for `find_by_indication` when no exact
|
||||
keyword phrase match exists — catches paraphrases ("sốt cao" vs
|
||||
"thân nhiệt tăng") a literal phrase match cannot. Deliberately narrow
|
||||
(`section_key=chi_dinh` only, never the whole corpus) so this stays
|
||||
a targeted fallback for one specific gap, not a return to unranked
|
||||
similarity search — see ADR 0008 on why the live path otherwise
|
||||
avoids `search()`. One hit per drug, highest-scoring chunk kept
|
||||
(Qdrant returns points pre-sorted by score)."""
|
||||
from qdrant_client.models import FieldCondition, Filter, MatchValue
|
||||
|
||||
vector = list(self._embedder.embed_query(query))
|
||||
if len(vector) != self._embedder.dimensions:
|
||||
raise ValueError(
|
||||
f"query vector has {len(vector)} dimensions; "
|
||||
f"expected {self._embedder.dimensions}"
|
||||
)
|
||||
points = self._client.search(
|
||||
collection_name=self._collection_name,
|
||||
query_vector=vector,
|
||||
query_filter=Filter(
|
||||
must=[
|
||||
FieldCondition(key="section_key", match=MatchValue(value="chi_dinh")),
|
||||
FieldCondition(key="chunk_kind", match=MatchValue(value="prose")),
|
||||
]
|
||||
),
|
||||
limit=limit * 4,
|
||||
with_payload=True,
|
||||
)
|
||||
hits: list[SearchHit] = []
|
||||
seen_drugs: set[str] = set()
|
||||
for point in points:
|
||||
payload = dict(point.payload or {})
|
||||
drug_id = payload.get("drug_id")
|
||||
if drug_id in seen_drugs:
|
||||
continue
|
||||
seen_drugs.add(drug_id)
|
||||
hits.append(SearchHit(_document(payload), float(point.score)))
|
||||
if len(hits) >= limit:
|
||||
break
|
||||
return hits
|
||||
|
||||
|
||||
class QdrantParentStore:
|
||||
def __init__(self, client: Any, collection_name: str) -> None:
|
||||
|
||||
@@ -3,7 +3,7 @@ from __future__ import annotations
|
||||
from qdrant_client import QdrantClient
|
||||
|
||||
from adapters.embedding import BedrockCohereQueryEmbedder
|
||||
from adapters.postgres import PostgresTraceRepository
|
||||
from adapters.postgres import PostgresConversationStore, PostgresTraceRepository
|
||||
from adapters.qdrant import QdrantParentStore, QdrantRetriever
|
||||
from config import Settings
|
||||
from rag.agent import RagAgent
|
||||
@@ -180,5 +180,8 @@ def build_runtime(settings: Settings):
|
||||
retrieval=retrieval,
|
||||
answers=answers,
|
||||
autocomplete=resolver,
|
||||
max_wall_clock_ms=settings.max_wall_clock_ms,
|
||||
max_llm_calls_per_turn=settings.max_llm_calls_per_turn,
|
||||
store=PostgresConversationStore(settings.postgres_dsn),
|
||||
)
|
||||
return answers, agent, trace_writer, metrics
|
||||
|
||||
@@ -40,6 +40,13 @@ class Settings(BaseSettings):
|
||||
Path(__file__).resolve().parents[2]
|
||||
/ "ingestion/data/verified/drug_entities.json"
|
||||
)
|
||||
# F-08: a per-turn budget across RagAgent's sequential Bedrock calls
|
||||
# (understand, sufficiency, generate, up to 2 entailment retries).
|
||||
# Defaults sized with headroom above what a normal turn measures live
|
||||
# (~8-9s, 4-5 calls) — see rag/agent.py's MAX_WALL_CLOCK_MS/
|
||||
# MAX_LLM_CALLS_PER_TURN for the full rationale.
|
||||
max_wall_clock_ms: int = 20_000
|
||||
max_llm_calls_per_turn: int = 8
|
||||
|
||||
|
||||
@lru_cache
|
||||
|
||||
@@ -1,13 +1,20 @@
|
||||
from pathlib import Path
|
||||
|
||||
from adapters.postgres import PostgresTraceRepository
|
||||
from adapters.postgres import PostgresConversationStore, PostgresTraceRepository
|
||||
from config import get_settings
|
||||
|
||||
|
||||
def main() -> None:
|
||||
migration = Path(__file__).parent / "migrations/001_rag_retrieval_trace.sql"
|
||||
PostgresTraceRepository(get_settings().postgres_dsn).migrate(migration)
|
||||
print(f"Applied {migration.name}")
|
||||
dsn = get_settings().postgres_dsn
|
||||
migrations_dir = Path(__file__).parent / "migrations"
|
||||
|
||||
trace_migration = migrations_dir / "001_rag_retrieval_trace.sql"
|
||||
PostgresTraceRepository(dsn).migrate(trace_migration)
|
||||
print(f"Applied {trace_migration.name}")
|
||||
|
||||
conversation_migration = migrations_dir / "002_rag_conversation_turn.sql"
|
||||
PostgresConversationStore(dsn).migrate(conversation_migration)
|
||||
print(f"Applied {conversation_migration.name}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
-- F-08 durable conversation history: replaces RagAgent's in-process dict
|
||||
-- (lost on restart, not shared across workers). Append-only; `id`'s
|
||||
-- insertion order is the "oldest -> newest" ordering the understanding LLM
|
||||
-- prompt already expects, no separate turn-index column needed.
|
||||
CREATE TABLE IF NOT EXISTS rag_conversation_turn (
|
||||
id bigserial PRIMARY KEY,
|
||||
conversation_id text NOT NULL,
|
||||
line text NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS rag_conversation_turn_conv_idx
|
||||
ON rag_conversation_turn (conversation_id, id);
|
||||
+279
-29
@@ -17,17 +17,50 @@ This module owns routing only; it states no medical fact of its own.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Protocol
|
||||
|
||||
from .answer import Citation, GroundedAnswerService
|
||||
from .budget import RequestBudget
|
||||
from .models import EvidenceDecision, RetrievalResult
|
||||
from .policy import looks_non_human
|
||||
from .service import RetrievalService
|
||||
from .understanding import QueryFrame, QueryUnderstander
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
TUONG_TAC = "tuong_tac_thuoc"
|
||||
HISTORY_TURNS = 6
|
||||
# F-08: measured live 2026-08-07, a normal answerable turn makes 4 sequential
|
||||
# Bedrock calls (understand, sufficiency, generate, entailment) and costs
|
||||
# ~8-9s; a noisy entailment retry adds a 5th. Defaults sized with headroom
|
||||
# above that measured normal case, not at its exact edge, so ordinary
|
||||
# traffic never trips the budget — it exists for the pathological case
|
||||
# (a stuck/slow call, or an unexpectedly long retry chain), not to shave
|
||||
# time off the common path.
|
||||
MAX_WALL_CLOCK_MS = 20_000
|
||||
MAX_LLM_CALLS_PER_TURN = 8
|
||||
# Found live 2026-08-07 (50-question hand-typed browser audit): the
|
||||
# understanding LLM can get stuck re-asking the same (or a near-identical)
|
||||
# clarify question forever — reproduced 3 times independently, one case
|
||||
# never converged after 5 real answered turns. `understanding.py`'s prior-
|
||||
# frame merge (F-11) fixes most of the underlying cause, but a code-level
|
||||
# backstop is still needed: nothing bounds a model that just keeps deciding
|
||||
# needs_clarify=true no matter what. After this many CONSECUTIVE clarify
|
||||
# turns on the same conversation, force a hard stop instead of asking again.
|
||||
MAX_CONSECUTIVE_CLARIFY = 4
|
||||
|
||||
|
||||
class ConversationStore(Protocol):
|
||||
"""Durable, cross-worker alternative to the in-process history dict —
|
||||
the gap ADR 0008 names as still open. Satisfied by
|
||||
`adapters.postgres.PostgresConversationStore`; entirely optional — with
|
||||
none configured, `RagAgent` behaves exactly as before this existed."""
|
||||
|
||||
def recent(self, conversation_id: str, limit: int) -> list[str]: ...
|
||||
def append(self, conversation_id: str, line: str) -> None: ...
|
||||
|
||||
|
||||
class AutocompleteSource(Protocol):
|
||||
@@ -47,6 +80,7 @@ class AgentReply:
|
||||
drugs: tuple[str, ...] = ()
|
||||
turn_type: str = ""
|
||||
generated: bool = False
|
||||
quick_replies: tuple[str, ...] = ()
|
||||
|
||||
|
||||
class RagAgent:
|
||||
@@ -57,13 +91,23 @@ class RagAgent:
|
||||
answers: GroundedAnswerService,
|
||||
autocomplete: AutocompleteSource | None = None,
|
||||
history_turns: int = HISTORY_TURNS,
|
||||
max_wall_clock_ms: int = MAX_WALL_CLOCK_MS,
|
||||
max_llm_calls_per_turn: int = MAX_LLM_CALLS_PER_TURN,
|
||||
store: ConversationStore | None = None,
|
||||
) -> None:
|
||||
self._understander = understander
|
||||
self._retrieval = retrieval
|
||||
self._answers = answers
|
||||
self._autocomplete = autocomplete
|
||||
self._history_turns = history_turns
|
||||
self._max_wall_clock_ms = max_wall_clock_ms
|
||||
self._max_llm_calls_per_turn = max_llm_calls_per_turn
|
||||
self._store = store
|
||||
self._history: dict[str, list[str]] = {}
|
||||
# In-process only, same durability caveat as `_history` (ADR 0008's
|
||||
# named gap) — lost on restart, not shared across workers.
|
||||
self._last_frame: dict[str, QueryFrame] = {}
|
||||
self._clarify_streak: dict[str, int] = {}
|
||||
|
||||
def complete(self, prefix: str, k: int = 8) -> list[str]:
|
||||
"""Display names matching a typed prefix, for input autocomplete."""
|
||||
@@ -72,20 +116,108 @@ class RagAgent:
|
||||
return [_display_name(drug_id) for drug_id in self._autocomplete.complete(prefix, k)]
|
||||
|
||||
def handle(self, turn: str, conversation_id: str | None = None) -> AgentReply:
|
||||
history = self._history.get(conversation_id, []) if conversation_id else []
|
||||
frame = self._understander.understand(turn, tuple(history))
|
||||
reply = self._route(turn, frame)
|
||||
# F-08: one budget per turn, threaded through every LLM call this
|
||||
# turn makes (understand, then whatever `_route` reaches).
|
||||
t0 = time.monotonic()
|
||||
budget = RequestBudget.start(self._max_wall_clock_ms, self._max_llm_calls_per_turn)
|
||||
history = self._get_history(conversation_id)
|
||||
t1 = time.monotonic()
|
||||
prior_frame = self._last_frame.get(conversation_id) if conversation_id else None
|
||||
frame = self._understander.understand(
|
||||
turn, tuple(history), budget=budget, prior_frame=prior_frame
|
||||
)
|
||||
t2 = time.monotonic()
|
||||
reply = self._route(turn, frame, budget)
|
||||
reply = self._enforce_clarify_circuit_breaker(conversation_id, reply)
|
||||
t3 = time.monotonic()
|
||||
if conversation_id is not None:
|
||||
self._remember(conversation_id, turn, reply)
|
||||
self._last_frame[conversation_id] = frame
|
||||
t4 = time.monotonic()
|
||||
# Temporary instrumentation (2026-08-07): added specifically to
|
||||
# pinpoint a live, reproduced-in-browser case of the FIRST LLM call
|
||||
# already reporting the F-08 budget exhausted — i.e. >20s elapsed
|
||||
# before even one Bedrock call was attempted, with no code between
|
||||
# `RequestBudget.start()` and that first `budget.require()` that
|
||||
# should plausibly take anywhere near that long. Logs unconditionally
|
||||
# (not just on the slow path) so a normal turn's timing is on record
|
||||
# too, for comparison.
|
||||
# .warning, not .info: uvicorn's default logging config only wires
|
||||
# handlers onto its own "uvicorn"/"uvicorn.access" loggers, not the
|
||||
# root logger, so a plain .info() here would silently go nowhere —
|
||||
# confirmed by `understanding.py`'s existing warning-level log
|
||||
# already showing up in the same server output this session.
|
||||
logger.warning(
|
||||
"handle() timing: history=%.2fs understand=%.2fs route=%.2fs "
|
||||
"remember=%.2fs total=%.2fs",
|
||||
t1 - t0, t2 - t1, t3 - t2, t4 - t3, t4 - t0,
|
||||
)
|
||||
return reply
|
||||
|
||||
def _route(self, turn: str, frame: QueryFrame) -> AgentReply:
|
||||
def _enforce_clarify_circuit_breaker(
|
||||
self, conversation_id: str | None, reply: AgentReply
|
||||
) -> AgentReply:
|
||||
"""Hard stop after `MAX_CONSECUTIVE_CLARIFY` clarify turns in a row.
|
||||
|
||||
Every other failure mode in this file degrades to a bounded, honest
|
||||
abstain — this is the one path that previously had no bound at all:
|
||||
a model that keeps deciding needs_clarify=true has no natural exit,
|
||||
and the user has no way out except abandoning the conversation. Any
|
||||
non-clarify decision (answered, or a different abstain reason) resets
|
||||
the streak — this only fires on genuinely consecutive clarifies.
|
||||
"""
|
||||
if conversation_id is None:
|
||||
return reply
|
||||
if reply.decision != "clarify":
|
||||
self._clarify_streak.pop(conversation_id, None)
|
||||
return reply
|
||||
streak = self._clarify_streak.get(conversation_id, 0) + 1
|
||||
if streak >= MAX_CONSECUTIVE_CLARIFY:
|
||||
self._clarify_streak.pop(conversation_id, None)
|
||||
return AgentReply(
|
||||
"abstain", "clarify_loop_exhausted",
|
||||
answer=(
|
||||
"Hệ thống chưa xác định đủ thông tin sau nhiều lần hỏi lại. "
|
||||
"Anh/chị vui lòng gõ lại TOÀN BỘ câu hỏi trong một tin nhắn "
|
||||
"đầy đủ (tên thuốc, đối tượng, cân nặng/tuổi nếu có, đường "
|
||||
"dùng), hoặc bấm \"Tạo phiên tra cứu mới\" để bắt đầu lại."
|
||||
),
|
||||
turn_type=reply.turn_type,
|
||||
)
|
||||
self._clarify_streak[conversation_id] = streak
|
||||
return reply
|
||||
|
||||
def _get_history(self, conversation_id: str | None) -> list[str]:
|
||||
if conversation_id is None:
|
||||
return []
|
||||
if self._store is not None:
|
||||
try:
|
||||
return self._store.recent(conversation_id, self._history_turns * 2)
|
||||
except Exception:
|
||||
# Fail-open (F-09's precedent): a store outage means this
|
||||
# turn is understood fresh, with no memory of earlier ones —
|
||||
# worse UX, not a 500. `routers/rag.py` wraps
|
||||
# `PostgresTraceRepository.save()` the same bare way for the
|
||||
# same reason.
|
||||
return []
|
||||
return self._history.get(conversation_id, [])
|
||||
|
||||
def _route(self, turn: str, frame: QueryFrame, budget: RequestBudget) -> AgentReply:
|
||||
tt = frame.turn_type
|
||||
|
||||
if frame.needs_clarify and frame.clarify_reason:
|
||||
return AgentReply("clarify", "needs_more_info",
|
||||
# `system_error` set means this isn't a real clarify at all — the
|
||||
# understanding call itself failed (provider outage, malformed
|
||||
# output) and failed closed to this same shape. Surface the real
|
||||
# reason instead of the generic "needs_more_info" so a technical
|
||||
# failure is distinguishable from an ordinary question back, both
|
||||
# in the API response and in `/metrics`/traces (found live
|
||||
# 2026-08-07: these were indistinguishable, which is why a real
|
||||
# outage looked identical to normal clarify traffic).
|
||||
return AgentReply("clarify", frame.system_error or "needs_more_info",
|
||||
clarification=frame.clarify_reason,
|
||||
drugs=frame.drugs, turn_type=tt)
|
||||
drugs=frame.drugs, turn_type=tt,
|
||||
quick_replies=frame.quick_replies)
|
||||
|
||||
if tt == "smalltalk":
|
||||
return AgentReply(
|
||||
@@ -111,40 +243,54 @@ class RagAgent:
|
||||
answer=f"Không tìm thấy \"{names}\" trong Dược thư Quốc gia Việt Nam.",
|
||||
turn_type=tt)
|
||||
if tt == "symptom_to_drug":
|
||||
# Reverse lookup (indication/adverse-effect -> drugs) is a distinct
|
||||
# retrieval mode, not yet wired. Be honest rather than abstain blank.
|
||||
if frame.indication:
|
||||
return self._symptom_to_drug(turn, frame, budget)
|
||||
return AgentReply(
|
||||
"clarify", "reverse_lookup_not_ready",
|
||||
clarification="Tra ngược theo triệu chứng/chỉ định đang được bổ "
|
||||
"sung. Anh/chị cho biết tên thuốc cụ thể để tôi tra giúp?",
|
||||
"clarify", "no_indication",
|
||||
clarification="Anh/chị mô tả triệu chứng hoặc chỉ định cần tra giúp "
|
||||
"em với?",
|
||||
turn_type=tt)
|
||||
return AgentReply(
|
||||
"clarify", "no_drug",
|
||||
clarification="Anh/chị muốn tra thuốc nào?", turn_type=tt)
|
||||
|
||||
if tt == "interaction" and len(frame.drugs) >= 2:
|
||||
return self._interaction(turn, frame)
|
||||
return self._interaction(turn, frame, budget)
|
||||
|
||||
# drug_attribute / drug_overview / dosing_calc / fallback: one drug + section
|
||||
return self._single_drug(turn, frame)
|
||||
return self._single_drug(turn, frame, budget)
|
||||
|
||||
def _single_drug(self, turn: str, frame: QueryFrame) -> AgentReply:
|
||||
def _single_drug(self, turn: str, frame: QueryFrame, budget: RequestBudget) -> AgentReply:
|
||||
query = _synthesize_query(turn, frame)
|
||||
result = self._retrieval.retrieve_framed(
|
||||
frame.drugs[0], frame.attribute, turn,
|
||||
frame.drugs[0], frame.attribute, query,
|
||||
is_overview=frame.turn_type == "drug_overview",
|
||||
)
|
||||
return self._grounded(turn, result, frame)
|
||||
return self._grounded(query, result, frame, budget=budget)
|
||||
|
||||
def _interaction(self, turn: str, frame: QueryFrame) -> AgentReply:
|
||||
def _interaction(self, turn: str, frame: QueryFrame, budget: RequestBudget) -> AgentReply:
|
||||
"""Gather the interaction section of each named drug and synthesise.
|
||||
|
||||
Absence of a match is stated as "not found in each drug's interaction
|
||||
section", never as "safe" — the answer layer's grounding still applies.
|
||||
|
||||
A quarantined table/formula in EITHER drug's interaction section
|
||||
forces the whole combined answer to VERIFY_PDF via
|
||||
`RetrievalService.decide` — the same policy the single-drug path
|
||||
already applies to a section with quarantined content. Previously
|
||||
this only kept `part.decision == ANSWERABLE` parts, so a quarantined
|
||||
drug's evidence (and the "table exists, verify PDF" notice it must
|
||||
produce per the quarantine contract) was silently dropped instead of
|
||||
surfaced; a confident interaction answer could omit exactly the
|
||||
unverified contraindication table it should have flagged. Generating
|
||||
a synthesis claim from one verified and one unverified source is not
|
||||
safer than generating from either alone, so both now block
|
||||
generation the same way.
|
||||
"""
|
||||
evidences = []
|
||||
for drug_id in frame.drugs:
|
||||
part = self._retrieval.retrieve_framed(drug_id, TUONG_TAC, turn)
|
||||
if part.decision == EvidenceDecision.ANSWERABLE:
|
||||
if part.decision in (EvidenceDecision.ANSWERABLE, EvidenceDecision.VERIFY_PDF):
|
||||
evidences.extend(part.evidence)
|
||||
if not evidences:
|
||||
listed = " và ".join(frame.drugs)
|
||||
@@ -153,16 +299,47 @@ class RagAgent:
|
||||
answer=f"Không tìm thấy mục tương tác thuốc cho {listed} trong Dược "
|
||||
"thư. Điều này KHÔNG có nghĩa là an toàn khi phối hợp.",
|
||||
drugs=frame.drugs, turn_type=frame.turn_type)
|
||||
combined = RetrievalResult(
|
||||
EvidenceDecision.ANSWERABLE, "interaction_evidence",
|
||||
tuple(evidences),
|
||||
combined = self._retrieval.decide(tuple(evidences))
|
||||
return self._grounded(turn, combined, frame, budget=budget)
|
||||
|
||||
def _symptom_to_drug(
|
||||
self, turn: str, frame: QueryFrame, budget: RequestBudget
|
||||
) -> AgentReply:
|
||||
"""Reverse lookup: a symptom/indication -> which drugs' `chi_dinh`
|
||||
actually names it. A factual list from the formulary, not a
|
||||
treatment ranking or recommendation — no drug is preferred over
|
||||
another here, only cited as indicated ([[feedback_no_recommendation_gate]]:
|
||||
this audience is doctors/pharmacists, a lookup like this is normal use).
|
||||
Absence is stated plainly, never as "no such drug exists" — the
|
||||
formulary may simply not name this indication under any monograph.
|
||||
"""
|
||||
result = self._retrieval.retrieve_by_indication(frame.indication)
|
||||
if result.decision == EvidenceDecision.ABSTAIN:
|
||||
return AgentReply(
|
||||
"abstain", result.reason,
|
||||
answer=f"Không tìm thấy thuốc nào trong Dược thư Quốc gia Việt Nam ghi "
|
||||
f"nhận chỉ định cho \"{frame.indication}\". Điều này KHÔNG có "
|
||||
"nghĩa là không có thuốc điều trị — vui lòng tra theo tên thuốc "
|
||||
"cụ thể nếu đã biết.",
|
||||
turn_type=frame.turn_type)
|
||||
# `matched_doc_id` is always `{drug_id}__chi_dinh__{part_index}` — the
|
||||
# drugs actually found, not `frame.drugs` (empty by construction for
|
||||
# this turn_type; the router only reaches here with no named drug).
|
||||
matched_drugs = tuple(dict.fromkeys(
|
||||
evidence.matched_doc_id.split("__")[0] for evidence in result.evidence
|
||||
))
|
||||
return self._grounded(
|
||||
turn, result, frame, drugs=matched_drugs, list_mode=True, budget=budget
|
||||
)
|
||||
return self._grounded(turn, combined, frame)
|
||||
|
||||
def _grounded(
|
||||
self, turn: str, result: RetrievalResult, frame: QueryFrame
|
||||
self, turn: str, result: RetrievalResult, frame: QueryFrame,
|
||||
drugs: tuple[str, ...] | None = None, list_mode: bool = False,
|
||||
budget: RequestBudget | None = None,
|
||||
) -> AgentReply:
|
||||
ga = self._answers.answer_from_result(turn, result)
|
||||
ga = self._answers.answer_from_result(
|
||||
turn, result, list_mode=list_mode, budget=budget
|
||||
)
|
||||
decision = ga.result.decision.value
|
||||
if ga.clarification is not None:
|
||||
decision = "clarify"
|
||||
@@ -172,18 +349,34 @@ class RagAgent:
|
||||
answer=ga.answer,
|
||||
clarification=ga.clarification,
|
||||
citations=ga.citations,
|
||||
drugs=frame.drugs,
|
||||
drugs=drugs if drugs is not None else frame.drugs,
|
||||
turn_type=frame.turn_type,
|
||||
generated=ga.generated,
|
||||
quick_replies=ga.quick_replies,
|
||||
)
|
||||
|
||||
def _remember(self, conversation_id: str, turn: str, reply: AgentReply) -> None:
|
||||
history = self._history.setdefault(conversation_id, [])
|
||||
history.append(f"Người dùng: {turn}")
|
||||
lines = [f"Người dùng: {turn}"]
|
||||
spoken = reply.answer or reply.clarification
|
||||
if spoken:
|
||||
history.append(f"Trợ lý: {spoken[:300]}")
|
||||
# Keep only the recent window; the LLM re-reads it every turn.
|
||||
lines.append(f"Trợ lý: {spoken[:300]}")
|
||||
|
||||
if self._store is not None:
|
||||
try:
|
||||
for line in lines:
|
||||
self._store.append(conversation_id, line)
|
||||
except Exception:
|
||||
# Fail-open: this turn's memory is lost, not the response
|
||||
# already computed and about to be returned to the caller.
|
||||
pass
|
||||
return
|
||||
|
||||
history = self._history.setdefault(conversation_id, [])
|
||||
history.extend(lines)
|
||||
# Keep only the recent window; the LLM re-reads it every turn. Only
|
||||
# needed for the in-process dict — the store path windows at READ
|
||||
# time instead (`recent(..., limit)`), so old rows just sit unused
|
||||
# rather than needing a delete on every turn.
|
||||
excess = len(history) - self._history_turns * 2
|
||||
if excess > 0:
|
||||
del history[:excess]
|
||||
@@ -192,3 +385,60 @@ class RagAgent:
|
||||
def _display_name(drug_id: str) -> str:
|
||||
"""A readable display name from a drug id ('paracetamol_acetaminophen')."""
|
||||
return drug_id.replace("_", " ").title()
|
||||
|
||||
|
||||
_POPULATION_LABELS = {
|
||||
"tre_em": "trẻ em",
|
||||
"tre_so_sinh": "trẻ sơ sinh",
|
||||
"nguoi_lon": "người lớn",
|
||||
"nguoi_cao_tuoi": "người cao tuổi",
|
||||
"phu_nu_co_thai": "phụ nữ có thai",
|
||||
"phu_nu_cho_con_bu": "phụ nữ cho con bú",
|
||||
"suy_than": "suy thận",
|
||||
"suy_gan": "suy gan",
|
||||
}
|
||||
|
||||
_ROUTE_LABELS = {
|
||||
"uong": "uống",
|
||||
"tiem_tinh_mach": "tiêm tĩnh mạch",
|
||||
"tiem_bap": "tiêm bắp",
|
||||
"tiem_duoi_da": "tiêm dưới da",
|
||||
"dat_truc_trang": "đặt trực tràng",
|
||||
"boi_ngoai_da": "bôi ngoài da",
|
||||
"nho_mat": "nhỏ mắt",
|
||||
"nho_mui": "nhỏ mũi",
|
||||
"khac": "khác",
|
||||
}
|
||||
|
||||
|
||||
def _synthesize_query(turn: str, frame: QueryFrame) -> str:
|
||||
"""Fold the structured context `understanding.py` resolved — possibly
|
||||
across several turns — into one self-contained question.
|
||||
|
||||
`GroundedAnswerService.answer_from_result` has no conversation history of
|
||||
its own; the `query` string it receives IS the entire context its
|
||||
sufficiency-check and generation LLM calls see. Passing the bare current
|
||||
turn loses everything resolved earlier: a reply like "Uống" answering a
|
||||
route question three turns into a dose conversation would reach
|
||||
generation as just "Uống", indistinguishable from a user who typed
|
||||
nothing else — the two P0s the 2026-08-06 audit named (population/
|
||||
weight/age/route extracted but discarded downstream) are exactly this
|
||||
gap. Redundant when the turn is already self-contained (a fresh
|
||||
single-shot question re-states its own population/route, so this just
|
||||
repeats it) — harmless, since omission is the failure mode, not
|
||||
repetition.
|
||||
"""
|
||||
parts = [turn]
|
||||
if frame.population:
|
||||
parts.append(f"Đối tượng: {_POPULATION_LABELS.get(frame.population, frame.population)}")
|
||||
if frame.age_text:
|
||||
parts.append(f"Tuổi: {frame.age_text}")
|
||||
if frame.weight_kg is not None:
|
||||
parts.append(f"Cân nặng: {frame.weight_kg:g} kg")
|
||||
if frame.route:
|
||||
parts.append(f"Đường dùng: {_ROUTE_LABELS.get(frame.route, frame.route)}")
|
||||
if frame.indication:
|
||||
parts.append(f"Chỉ định/triệu chứng: {frame.indication}")
|
||||
if len(parts) == 1:
|
||||
return turn
|
||||
return ". ".join(parts) + "."
|
||||
|
||||
+207
-63
@@ -5,12 +5,17 @@ import re
|
||||
from dataclasses import dataclass, replace
|
||||
|
||||
from . import grounding, metrics as metric_names
|
||||
from .budget import RequestBudget, RequestBudgetExhausted
|
||||
from .metrics import Metrics, NullMetrics
|
||||
from .models import EvidenceDecision, QueryIntent, RetrievalResult, SubjectScope
|
||||
from .ports import AnswerGenerationUnavailable, AnswerGenerator
|
||||
from .prompt import build_entailment_request, build_request, build_sufficiency_request
|
||||
from .routing import QueryRoutingService
|
||||
|
||||
# See `_verify_entailment`'s docstring for the measured trade-off behind
|
||||
# widening this from 2 to 3.
|
||||
_ENTAILMENT_MAX_ATTEMPTS = 3
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Citation:
|
||||
@@ -22,6 +27,10 @@ class Citation:
|
||||
bbox: tuple[float, float, float, float] | None = None
|
||||
source_crop: str | None = None
|
||||
attachment: str | None = None
|
||||
# The exact retrieved text this citation stands for — the same string
|
||||
# handed to the generator/entailment checks, so the UI can show precisely
|
||||
# what was retrieved rather than a fabricated summary of it.
|
||||
evidence_text: str = ""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -34,12 +43,35 @@ class GroundedAnswer:
|
||||
# (e.g. a dose question with no age/weight). The answer field carries the
|
||||
# question; the caller renders it as a clarification, not a final answer.
|
||||
clarification: str | None = None
|
||||
# Short suggested replies for `clarification`, e.g. ("Người lớn", "Trẻ
|
||||
# em") — only populated when the sufficiency check judged the question
|
||||
# to have a few natural discrete answers, never invented client-side.
|
||||
quick_replies: tuple[str, ...] = ()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _GenOutcome:
|
||||
answer: str | None = None
|
||||
clarification: str | None = None
|
||||
# The specific reason a rejection happened — the exact string already
|
||||
# used for the GENERATION_REJECTED metric, propagated here so
|
||||
# `answer_from_result` can put it in the API response's `reason` field
|
||||
# instead of a generic catch-all. `None` when `answer`/`clarification`
|
||||
# is set (nothing was rejected).
|
||||
reject_reason: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _RawAttempt:
|
||||
"""One raw `_attempt_generation` call, before any metric is charged —
|
||||
lets `_generate` retry the noisy `insufficient` case without
|
||||
double-counting a rejection metric across both attempts."""
|
||||
answer: str | None = None
|
||||
clarification: str | None = None
|
||||
insufficient: bool = False
|
||||
outage: bool = False
|
||||
budget_exhausted: bool = False
|
||||
malformed: bool = False
|
||||
|
||||
|
||||
class GroundedAnswerService:
|
||||
@@ -57,10 +89,13 @@ class GroundedAnswerService:
|
||||
pass confirming each cited claim's *content* — not just its numbers —
|
||||
is actually stated by that block). If a configured generation fails
|
||||
any check, or the provider itself is unreachable, or its output is
|
||||
malformed, the turn **abstains** (`reason="generation_unavailable"`
|
||||
or the specific `grounding.verify` reason) rather than silently
|
||||
degrading to a raw source dump — this product is a real LLM chatbot,
|
||||
and a citation-stapled paragraph of book text is not an acceptable
|
||||
malformed, the turn **abstains** with the specific reason that failed
|
||||
it (`provider_unavailable`, `malformed_output`, `evidence_insufficient`,
|
||||
a `grounding.verify` reason, or `unsupported_claim`; falls back to
|
||||
the generic `generation_unavailable` only if none of those was set)
|
||||
rather than silently degrading to a raw source dump — this product is
|
||||
a real LLM chatbot, and a citation-stapled paragraph of book text is
|
||||
not an acceptable
|
||||
stand-in for an answer the model was supposed to produce.
|
||||
"""
|
||||
|
||||
@@ -94,12 +129,27 @@ class GroundedAnswerService:
|
||||
return self.answer_from_result(query, result)
|
||||
|
||||
def answer_from_result(
|
||||
self, query: str, result: RetrievalResult
|
||||
self, query: str, result: RetrievalResult, list_mode: bool = False,
|
||||
budget: RequestBudget | None = None,
|
||||
) -> GroundedAnswer:
|
||||
"""Everything after retrieval — grounding, sufficiency, generation,
|
||||
citations. Split out so the new understanding-driven orchestrator
|
||||
(`rag/agent.py`) reuses the safe answer path without going through the
|
||||
old `QueryRoutingService` text resolution."""
|
||||
old `QueryRoutingService` text resolution.
|
||||
|
||||
`list_mode=True`: the evidence is several DIFFERENT drugs' own
|
||||
sections (symptom_to_drug), not alternative phrasings of one drug's
|
||||
answer — the sufficiency clarify ("which kind of headache?") that's
|
||||
right for a single dose question doesn't fit a reverse lookup, whose
|
||||
whole point is to show what the formulary has and let the clinician
|
||||
narrow it themselves; skipped here the same way a bare-name intro
|
||||
already skips it.
|
||||
|
||||
`budget` (F-08): threaded through to every LLM call this method
|
||||
makes (sufficiency, generate, up to 2 entailment). `None` (the
|
||||
default) means unbounded, unchanged from before F-08 — only
|
||||
`RagAgent` constructs a real budget today.
|
||||
"""
|
||||
if result.decision == EvidenceDecision.ABSTAIN:
|
||||
self._metrics.increment(metric_names.ABSTENTION, reason=result.reason)
|
||||
return GroundedAnswer(result, None)
|
||||
@@ -135,11 +185,22 @@ class GroundedAnswerService:
|
||||
# Reasoning step BEFORE answering: if the turn is under-specified (a dose
|
||||
# with several bands and no age/weight/condition), ask instead of dumping.
|
||||
# A separate focused call is more reliable than folding it into generation.
|
||||
clarify_q = self._check_sufficiency(query, evidence_texts, result.is_drug_overview)
|
||||
if clarify_q is not None:
|
||||
return GroundedAnswer(result, clarify_q, (), clarification=clarify_q)
|
||||
sufficiency = (
|
||||
None if list_mode
|
||||
else self._check_sufficiency(
|
||||
query, evidence_texts, result.is_drug_overview, budget=budget
|
||||
)
|
||||
)
|
||||
if sufficiency is not None:
|
||||
clarify_q, quick_replies = sufficiency
|
||||
return GroundedAnswer(
|
||||
result, clarify_q, (), clarification=clarify_q, quick_replies=quick_replies
|
||||
)
|
||||
|
||||
outcome = self._generate(query, evidence_texts, intro=result.is_drug_overview)
|
||||
outcome = self._generate(
|
||||
query, evidence_texts, intro=result.is_drug_overview, list_mode=list_mode,
|
||||
budget=budget,
|
||||
)
|
||||
if outcome.clarification is not None:
|
||||
# The model judged the turn under-specified (a dose with no
|
||||
# age/weight/renal-function/indication…) and asked back instead of
|
||||
@@ -163,14 +224,23 @@ class GroundedAnswerService:
|
||||
# source dump is not an acceptable stand-in for a failed
|
||||
# generation, so this abstains instead of silently degrading to
|
||||
# one.
|
||||
self._metrics.increment(
|
||||
metric_names.ABSTENTION, reason="generation_unavailable"
|
||||
)
|
||||
# The specific check that failed (provider_unavailable,
|
||||
# malformed_output, evidence_insufficient, ungrounded_number,
|
||||
# uncited_claim, unsupported_claim, request_budget_exhausted) —
|
||||
# found live 2026-08-07: every one of these used to collapse into
|
||||
# the same generic "generation_unavailable" by the time it
|
||||
# reached the API response/trace, so a real, diagnosable cause
|
||||
# (e.g. a genuine provider outage) was indistinguishable from
|
||||
# ordinary entailment noise without reading server-side metrics
|
||||
# by hand. `outcome.reject_reason` already carries the granular
|
||||
# value the metric above uses — just propagate it.
|
||||
reason = outcome.reject_reason or "generation_unavailable"
|
||||
self._metrics.increment(metric_names.ABSTENTION, reason=reason)
|
||||
return GroundedAnswer(
|
||||
replace(
|
||||
result,
|
||||
decision=EvidenceDecision.ABSTAIN,
|
||||
reason="generation_unavailable",
|
||||
reason=reason,
|
||||
),
|
||||
None,
|
||||
)
|
||||
@@ -182,67 +252,107 @@ class GroundedAnswerService:
|
||||
self._metrics.increment(metric_names.GENERATION_SERVED)
|
||||
return GroundedAnswer(result, outcome.answer, citations, generated=True)
|
||||
|
||||
def _generate(
|
||||
self, query: str, evidence_texts: tuple[str, ...], intro: bool = False
|
||||
) -> "_GenOutcome":
|
||||
"""A verified generation, a clarifying question, or empty to fall back."""
|
||||
if self._generator is None or not evidence_texts:
|
||||
return _GenOutcome()
|
||||
|
||||
request = build_request(query, evidence_texts, intro=intro)
|
||||
def _attempt_generation(
|
||||
self, request: "GenerationRequest", budget: RequestBudget | None
|
||||
) -> "_RawAttempt":
|
||||
"""One raw generation call, parsed but not yet metric-counted or
|
||||
verified — the caller decides whether to retry before charging a
|
||||
metric to any particular reason."""
|
||||
try:
|
||||
if budget is not None:
|
||||
budget.require()
|
||||
raw = self._generator.generate(request.system, request.user, request.schema)
|
||||
except RequestBudgetExhausted:
|
||||
return _RawAttempt(budget_exhausted=True)
|
||||
except AnswerGenerationUnavailable:
|
||||
self._metrics.increment(
|
||||
metric_names.GENERATION_REJECTED, reason="provider_unavailable"
|
||||
)
|
||||
return _GenOutcome()
|
||||
return _RawAttempt(outage=True)
|
||||
|
||||
try:
|
||||
payload = json.loads(raw)
|
||||
answer = payload["answer"]
|
||||
sufficient = payload["evidence_sufficient"]
|
||||
except (ValueError, TypeError, KeyError):
|
||||
self._metrics.increment(
|
||||
metric_names.GENERATION_REJECTED, reason="malformed_output"
|
||||
)
|
||||
return _GenOutcome()
|
||||
return _RawAttempt(malformed=True)
|
||||
|
||||
# The model asked for a missing detail (age/weight/renal function/
|
||||
# indication…) instead of listing every band. A clarify is not a grounded
|
||||
# claim, so it skips the number check — it states no dose.
|
||||
clarify = payload.get("clarifying_question") if isinstance(payload, dict) else None
|
||||
if isinstance(clarify, str) and clarify.strip():
|
||||
return _GenOutcome(clarification=clarify.strip())
|
||||
return _RawAttempt(clarification=clarify.strip())
|
||||
|
||||
if not isinstance(answer, str) or not isinstance(sufficient, bool):
|
||||
return _RawAttempt(malformed=True)
|
||||
if not sufficient:
|
||||
return _RawAttempt(insufficient=True)
|
||||
return _RawAttempt(answer=answer)
|
||||
|
||||
def _generate(
|
||||
self, query: str, evidence_texts: tuple[str, ...], intro: bool = False,
|
||||
list_mode: bool = False, budget: RequestBudget | None = None,
|
||||
) -> "_GenOutcome":
|
||||
"""A verified generation, a clarifying question, or empty to fall back."""
|
||||
if self._generator is None or not evidence_texts:
|
||||
return _GenOutcome()
|
||||
|
||||
request = build_request(query, evidence_texts, intro=intro, list_mode=list_mode)
|
||||
attempt = self._attempt_generation(request, budget)
|
||||
if attempt.insufficient:
|
||||
# Empirically noisy (found live 2026-08-07, reproduced 3/3 on a
|
||||
# fresh retry): the model's own evidence_sufficient=false
|
||||
# self-assessment sometimes flips to a correct, fully grounded
|
||||
# answer when asked again with the IDENTICAL evidence — the same
|
||||
# one-retry pattern `_verify_entailment` already uses below for
|
||||
# its own noisy judge call. Only the terminal "insufficient AND
|
||||
# no clarifying question" case retries; a legitimate ask-for-
|
||||
# more-detail clarify is untouched.
|
||||
attempt = self._attempt_generation(request, budget)
|
||||
|
||||
if attempt.budget_exhausted:
|
||||
self._metrics.increment(
|
||||
metric_names.GENERATION_REJECTED, reason="request_budget_exhausted"
|
||||
)
|
||||
return _GenOutcome(reject_reason="request_budget_exhausted")
|
||||
if attempt.outage:
|
||||
self._metrics.increment(
|
||||
metric_names.GENERATION_REJECTED, reason="provider_unavailable"
|
||||
)
|
||||
return _GenOutcome(reject_reason="provider_unavailable")
|
||||
if attempt.malformed:
|
||||
self._metrics.increment(
|
||||
metric_names.GENERATION_REJECTED, reason="malformed_output"
|
||||
)
|
||||
return _GenOutcome()
|
||||
if not sufficient:
|
||||
# The model says the evidence does not answer the question. Showing
|
||||
# the retrieved section verbatim lets the clinician judge that.
|
||||
return _GenOutcome(reject_reason="malformed_output")
|
||||
if attempt.clarification is not None:
|
||||
return _GenOutcome(clarification=attempt.clarification)
|
||||
if attempt.insufficient:
|
||||
# The model says the evidence does not answer the question, on
|
||||
# both attempts. Showing the retrieved section verbatim lets the
|
||||
# clinician judge that.
|
||||
self._metrics.increment(
|
||||
metric_names.GENERATION_REJECTED, reason="evidence_insufficient"
|
||||
)
|
||||
return _GenOutcome()
|
||||
return _GenOutcome(reject_reason="evidence_insufficient")
|
||||
|
||||
answer = attempt.answer
|
||||
report = grounding.verify(answer, evidence_texts)
|
||||
if not report.grounded:
|
||||
self._metrics.increment(
|
||||
metric_names.GENERATION_REJECTED, reason=report.reason
|
||||
)
|
||||
return _GenOutcome()
|
||||
return _GenOutcome(reject_reason=report.reason)
|
||||
|
||||
if not self._verify_entailment(answer, evidence_texts):
|
||||
if not self._verify_entailment(answer, evidence_texts, budget=budget):
|
||||
self._metrics.increment(
|
||||
metric_names.GENERATION_REJECTED, reason="unsupported_claim"
|
||||
)
|
||||
return _GenOutcome()
|
||||
return _GenOutcome(reject_reason="unsupported_claim")
|
||||
return _GenOutcome(answer=answer)
|
||||
|
||||
def _verify_entailment(self, answer: str, evidence_texts: tuple[str, ...]) -> bool:
|
||||
def _verify_entailment(
|
||||
self, answer: str, evidence_texts: tuple[str, ...],
|
||||
budget: RequestBudget | None = None,
|
||||
) -> bool:
|
||||
"""A second, adversarial LLM pass over an answer that already passed
|
||||
`grounding.verify`.
|
||||
|
||||
@@ -254,14 +364,22 @@ class GroundedAnswerService:
|
||||
is checked against only the evidence block(s) it names, by a model
|
||||
told to compare wording, not to reason about medicine.
|
||||
|
||||
Fails closed on an outage or malformed output. A single rejection is
|
||||
NOT: live probing (2026-08-06) found the judge call itself is noisy
|
||||
— the identical claim/evidence pair, called three times, came back
|
||||
entailed twice and rejected once, discarding a correct, well-cited
|
||||
interaction answer. So a reject triggers one same-claim retry, and
|
||||
only a second, agreeing reject discards the generation; a single
|
||||
provider outage/malformed response still fails closed immediately
|
||||
(that failure mode is reliable, not noisy — no retry needed there).
|
||||
Fails closed on an outage or malformed output — that failure mode is
|
||||
reliable, not noisy, so it stops immediately rather than spending
|
||||
retries on it. A single rejection is NOT reliable: live probing
|
||||
(2026-08-06) found the judge call itself is noisy — the identical
|
||||
claim/evidence pair, called three times, came back entailed twice
|
||||
and rejected once, discarding a correct, well-cited interaction
|
||||
answer. Up to `_ENTAILMENT_MAX_ATTEMPTS` same-claim calls run;
|
||||
accept on the first `True`, discard only if every attempt agrees
|
||||
reject. Widened from 2 to 3 attempts 2026-08-07 after a live
|
||||
adversarial sample (50 real questions) measured this specific check
|
||||
as roughly half of all false abstentions on genuinely answerable
|
||||
questions. Trade-off, stated plainly: this raises the bar a
|
||||
genuinely fabricated claim must now clear too (it survives if ANY
|
||||
one of 3 noisy calls wrongly accepts it, not just 1 of 2) — accepted
|
||||
because the probed noise is symmetric and the entailment prompt
|
||||
itself is unchanged, not because the risk is zero.
|
||||
An answer with no claim text at all (nothing between or after its
|
||||
citation markers) is vacuously fine — nothing to verify, no call.
|
||||
"""
|
||||
@@ -274,18 +392,23 @@ class GroundedAnswerService:
|
||||
return True
|
||||
|
||||
request = build_entailment_request(claims)
|
||||
first = self._run_entailment_check(request)
|
||||
if first is None:
|
||||
return False
|
||||
if first:
|
||||
return True
|
||||
second = self._run_entailment_check(request)
|
||||
return bool(second)
|
||||
for _ in range(_ENTAILMENT_MAX_ATTEMPTS):
|
||||
verdict = self._run_entailment_check(request, budget=budget)
|
||||
if verdict is None:
|
||||
return False
|
||||
if verdict:
|
||||
return True
|
||||
return False
|
||||
|
||||
def _run_entailment_check(self, request) -> bool | None:
|
||||
"""One entailment call. `None` = outage/malformed (fails closed by the
|
||||
caller without a retry); `True`/`False` = the judge's verdict."""
|
||||
def _run_entailment_check(
|
||||
self, request, budget: RequestBudget | None = None
|
||||
) -> bool | None:
|
||||
"""One entailment call. `None` = outage/malformed/budget-exhausted
|
||||
(fails closed by the caller without a retry); `True`/`False` = the
|
||||
judge's verdict."""
|
||||
try:
|
||||
if budget is not None:
|
||||
budget.require()
|
||||
raw = self._generator.generate(request.system, request.user, request.schema)
|
||||
except AnswerGenerationUnavailable:
|
||||
return None
|
||||
@@ -300,17 +423,31 @@ class GroundedAnswerService:
|
||||
return entailed and not unsupported
|
||||
|
||||
def _check_sufficiency(
|
||||
self, query: str, evidence_texts: tuple[str, ...], intro: bool = False
|
||||
) -> str | None:
|
||||
self, query: str, evidence_texts: tuple[str, ...], intro: bool = False,
|
||||
budget: RequestBudget | None = None,
|
||||
) -> tuple[str, tuple[str, ...]] | None:
|
||||
"""A focused reasoning call: is the turn specific enough to answer, or
|
||||
must we ask? Returns a clarifying question, or None to proceed.
|
||||
must we ask? Returns (clarifying_question, quick_replies), or None to
|
||||
proceed. `quick_replies` is often empty — only populated when the
|
||||
model judged the missing detail has a few natural discrete answers
|
||||
(e.g. "Người lớn"/"Trẻ em"), never invented here.
|
||||
|
||||
Skipped without a model, for a bare-name intro (not a dose), or for a
|
||||
single evidence block (nothing to disambiguate)."""
|
||||
single evidence block (nothing to disambiguate).
|
||||
|
||||
Fails OPEN on outage/budget-exhaustion (returns None, proceeds to
|
||||
generate) — deliberately different from every other call in this
|
||||
file, which fail closed. This is a reasoning heuristic, not a safety
|
||||
check; grounding + entailment remain the real gate on whatever gets
|
||||
generated next, so skipping this one costs UX quality (a dose
|
||||
question that should have asked for age/weight might not), not
|
||||
safety."""
|
||||
if self._generator is None or intro or len(evidence_texts) < 2:
|
||||
return None
|
||||
request = build_sufficiency_request(query, evidence_texts)
|
||||
try:
|
||||
if budget is not None:
|
||||
budget.require()
|
||||
raw = self._generator.generate(request.system, request.user, request.schema)
|
||||
except AnswerGenerationUnavailable:
|
||||
return None
|
||||
@@ -321,7 +458,13 @@ class GroundedAnswerService:
|
||||
if isinstance(payload, dict) and payload.get("sufficient") is False:
|
||||
question = payload.get("clarifying_question")
|
||||
if isinstance(question, str) and question.strip():
|
||||
return question.strip()
|
||||
raw_replies = payload.get("quick_replies")
|
||||
replies = tuple(
|
||||
reply.strip()
|
||||
for reply in raw_replies
|
||||
if isinstance(reply, str) and reply.strip()
|
||||
) if isinstance(raw_replies, list) else ()
|
||||
return question.strip(), replies
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
@@ -362,5 +505,6 @@ class GroundedAnswerService:
|
||||
# real crop path wins; otherwise the block id plus the
|
||||
# structured page/bbox fields is enough to render later.
|
||||
attachment=source.source_crop or source.block_id,
|
||||
evidence_text=evidence.text,
|
||||
)))
|
||||
return citations
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
"""A per-request LLM-call budget — F-08.
|
||||
|
||||
`RagAgent.handle()` makes up to ~5 sequential Bedrock calls per turn
|
||||
(understand, sufficiency, generate, up to 2 entailment retries) with no
|
||||
aggregate deadline before this: each call is bounded only by its own fixed
|
||||
provider timeout (`read_timeout=60` in `adapters/bedrock_converse.py`, times
|
||||
up to 3 retries at "standard" backoff — worst case several minutes for one
|
||||
stuck call, let alone five). Measured live 2026-08-07: a normal answerable
|
||||
turn costs ~8-9s total; nothing bounds the pathological case.
|
||||
|
||||
Checked before each call, not wrapped around an already-running one — this
|
||||
bounds how many MORE calls get a chance to start once time/calls run out. It
|
||||
does not cancel a call already in flight past its own provider timeout; a
|
||||
hard per-call cancellation would need cooperative cancellation support from
|
||||
`adapters/bedrock_converse.py`'s boto3 client, a larger change than this
|
||||
budget object alone. Still a real improvement: five calls each capable of
|
||||
running to their own 60s+ limit, one after another, is the actual gap this
|
||||
closes.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
|
||||
from .ports import AnswerGenerationUnavailable
|
||||
|
||||
|
||||
class RequestBudgetExhausted(AnswerGenerationUnavailable):
|
||||
"""The per-request budget ran out before a call could be attempted.
|
||||
|
||||
Subclasses `AnswerGenerationUnavailable` deliberately: every existing
|
||||
`except AnswerGenerationUnavailable:` fail-closed/fail-open handler
|
||||
already does the right thing for this with no changes — a caller that
|
||||
wants to log a distinct reason (budget vs. genuine outage) catches this
|
||||
subclass specifically before the general one.
|
||||
"""
|
||||
|
||||
|
||||
@dataclass
|
||||
class RequestBudget:
|
||||
deadline: float
|
||||
calls_remaining: int
|
||||
|
||||
@classmethod
|
||||
def start(cls, max_wall_clock_ms: int, max_calls: int) -> "RequestBudget":
|
||||
return cls(
|
||||
deadline=time.monotonic() + max_wall_clock_ms / 1000,
|
||||
calls_remaining=max_calls,
|
||||
)
|
||||
|
||||
def has_budget(self) -> bool:
|
||||
return self.calls_remaining > 0 and time.monotonic() < self.deadline
|
||||
|
||||
def spend(self) -> None:
|
||||
self.calls_remaining -= 1
|
||||
|
||||
def require(self) -> None:
|
||||
"""Raise if there's no budget for one more call, else spend it.
|
||||
The single call site every LLM-call wrapper below should make
|
||||
immediately before its actual provider call."""
|
||||
if not self.has_budget():
|
||||
raise RequestBudgetExhausted(
|
||||
"request budget exhausted (calls or wall-clock deadline)"
|
||||
)
|
||||
self.spend()
|
||||
@@ -1,371 +0,0 @@
|
||||
"""Conversation state, and the rules for carrying context across turns.
|
||||
|
||||
Pure domain. Everything here works without an LLM, which is deliberate: the
|
||||
part of "understanding a follow-up" that matters clinically — *which drug is
|
||||
this still about* — must be deterministic and testable, not inferred.
|
||||
|
||||
Two structures with different jobs:
|
||||
|
||||
`Focus` is structured and drives routing. It is what makes "còn trẻ em thì
|
||||
sao?" resolvable at all.
|
||||
|
||||
`summary` is prose for the generator. It records **what was discussed**, never
|
||||
clinical content: a dose restated from a summary carries no citation and could
|
||||
not be grounding-verified, because that check compares against retrieved
|
||||
evidence and a summary is not evidence.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field, replace
|
||||
from typing import Literal, Protocol
|
||||
|
||||
# A drug named six turns ago is not context, it is a hazard: conversations
|
||||
# drift, and inheriting a stale drug produces a confident answer about the
|
||||
# wrong medicine.
|
||||
FOCUS_TTL_TURNS = 6
|
||||
|
||||
# Three exchanges kept verbatim; older turns are folded into the summary.
|
||||
RECENT_TURNS = 6
|
||||
|
||||
Role = Literal["user", "assistant"]
|
||||
Verbosity = Literal["concise", "detailed"]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Turn:
|
||||
role: Role
|
||||
text: str
|
||||
at: str
|
||||
drug_id: str | None = None
|
||||
section_key: str | None = None
|
||||
# Storing what answered a turn is what lets the planner reuse evidence
|
||||
# instead of retrieving the same section again.
|
||||
evidence_ids: tuple[str, ...] = ()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Focus:
|
||||
"""The entities a follow-up may inherit, each with the turn that set it."""
|
||||
|
||||
drug_id: str | None = None
|
||||
drug_name: str | None = None
|
||||
section_key: str | None = None
|
||||
population: str | None = None
|
||||
verbosity: Verbosity | None = None
|
||||
set_at_turn: dict[str, int] = field(default_factory=dict)
|
||||
|
||||
def age_of(self, name: str, turn_count: int) -> int | None:
|
||||
set_at = self.set_at_turn.get(name)
|
||||
return None if set_at is None else turn_count - set_at
|
||||
|
||||
def is_fresh(self, name: str, turn_count: int, ttl: int = FOCUS_TTL_TURNS) -> bool:
|
||||
age = self.age_of(name, turn_count)
|
||||
return age is not None and age <= ttl
|
||||
|
||||
def with_field(self, name: str, value, turn: int) -> "Focus":
|
||||
stamps = dict(self.set_at_turn)
|
||||
stamps[name] = turn
|
||||
return replace(self, **{name: value}, set_at_turn=stamps)
|
||||
|
||||
def expire(self, turn_count: int, ttl: int = FOCUS_TTL_TURNS) -> "Focus":
|
||||
"""Drops every field older than the TTL, stamps included."""
|
||||
kept = {
|
||||
name: getattr(self, name)
|
||||
for name in ("drug_id", "drug_name", "section_key", "population", "verbosity")
|
||||
if self.is_fresh(name, turn_count, ttl)
|
||||
}
|
||||
stamps = {
|
||||
name: at for name, at in self.set_at_turn.items() if name in kept
|
||||
}
|
||||
return Focus(**kept, set_at_turn=stamps)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ConversationState:
|
||||
conversation_id: str
|
||||
recent: tuple[Turn, ...] = ()
|
||||
summary: str = ""
|
||||
focus: Focus = field(default_factory=Focus)
|
||||
turn_count: int = 0
|
||||
# Turns evicted from `recent` since the last time `overflow()` was
|
||||
# consumed and cleared (by the caller passing `pending_overflow=()` to
|
||||
# `replace()` after folding them into the summary). NOT derivable from
|
||||
# `recent` alone — `recent` is already capped at `window`, so comparing
|
||||
# its length against `window` can never find anything (see the bug note
|
||||
# on `overflow` below). Plumbing, not conversation content.
|
||||
pending_overflow: tuple[Turn, ...] = ()
|
||||
|
||||
def append(self, turn: Turn, window: int = RECENT_TURNS) -> "ConversationState":
|
||||
"""Adds a turn and evicts the oldest beyond the window.
|
||||
|
||||
Eviction accumulates the dropped turns into `pending_overflow` for
|
||||
the caller's summariser to fold via `overflow()`, rather than
|
||||
discarding them here — this type does not decide what a summary
|
||||
says. Accumulates rather than overwrites because one turn commonly
|
||||
triggers two `append()` calls in a row (user, then assistant); each
|
||||
can evict at most one turn, and the second call must not lose the
|
||||
first's.
|
||||
"""
|
||||
combined = (*self.recent, turn)
|
||||
recent = combined[-window:]
|
||||
dropped = combined[:-window] if len(combined) > window else ()
|
||||
return replace(
|
||||
self,
|
||||
recent=recent,
|
||||
turn_count=self.turn_count + 1,
|
||||
pending_overflow=(*self.pending_overflow, *dropped),
|
||||
)
|
||||
|
||||
def overflow(self) -> tuple[Turn, ...]:
|
||||
"""Turns evicted from `recent` and not yet folded into the summary.
|
||||
|
||||
Bug fixed 2026-08-06 (Codex review, F-06): this used to check
|
||||
`len(self.recent) > window`, but `recent` is already truncated to
|
||||
`window` by every `append()` call, so that comparison could never be
|
||||
true — dropped turns were silently discarded and the summariser
|
||||
never received them, no matter how long a conversation ran. The
|
||||
caller must clear `pending_overflow` (pass `pending_overflow=()` to
|
||||
`replace()`) after folding, or the same turns fold again next time.
|
||||
"""
|
||||
return self.pending_overflow
|
||||
|
||||
def inherited(self, name: str):
|
||||
"""A focus value only if it is still fresh; otherwise None."""
|
||||
return getattr(self.focus, name) if self.focus.is_fresh(name, self.turn_count) else None
|
||||
|
||||
|
||||
# --- follow-up resolution -----------------------------------------------------
|
||||
|
||||
# Phrases that mean "same question, different population". Longest-first for the
|
||||
# same reason `sections.py` sorts that way: "phụ nữ cho con bú" must be tested
|
||||
# before "phụ nữ", or the more specific reading is never reached.
|
||||
POPULATION_PHRASES: dict[str, str] = {
|
||||
"phụ nữ cho con bú": "phu_nu_cho_con_bu",
|
||||
"người cao tuổi": "nguoi_cao_tuoi",
|
||||
"phụ nữ có thai": "phu_nu_co_thai",
|
||||
"người suy thận": "suy_than",
|
||||
"người suy gan": "suy_gan",
|
||||
"trẻ sơ sinh": "tre_so_sinh",
|
||||
"người lớn": "nguoi_lon",
|
||||
"bà bầu": "phu_nu_co_thai",
|
||||
"trẻ nhỏ": "tre_em",
|
||||
"trẻ em": "tre_em",
|
||||
"người già": "nguoi_cao_tuoi",
|
||||
}
|
||||
|
||||
VERBOSITY_PHRASES: dict[str, Verbosity] = {
|
||||
"giải thích kỹ hơn": "detailed",
|
||||
"nói rõ hơn": "detailed",
|
||||
"chi tiết hơn": "detailed",
|
||||
"ngắn gọn": "concise",
|
||||
"tóm tắt": "concise",
|
||||
}
|
||||
|
||||
# A turn that is only a qualifier — no drug, no attribute — is a follow-up by
|
||||
# construction. These are the openers that mark one.
|
||||
FOLLOWUP_MARKERS = ("còn", "thế còn", "vậy còn", "so với", "thuốc vừa", "cái đó", "nó")
|
||||
|
||||
# Greetings, thanks, farewells and bare acknowledgements. A turn made up only of
|
||||
# these is social, not a failed drug lookup: answering "Chưa xác định được
|
||||
# thuốc" to "chào bạn" reads as broken. Longest-first so "cảm ơn nhiều" is
|
||||
# stripped before "cảm ơn".
|
||||
SMALLTALK_PHRASES = (
|
||||
"xin chào", "chào bạn", "chào ad", "cảm ơn nhiều", "cảm ơn bạn", "cám ơn",
|
||||
"cảm ơn", "tạm biệt", "hay quá", "tuyệt vời", "hiểu rồi", "được rồi",
|
||||
"chào", "hello", "hi", "alo", "thanks", "thank", "ok", "oke", "okie",
|
||||
"ừ", "uh", "haha", "hihi", "bye",
|
||||
)
|
||||
|
||||
|
||||
def is_smalltalk(text: str) -> bool:
|
||||
"""True when a turn carries nothing but social phrases.
|
||||
|
||||
Deliberately conservative: it strips every known social phrase and returns
|
||||
True only if what remains is empty. "chào bạn, liều paracetamol?" keeps
|
||||
"liều paracetamol" after stripping, so it is treated as a real question —
|
||||
a greeting must never swallow the medical part of a turn.
|
||||
"""
|
||||
remainder = _normalise(text).strip(" .,!?;:")
|
||||
for phrase in sorted(SMALLTALK_PHRASES, key=len, reverse=True):
|
||||
# Space-pad both sides so a short phrase ("hi", "ok") matches a whole
|
||||
# word only, never a substring of "chi" or "block".
|
||||
remainder = f" {remainder} ".replace(f" {phrase} ", " ").strip(" .,!?;:")
|
||||
return not remainder
|
||||
|
||||
|
||||
def _normalise(text: str) -> str:
|
||||
return " ".join(text.casefold().split())
|
||||
|
||||
|
||||
def _longest_first(phrases: dict[str, str]) -> list[tuple[str, str]]:
|
||||
return sorted(phrases.items(), key=lambda item: -len(item[0]))
|
||||
|
||||
|
||||
def detect_population(text: str) -> str | None:
|
||||
normalised = _normalise(text)
|
||||
for phrase, tag in _longest_first(POPULATION_PHRASES):
|
||||
if phrase in normalised:
|
||||
return tag
|
||||
return None
|
||||
|
||||
|
||||
def detect_verbosity(text: str) -> Verbosity | None:
|
||||
normalised = _normalise(text)
|
||||
for phrase, level in _longest_first(VERBOSITY_PHRASES):
|
||||
if phrase in normalised:
|
||||
return level
|
||||
return None
|
||||
|
||||
|
||||
def looks_like_followup(text: str) -> bool:
|
||||
normalised = _normalise(text)
|
||||
return any(normalised.startswith(marker) for marker in FOLLOWUP_MARKERS)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ResolvedQuestion:
|
||||
"""What this turn is asking, after the conversation is taken into account."""
|
||||
|
||||
text: str
|
||||
drug_id: str | None
|
||||
section_key: str | None
|
||||
population: str | None
|
||||
verbosity: Verbosity | None
|
||||
inherited_drug: bool
|
||||
inherited_section: bool
|
||||
|
||||
@property
|
||||
def needs_carry_over_notice(self) -> bool:
|
||||
"""Whether the answer must name what it inherited.
|
||||
|
||||
An inherited drug that is wrong is a wrong-drug answer, so the answer
|
||||
has to say which drug it decided this was about.
|
||||
"""
|
||||
return self.inherited_drug
|
||||
|
||||
|
||||
def resolve_against(
|
||||
state: ConversationState,
|
||||
text: str,
|
||||
drug_id: str | None,
|
||||
section_key: str | None,
|
||||
) -> ResolvedQuestion:
|
||||
"""Fills gaps in this turn from conversation focus, freshness permitting.
|
||||
|
||||
`drug_id` and `section_key` are what this turn resolved on its own — the
|
||||
existing resolvers decide those, unchanged. Only what the turn left blank
|
||||
is inherited, so an explicit mention always wins over context.
|
||||
"""
|
||||
inherited_drug = False
|
||||
inherited_section = False
|
||||
|
||||
if drug_id is None:
|
||||
carried = state.inherited("drug_id")
|
||||
if carried is not None:
|
||||
drug_id, inherited_drug = carried, True
|
||||
|
||||
if section_key is None:
|
||||
carried = state.inherited("section_key")
|
||||
if carried is not None:
|
||||
section_key, inherited_section = carried, True
|
||||
|
||||
population = detect_population(text) or state.inherited("population")
|
||||
verbosity = detect_verbosity(text) or state.inherited("verbosity")
|
||||
|
||||
return ResolvedQuestion(
|
||||
text=text,
|
||||
drug_id=drug_id,
|
||||
section_key=section_key,
|
||||
population=population,
|
||||
verbosity=verbosity,
|
||||
inherited_drug=inherited_drug,
|
||||
inherited_section=inherited_section,
|
||||
)
|
||||
|
||||
|
||||
def update_focus(
|
||||
state: ConversationState,
|
||||
resolved: ResolvedQuestion,
|
||||
) -> Focus:
|
||||
"""Focus after this turn, stamped with the current turn index."""
|
||||
focus = state.focus.expire(state.turn_count)
|
||||
turn = state.turn_count
|
||||
for name, value in (
|
||||
("drug_id", resolved.drug_id),
|
||||
("section_key", resolved.section_key),
|
||||
("population", resolved.population),
|
||||
("verbosity", resolved.verbosity),
|
||||
):
|
||||
if value is not None:
|
||||
focus = focus.with_field(name, value, turn)
|
||||
return focus
|
||||
|
||||
|
||||
# --- persistence and summary --------------------------------------------------
|
||||
#
|
||||
# Protocol + no-LLM default co-located, matching how `reasoning.py` ships
|
||||
# `SufficiencyAssessor`/`DeterministicAssessor` and `metrics.py` ships
|
||||
# `Metrics`/`NullMetrics`. The Postgres-backed store lives in `adapters/`.
|
||||
|
||||
|
||||
class ConversationStore(Protocol):
|
||||
"""Loads and persists one conversation's state.
|
||||
|
||||
`load` returns a fresh empty state for an unknown id rather than raising: a
|
||||
first turn has no prior state, and that is not an error.
|
||||
"""
|
||||
|
||||
def load(self, conversation_id: str) -> "ConversationState": ...
|
||||
def save(self, state: "ConversationState") -> None: ...
|
||||
|
||||
|
||||
class InMemoryConversationStore:
|
||||
"""Reference implementation and the offline/test default."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._states: dict[str, ConversationState] = {}
|
||||
|
||||
def load(self, conversation_id: str) -> ConversationState:
|
||||
return self._states.get(conversation_id, ConversationState(conversation_id))
|
||||
|
||||
def save(self, state: ConversationState) -> None:
|
||||
self._states[state.conversation_id] = state
|
||||
|
||||
|
||||
class Summariser(Protocol):
|
||||
"""Folds turns evicted from the recent window into rolling prose.
|
||||
|
||||
Contract, load-bearing for safety: the summary records *what was discussed*,
|
||||
never a clinical value. A dose copied into a summary carries no citation and
|
||||
cannot be grounding-verified — the check compares against retrieved
|
||||
evidence, and a summary is not evidence.
|
||||
"""
|
||||
|
||||
def fold(self, prev_summary: str, dropped: tuple["Turn", ...]) -> str: ...
|
||||
|
||||
|
||||
class DeterministicSummariser:
|
||||
"""No-LLM default: one topic line per evicted user turn, capped.
|
||||
|
||||
Records only the drug and section a turn was *about* — labels, never cell
|
||||
values — so the no-clinical-content rule holds by construction rather than
|
||||
by trusting a generator not to leak a dose.
|
||||
"""
|
||||
|
||||
MAX_CHARS = 1600 # ~400 tokens, per ADR 0007 §2
|
||||
|
||||
def fold(self, prev_summary: str, dropped: tuple[Turn, ...]) -> str:
|
||||
lines = [prev_summary] if prev_summary else []
|
||||
for turn in dropped:
|
||||
if turn.role != "user":
|
||||
continue
|
||||
drug = turn.drug_id or "thuốc chưa xác định"
|
||||
section = turn.section_key or "thông tin chung"
|
||||
lines.append(f"- đã hỏi {section} của {drug}")
|
||||
text = "\n".join(lines)
|
||||
# Keep the most recent topics when over budget: drop oldest lines, not
|
||||
# mid-line characters, so the summary never ends on a fragment.
|
||||
while len(text) > self.MAX_CHARS and len(lines) > 1:
|
||||
lines.pop(0)
|
||||
text = "\n".join(lines)
|
||||
return text
|
||||
@@ -1,424 +0,0 @@
|
||||
"""Orchestration: turns a stateless single-turn engine into a conversation.
|
||||
|
||||
This is the glue ADR 0007 specified and nothing yet called. It owns no rules of
|
||||
its own — inheritance lives in `conversation.py`, the bounded loop in
|
||||
`reasoning.py`, grounding in `grounding.py`. Its whole job is the sequence:
|
||||
|
||||
load state
|
||||
→ resolve this turn, then inherit gaps from focus
|
||||
→ derive clarify signals from resolver state (never a model score)
|
||||
→ run the bounded loop (retrieve / generate / verify)
|
||||
→ update focus, append turns, summarise overflow, save
|
||||
→ name any inherited drug in the answer
|
||||
|
||||
Everything here runs with no LLM and no live service: the collaborators are
|
||||
protocols, so a turn can be exercised end-to-end with fakes.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, replace
|
||||
from typing import Protocol
|
||||
|
||||
from . import metrics as metric_names
|
||||
from .answer import GroundedAnswer, GroundedAnswerService
|
||||
from .conversation import (
|
||||
ConversationState,
|
||||
ConversationStore,
|
||||
Summariser,
|
||||
Turn,
|
||||
is_smalltalk,
|
||||
resolve_against,
|
||||
update_focus,
|
||||
)
|
||||
from .metrics import Metrics, NullMetrics
|
||||
from .models import EvidenceDecision, QueryIntent, SubjectScope
|
||||
from .reasoning import (
|
||||
BudgetExhausted,
|
||||
Clarification,
|
||||
ClarifyReason,
|
||||
DeterministicAssessor,
|
||||
Generate,
|
||||
LoopOutcome,
|
||||
MAX_RETRIEVAL_ROUNDS,
|
||||
Retrieve,
|
||||
SufficiencyAssessor,
|
||||
TurnBudget,
|
||||
clarify_for,
|
||||
run_turn,
|
||||
)
|
||||
from .routing import CatalogDrugResolver, DrugResolutionStatus, normalize_name
|
||||
from .sections import SectionResolver
|
||||
|
||||
# Turns that only confirm a prior suggestion. They resolve no drug and must not
|
||||
# be fuzzy-matched against the catalog (which returns garbage like terbinafin).
|
||||
# Stored normalised (normalize_name strips diacritics: "đúng" -> "dung"), or the
|
||||
# lookup below never matches.
|
||||
_CONFIRMATION_WORDS = (
|
||||
"đúng", "đúng rồi", "đúng vậy", "phải", "phải rồi", "chuẩn", "chuẩn rồi",
|
||||
"chính xác", "ừ", "uh", "ok", "oke", "yes", "vâng",
|
||||
)
|
||||
_CONFIRMATIONS = frozenset(normalize_name(word) for word in _CONFIRMATION_WORDS)
|
||||
|
||||
|
||||
def _is_confirmation(text: str) -> bool:
|
||||
return normalize_name(text) in _CONFIRMATIONS
|
||||
|
||||
SUMMARY_EVERY = 4 # regenerate the summary at most every S turns, per ADR 0007 §2
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TurnResolution:
|
||||
"""What one turn resolved on its own, before conversation is considered.
|
||||
|
||||
`drug_status` is the resolver's verdict — resolved / not_found / ambiguous —
|
||||
kept distinct from `drug_id` so an ambiguous turn (asks which drug) reads
|
||||
differently from a bare follow-up (inherits the drug).
|
||||
"""
|
||||
|
||||
drug_id: str | None
|
||||
section_key: str | None
|
||||
drug_status: str
|
||||
|
||||
|
||||
class TurnResolverPort(Protocol):
|
||||
def resolve_turn(self, text: str) -> TurnResolution: ...
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TurnResponse:
|
||||
answer: str | None
|
||||
clarification: Clarification | None
|
||||
evidence_texts: tuple[str, ...]
|
||||
stopped_because: str
|
||||
inherited_drug: str | None
|
||||
generated: bool
|
||||
|
||||
|
||||
class ConversationalRagService:
|
||||
def __init__(
|
||||
self,
|
||||
store: ConversationStore,
|
||||
summariser: Summariser,
|
||||
resolver: TurnResolverPort,
|
||||
retrieve: Retrieve,
|
||||
generate: Generate,
|
||||
metrics: Metrics | None = None,
|
||||
summary_every: int = SUMMARY_EVERY,
|
||||
) -> None:
|
||||
self._store = store
|
||||
self._summariser = summariser
|
||||
self._resolver = resolver
|
||||
self._retrieve = retrieve
|
||||
self._generate = generate
|
||||
self._metrics = metrics or NullMetrics()
|
||||
self._summary_every = summary_every
|
||||
|
||||
def answer(
|
||||
self, conversation_id: str, text: str, budget: TurnBudget | None = None
|
||||
) -> TurnResponse:
|
||||
state = self._store.load(conversation_id)
|
||||
|
||||
turn = self._resolver.resolve_turn(text)
|
||||
resolved = resolve_against(state, text, turn.drug_id, turn.section_key)
|
||||
|
||||
signals = self._clarify_signals(resolved, turn)
|
||||
if resolved.inherited_drug:
|
||||
self._metrics.increment(metric_names.FOLLOWUP_INHERITED)
|
||||
|
||||
outcome = run_turn(
|
||||
state,
|
||||
resolved,
|
||||
self._retrieve,
|
||||
self._generate,
|
||||
clarify_signals=signals,
|
||||
budget=budget or TurnBudget(),
|
||||
metrics=self._metrics,
|
||||
)
|
||||
|
||||
self._persist(state, resolved, outcome)
|
||||
|
||||
answer = outcome.answer
|
||||
inherited = resolved.drug_id if resolved.needs_carry_over_notice else None
|
||||
if answer is not None and inherited is not None:
|
||||
# An inherited drug that is wrong is a wrong-drug answer, so the
|
||||
# answer has to say which drug it decided this was about.
|
||||
answer = f"Về {inherited}: {answer}"
|
||||
|
||||
return TurnResponse(
|
||||
answer=answer,
|
||||
clarification=outcome.clarification,
|
||||
evidence_texts=outcome.evidence_texts,
|
||||
stopped_because=outcome.stopped_because,
|
||||
inherited_drug=inherited,
|
||||
generated=outcome.generated,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _clarify_signals(resolved, turn: TurnResolution) -> tuple[str, ...]:
|
||||
"""Resolver states that should ask instead of guess.
|
||||
|
||||
Only fires when the drug is *still* unknown after inheritance: a
|
||||
follow-up like "còn trẻ em thì sao?" names no drug but inherits one, and
|
||||
must not be turned into a clarify.
|
||||
"""
|
||||
if resolved.drug_id is None:
|
||||
return (ClarifyReason.AMBIGUOUS_DRUG,)
|
||||
return ()
|
||||
|
||||
def _persist(
|
||||
self, state: ConversationState, resolved, outcome: LoopOutcome
|
||||
) -> None:
|
||||
focus = update_focus(state, resolved)
|
||||
state = ConversationState(
|
||||
conversation_id=state.conversation_id,
|
||||
recent=state.recent,
|
||||
summary=state.summary,
|
||||
focus=focus,
|
||||
turn_count=state.turn_count,
|
||||
)
|
||||
state = state.append(
|
||||
Turn("user", resolved.text, _now(), resolved.drug_id, resolved.section_key)
|
||||
)
|
||||
if outcome.answer is not None:
|
||||
state = state.append(
|
||||
Turn(
|
||||
"assistant",
|
||||
outcome.answer,
|
||||
_now(),
|
||||
resolved.drug_id,
|
||||
resolved.section_key,
|
||||
evidence_ids=tuple(str(i) for i in range(len(outcome.evidence_texts))),
|
||||
)
|
||||
)
|
||||
if state.turn_count % self._summary_every == 0 and state.overflow():
|
||||
summary = self._summariser.fold(state.summary, state.overflow())
|
||||
state = ConversationState(
|
||||
conversation_id=state.conversation_id,
|
||||
recent=state.recent,
|
||||
summary=summary,
|
||||
focus=state.focus,
|
||||
turn_count=state.turn_count,
|
||||
)
|
||||
self._store.save(state)
|
||||
|
||||
|
||||
def _now() -> str:
|
||||
# Timestamps are provenance, not logic; the domain never branches on them,
|
||||
# so a monotonic placeholder keeps this module free of wall-clock coupling.
|
||||
return ""
|
||||
|
||||
|
||||
# --- live chat core -----------------------------------------------------------
|
||||
#
|
||||
# The deployable multi-turn path. The loop is what *understands and clarifies*
|
||||
# a turn; retrieval, citation, VERIFY_PDF and grounding stay inside
|
||||
# GroundedAnswerService, untouched — so clarify + refine are added *around* the
|
||||
# safe engine, never inside it.
|
||||
|
||||
SMALLTALK_REPLY = (
|
||||
"Mình tra cứu Dược thư Quốc gia Việt Nam. Bạn muốn hỏi về thuốc nào, "
|
||||
"hoặc thuộc tính nào (liều dùng, chống chỉ định, tương tác…)?"
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ConversationTurnResult:
|
||||
answer: str | None
|
||||
clarification: Clarification | None
|
||||
grounded: GroundedAnswer | None
|
||||
smalltalk: bool
|
||||
inherited_drug: str | None
|
||||
reason: str
|
||||
|
||||
|
||||
class ConversationalLoopService:
|
||||
def __init__(
|
||||
self,
|
||||
answers: GroundedAnswerService,
|
||||
resolver: CatalogDrugResolver,
|
||||
section_resolver: SectionResolver,
|
||||
store: ConversationStore,
|
||||
assessor: SufficiencyAssessor | None = None,
|
||||
summariser: Summariser | None = None,
|
||||
metrics: Metrics | None = None,
|
||||
) -> None:
|
||||
self._answers = answers
|
||||
self._resolver = resolver
|
||||
self._section_resolver = section_resolver
|
||||
self._store = store
|
||||
self._assessor = assessor or DeterministicAssessor()
|
||||
self._summariser = summariser
|
||||
self._metrics = metrics or NullMetrics()
|
||||
|
||||
def answer(
|
||||
self,
|
||||
conversation_id: str,
|
||||
query: str,
|
||||
subject_scope: SubjectScope,
|
||||
intent: QueryIntent,
|
||||
budget: TurnBudget | None = None,
|
||||
) -> ConversationTurnResult:
|
||||
state = self._store.load(conversation_id)
|
||||
|
||||
resolution = self._resolver.resolve(query)
|
||||
# Only an EXACT name is auto-accepted. A fuzzy match (score < 1.0) is a
|
||||
# guess, and a formulary must not silently answer about a *different*
|
||||
# drug than the one meant — a typo is asked about ("did you mean…?"),
|
||||
# never resolved on a similarity threshold. Autocomplete at input is the
|
||||
# first line; this is the backstop when a wrong name is still submitted.
|
||||
is_exact = resolution.status == DrugResolutionStatus.RESOLVED and (
|
||||
resolution.score is None or resolution.score >= 0.999
|
||||
)
|
||||
drug_self = resolution.drug_id if is_exact else None
|
||||
|
||||
# Social turn that names no drug: answer as a person, not a failed lookup.
|
||||
if drug_self is None and is_smalltalk(query):
|
||||
self._append_user(state, query, None, None)
|
||||
return ConversationTurnResult(
|
||||
SMALLTALK_REPLY, None, None, True, None, "smalltalk"
|
||||
)
|
||||
|
||||
section = self._section_resolver.resolve(query)
|
||||
section_self = section.section_key if section else None
|
||||
resolved = resolve_against(state, query, drug_self, section_self)
|
||||
|
||||
# Clarify beats guessing: no drug even after inheritance. If the text is
|
||||
# a near-miss for real drug names, offer them ("did you mean") rather
|
||||
# than a bare "which drug?" — a typo should not dead-end.
|
||||
if resolved.drug_id is None:
|
||||
# A bare confirmation ("đúng") with no drug in context is not a drug
|
||||
# lookup — never fuzzy-match it (that returned terbinafin/tretinoin).
|
||||
if _is_confirmation(query):
|
||||
reason = "confirm_without_context"
|
||||
clarification = Clarification(
|
||||
reason=reason,
|
||||
question="Bạn muốn xác nhận thuốc nào? Vui lòng gõ tên thuốc để mình tra cứu.",
|
||||
options=(),
|
||||
)
|
||||
self._metrics.increment(metric_names.CLARIFY_ASKED, reason=reason)
|
||||
self._persist(state, resolved, None)
|
||||
return ConversationTurnResult(None, clarification, None, False, None, reason)
|
||||
# Only offer "did you mean" for a SHORT, drug-name-shaped miss (a
|
||||
# typo). Fuzzy-matching a whole sentence ("EPO điều trị thiếu máu…")
|
||||
# or a confirmation ("đúng") against 684 aliases returns confident
|
||||
# garbage — that is the did-you-mean loop the reviewer hit. A long or
|
||||
# confirming turn that resolves no drug is answered honestly, not
|
||||
# with a list of unrelated drugs.
|
||||
looks_like_name = len(normalize_name(query).split()) <= 4
|
||||
suggestions = (
|
||||
self._resolver.suggest(query, k=3, min_score=0.72)
|
||||
if looks_like_name and not _is_confirmation(query)
|
||||
else []
|
||||
)
|
||||
if suggestions:
|
||||
names = [self._drug_name(drug_id) for drug_id, _ in suggestions]
|
||||
reason = "did_you_mean"
|
||||
clarification = Clarification(
|
||||
reason=reason,
|
||||
question=f"Ý bạn là: {', '.join(names)}?",
|
||||
options=tuple(names),
|
||||
)
|
||||
else:
|
||||
reason = "drug_not_supported"
|
||||
clarification = Clarification(
|
||||
reason=reason,
|
||||
question=(
|
||||
"Không có thuốc này trong Dược thư Quốc gia. Vui lòng kiểm "
|
||||
"tra lại tên, hoặc gõ vài ký tự để chọn từ gợi ý."
|
||||
),
|
||||
options=(),
|
||||
)
|
||||
self._metrics.increment(metric_names.CLARIFY_ASKED, reason=reason)
|
||||
self._persist(state, resolved, None)
|
||||
return ConversationTurnResult(None, clarification, None, False, None, reason)
|
||||
if resolved.inherited_drug:
|
||||
self._metrics.increment(metric_names.FOLLOWUP_INHERITED)
|
||||
|
||||
# One call to the safe engine. The drug is passed already-resolved (incl.
|
||||
# an inherited follow-up drug), so the engine does NOT re-resolve it from
|
||||
# the turn text — that double-resolution is what abstained follow-ups as
|
||||
# "ambiguous". The turn's own text drives section routing; when it names
|
||||
# no attribute the drug-overview + rerank path finds the relevant part.
|
||||
grounded: GroundedAnswer | None = self._answers.answer(
|
||||
query, subject_scope, intent, drug_id=resolved.drug_id
|
||||
)
|
||||
|
||||
# The engine asked for a missing detail (age/weight/renal function/
|
||||
# indication…) rather than dumping every dose band — surface it as a
|
||||
# clarification, not a final answer.
|
||||
if grounded is not None and grounded.clarification is not None:
|
||||
self._metrics.increment(
|
||||
metric_names.CLARIFY_ASKED, reason="needs_more_info"
|
||||
)
|
||||
self._persist(state, resolved, None)
|
||||
return ConversationTurnResult(
|
||||
None,
|
||||
Clarification(
|
||||
reason="needs_more_info",
|
||||
question=grounded.clarification,
|
||||
options=(),
|
||||
),
|
||||
None,
|
||||
False,
|
||||
None,
|
||||
"needs_more_info",
|
||||
)
|
||||
|
||||
answer = grounded.answer if grounded else None
|
||||
inherited = resolved.drug_id if resolved.needs_carry_over_notice else None
|
||||
if answer is not None and inherited is not None:
|
||||
answer = f"Về {self._drug_name(inherited)}: {answer}"
|
||||
if grounded is not None:
|
||||
grounded = replace(grounded, answer=answer)
|
||||
|
||||
self._persist(state, resolved, grounded)
|
||||
return ConversationTurnResult(
|
||||
answer,
|
||||
None,
|
||||
grounded,
|
||||
False,
|
||||
inherited,
|
||||
grounded.result.reason if grounded else "no_answer",
|
||||
)
|
||||
|
||||
def complete(self, prefix: str, k: int = 8) -> list[str]:
|
||||
"""Display names matching a typed prefix, for input autocomplete."""
|
||||
return [self._drug_name(drug_id) for drug_id in self._resolver.complete(prefix, k)]
|
||||
|
||||
@staticmethod
|
||||
def _drug_name(drug_id: str) -> str:
|
||||
"""A readable display name from a drug id ('paracetamol_acetaminophen')."""
|
||||
return drug_id.replace("_", " ").title()
|
||||
|
||||
def _append_user(self, state, text, drug_id, section_key) -> None:
|
||||
state = state.append(Turn("user", text, _now(), drug_id, section_key))
|
||||
self._store.save(state)
|
||||
|
||||
def _persist(self, state, resolved, grounded) -> None:
|
||||
focus = update_focus(state, resolved)
|
||||
state = replace(state, focus=focus)
|
||||
state = state.append(
|
||||
Turn("user", resolved.text, _now(), resolved.drug_id, resolved.section_key)
|
||||
)
|
||||
if grounded is not None and grounded.answer is not None:
|
||||
state = state.append(
|
||||
Turn(
|
||||
"assistant",
|
||||
grounded.answer,
|
||||
_now(),
|
||||
resolved.drug_id,
|
||||
resolved.section_key,
|
||||
)
|
||||
)
|
||||
if (
|
||||
self._summariser is not None
|
||||
and state.turn_count % SUMMARY_EVERY == 0
|
||||
and state.overflow()
|
||||
):
|
||||
summary = self._summariser.fold(state.summary, state.overflow())
|
||||
# Clear what was just folded — `replace()` keeps every field not
|
||||
# named here, and `pending_overflow` accumulates across calls
|
||||
# (see `ConversationState.append`), so leaving it would fold the
|
||||
# same already-summarised turns again next time.
|
||||
state = replace(state, summary=summary, pending_overflow=())
|
||||
self._store.save(state)
|
||||
@@ -33,7 +33,14 @@ Quy tắc bắt buộc:
|
||||
[n] ở mọi câu — gắn một lần cho một cụm cùng nguồn là đủ. Không bịa số [n].
|
||||
5. Nếu BẰNG CHỨNG không đủ (thiếu đối tượng được hỏi, thiếu con số, hoặc chỉ nói
|
||||
chung chung), nói rõ là không đủ và đặt evidence_sufficient=false. Đó là câu
|
||||
trả lời hợp lệ. Không suy diễn để lấp chỗ trống.
|
||||
trả lời hợp lệ. Không suy diễn để lấp chỗ trống. TRƯỜNG HỢP NÀY BẮT BUỘC LUÔN
|
||||
điền `clarifying_question` giải thích NGẮN GỌN, CỤ THỂ vì sao — dù đó là vì
|
||||
người dùng chưa nêu đủ dữ kiện (hỏi lại dữ kiện còn thiếu, như quy tắc 7), hay
|
||||
đơn giản là chuyên luận KHÔNG đề cập nội dung này cho đối tượng/đường dùng
|
||||
đang hỏi (nói thẳng điều đó, ví dụ "Dược thư không nêu liều dùng đường nhỏ
|
||||
mắt của thuốc này"). TUYỆT ĐỐI KHÔNG để `clarifying_question`=null khi
|
||||
evidence_sufficient=false — một câu giải thích cụ thể luôn hữu ích hơn cho
|
||||
người đọc so với việc để trống.
|
||||
6. Giữ nguyên thuật ngữ chuyên môn của sách. Không diễn giải cho người
|
||||
không chuyên.
|
||||
7. HỎI LẠI khi thiếu dữ kiện — ĐÂY LÀ QUY TẮC QUAN TRỌNG NHẤT, ưu tiên hơn việc
|
||||
@@ -70,8 +77,11 @@ ANSWER_SCHEMA = {
|
||||
"clarifying_question": {
|
||||
"type": ["string", "null"],
|
||||
"description": (
|
||||
"Câu hỏi lại khi người dùng chưa nêu đủ dữ kiện (vd tuổi/cân nặng "
|
||||
"cho câu hỏi liều có nhiều mức). null nếu đã đủ dữ kiện để trả lời."
|
||||
"BẮT BUỘC khi evidence_sufficient=false: câu hỏi lại khi người dùng "
|
||||
"chưa nêu đủ dữ kiện (vd tuổi/cân nặng), HOẶC — nếu vấn đề là "
|
||||
"chuyên luận không có nội dung này — một câu nói thẳng điều đó (vd "
|
||||
"'Dược thư không nêu liều dùng đường nhỏ mắt của thuốc này'). "
|
||||
"null CHỈ khi evidence_sufficient=true."
|
||||
),
|
||||
},
|
||||
},
|
||||
@@ -94,18 +104,32 @@ Quy tắc:
|
||||
- Câu hỏi KHÔNG về liều (chống chỉ định, tương tác, tác dụng phụ, giới thiệu
|
||||
thuốc…) thì thường ĐỦ.
|
||||
|
||||
Trả về DUY NHẤT JSON: {"sufficient": bool, "clarifying_question": string|null}.
|
||||
Trả về DUY NHẤT JSON: {"sufficient": bool, "clarifying_question": string|null,
|
||||
"quick_replies": string[]}.
|
||||
Nếu CHƯA đủ: sufficient=false và clarifying_question hỏi NGẮN GỌN tất cả dữ kiện
|
||||
còn thiếu (vd: "Bé mấy tuổi, cân nặng bao nhiêu kg, dùng đường nào và để hạ sốt
|
||||
hay giảm đau?"). Nếu đủ: sufficient=true, clarifying_question=null."""
|
||||
hay giảm đau?"). Nếu đủ: sufficient=true, clarifying_question=null,
|
||||
quick_replies=[].
|
||||
|
||||
quick_replies: 2-4 phương án trả lời NGẮN (dưới ~20 ký tự mỗi phương án) cho
|
||||
CHÍNH câu clarifying_question vừa đặt ra, để người dùng bấm chọn thay vì gõ —
|
||||
CHỈ khi câu hỏi thực sự có vài lựa chọn rời rạc, tự nhiên (vd đối tượng: "Người
|
||||
lớn" / "Trẻ em"; đường dùng: "Uống" / "Tiêm"). Để mảng RỖNG nếu câu hỏi cần một
|
||||
con số cụ thể không có sẵn lựa chọn ngắn (vd hỏi cân nặng chính xác) — không
|
||||
được bịa ra các phương án number-ish giả."""
|
||||
|
||||
SUFFICIENCY_SCHEMA = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"sufficient": {"type": "boolean"},
|
||||
"clarifying_question": {"type": ["string", "null"]},
|
||||
"quick_replies": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
"maxItems": 4,
|
||||
},
|
||||
},
|
||||
"required": ["sufficient", "clarifying_question"],
|
||||
"required": ["sufficient", "clarifying_question", "quick_replies"],
|
||||
"additionalProperties": False,
|
||||
}
|
||||
|
||||
@@ -180,7 +204,8 @@ def build_entailment_request(claims: list[tuple[str, str]]) -> "GenerationReques
|
||||
|
||||
|
||||
def build_request(
|
||||
question: str, evidence_texts: tuple[str, ...], intro: bool = False
|
||||
question: str, evidence_texts: tuple[str, ...], intro: bool = False,
|
||||
list_mode: bool = False,
|
||||
) -> GenerationRequest:
|
||||
"""The prompt for one question over one ordered evidence list.
|
||||
|
||||
@@ -191,6 +216,14 @@ def build_request(
|
||||
`intro=True` is the "user typed only a drug name" case: instead of restating
|
||||
a section, write a short introduction — what the drug is, its class and its
|
||||
main indication — then invite a specific follow-up. Still evidence-only.
|
||||
|
||||
`list_mode=True` is the symptom_to_drug reverse-lookup case: each evidence
|
||||
block is a DIFFERENT drug's own `chi_dinh`, not alternative phrasings of
|
||||
one drug's section — found live 2026-08-07 that without this the model
|
||||
picked just one drug out of 8 real matches and answered only about that
|
||||
one, silently dropping the rest. Not a treatment ranking — a factual list
|
||||
([[feedback_no_recommendation_gate]] already covers why a "which is best"
|
||||
framing would be wrong for this audience anyway).
|
||||
"""
|
||||
if not evidence_texts:
|
||||
raise ValueError("cannot build a grounded prompt with no evidence")
|
||||
@@ -206,6 +239,16 @@ def build_request(
|
||||
"lượng. Kết thúc bằng một câu mời hỏi tiếp về thuộc tính cụ thể (liều "
|
||||
"dùng, chống chỉ định, thận trọng, tương tác…)."
|
||||
)
|
||||
elif list_mode:
|
||||
task = (
|
||||
f"CÂU HỎI: {question}\n\nMỖI đoạn BẰNG CHỨNG trên là chỉ định của MỘT "
|
||||
"thuốc KHÁC NHAU. Hãy LIỆT KÊ TẤT CẢ các thuốc mà bằng chứng cho thấy "
|
||||
"có chỉ định phù hợp với câu hỏi — không chỉ chọn một thuốc. Mỗi thuốc "
|
||||
"một câu ngắn, gắn đúng số nguồn [n] của thuốc đó. Đây là liệt kê tra "
|
||||
"cứu, KHÔNG phải khuyến cáo thuốc nào tốt hơn — không xếp hạng, không "
|
||||
"chọn thuốc \"phù hợp nhất\". Nếu KHÔNG thuốc nào trong bằng chứng thực "
|
||||
"sự phù hợp với câu hỏi, nói rõ điều đó thay vì liệt kê thuốc không liên quan."
|
||||
)
|
||||
else:
|
||||
task = f"CÂU HỎI: {question}"
|
||||
user = f"BẰNG CHỨNG:\n\n{blocks}\n\n{task}"
|
||||
|
||||
@@ -1,305 +0,0 @@
|
||||
"""The bounded reasoning loop.
|
||||
|
||||
Understand → plan → retrieve → assess → refine → generate → verify → repair.
|
||||
Every edge is bounded, and every budget is decremented **before** the call it
|
||||
pays for, so exhaustion degrades to the best answer so far rather than to an
|
||||
error.
|
||||
|
||||
Two rules hold across every path and are the reason this can be added to a
|
||||
formulary at all:
|
||||
|
||||
- `grounding.verify` still gates every generated answer. Reasoning chooses what
|
||||
to look up and how to phrase it; it is never a source of facts.
|
||||
- A clarify signal bypasses the loop entirely. Asking beats guessing, and the
|
||||
signals are resolver states — ambiguous drug, unresolved attribute — not a
|
||||
model's confidence score.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field, replace
|
||||
from typing import Protocol
|
||||
|
||||
from . import metrics as metric_names
|
||||
from .conversation import ConversationState, ResolvedQuestion
|
||||
from .metrics import Metrics, NullMetrics
|
||||
|
||||
MAX_RETRIEVAL_ROUNDS = 2
|
||||
MAX_REPAIRS = 1
|
||||
MAX_LLM_CALLS = 4
|
||||
MAX_WALL_CLOCK_MS = 20_000
|
||||
|
||||
|
||||
class BudgetExhausted(RuntimeError):
|
||||
"""Raised only inside the loop, never surfaced; the loop catches it."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class TurnBudget:
|
||||
"""Mutable on purpose: one budget is threaded through one turn."""
|
||||
|
||||
llm_calls: int = MAX_LLM_CALLS
|
||||
retrieval_rounds: int = MAX_RETRIEVAL_ROUNDS
|
||||
repairs: int = MAX_REPAIRS
|
||||
wall_clock_ms: int = MAX_WALL_CLOCK_MS
|
||||
elapsed_ms: int = 0
|
||||
|
||||
def spend_llm(self) -> None:
|
||||
if self.llm_calls <= 0:
|
||||
raise BudgetExhausted("llm_calls")
|
||||
self.llm_calls -= 1
|
||||
|
||||
def spend_retrieval(self) -> None:
|
||||
if self.retrieval_rounds <= 0:
|
||||
raise BudgetExhausted("retrieval_rounds")
|
||||
self.retrieval_rounds -= 1
|
||||
|
||||
def spend_repair(self) -> None:
|
||||
if self.repairs <= 0:
|
||||
raise BudgetExhausted("repairs")
|
||||
self.repairs -= 1
|
||||
|
||||
def out_of_time(self) -> bool:
|
||||
return self.elapsed_ms >= self.wall_clock_ms
|
||||
|
||||
|
||||
class ClarifyReason:
|
||||
AMBIGUOUS_DRUG = "ambiguous_drug"
|
||||
NO_ATTRIBUTE = "no_attribute"
|
||||
MULTI_ATTRIBUTE = "multi_attribute"
|
||||
STILL_INSUFFICIENT = "still_insufficient"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Clarification:
|
||||
reason: str
|
||||
question: str
|
||||
options: tuple[str, ...] = ()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Sufficiency:
|
||||
"""The assessor's verdict on retrieved evidence.
|
||||
|
||||
`missing` must name something specific — a section, a population, a second
|
||||
drug. "Feels incomplete" does not buy a retrieval round; a round is only
|
||||
spent when there is a concrete thing to go and fetch.
|
||||
"""
|
||||
|
||||
sufficient: bool
|
||||
missing: str | None = None
|
||||
refined_query: str | None = None
|
||||
|
||||
|
||||
class SufficiencyAssessor(Protocol):
|
||||
def assess(
|
||||
self, resolved: ResolvedQuestion, evidence_texts: tuple[str, ...]
|
||||
) -> Sufficiency: ...
|
||||
|
||||
|
||||
class DeterministicAssessor:
|
||||
"""The no-LLM default, and the reference for what the port must do.
|
||||
|
||||
Runs offline and is what the loop uses until a provider is enabled. It only
|
||||
reports insufficiency it can *demonstrate* — a population was asked for and
|
||||
no retrieved text mentions it — so it can never spin the loop on a feeling.
|
||||
"""
|
||||
|
||||
POPULATION_TERMS = {
|
||||
"nguoi_lon": ("người lớn",),
|
||||
"tre_em": ("trẻ em", "trẻ nhỏ", "trẻ "),
|
||||
"tre_so_sinh": ("sơ sinh",),
|
||||
"phu_nu_co_thai": ("thai", "mang thai"),
|
||||
"phu_nu_cho_con_bu": ("cho con bú", "sữa mẹ"),
|
||||
"nguoi_cao_tuoi": ("người cao tuổi", "người già"),
|
||||
"suy_than": ("suy thận", "clcr"),
|
||||
"suy_gan": ("suy gan",),
|
||||
}
|
||||
|
||||
def assess(
|
||||
self, resolved: ResolvedQuestion, evidence_texts: tuple[str, ...]
|
||||
) -> Sufficiency:
|
||||
if not evidence_texts:
|
||||
return Sufficiency(False, missing="no_evidence")
|
||||
if resolved.population is None:
|
||||
return Sufficiency(True)
|
||||
|
||||
terms = self.POPULATION_TERMS.get(resolved.population, ())
|
||||
haystack = " ".join(evidence_texts).casefold()
|
||||
if any(term in haystack for term in terms):
|
||||
return Sufficiency(True)
|
||||
return Sufficiency(
|
||||
False,
|
||||
missing=f"population:{resolved.population}",
|
||||
refined_query=f"{resolved.text} {terms[0] if terms else ''}".strip(),
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class LoopOutcome:
|
||||
"""What one turn produced, plus what it cost."""
|
||||
|
||||
answer: str | None
|
||||
clarification: Clarification | None
|
||||
evidence_texts: tuple[str, ...]
|
||||
retrieval_rounds_used: int
|
||||
repairs_used: int
|
||||
stopped_because: str
|
||||
generated: bool = False
|
||||
|
||||
|
||||
@dataclass
|
||||
class LoopTrace:
|
||||
"""Ordered record of stages, for the dashboard and for debugging."""
|
||||
|
||||
stages: list[str] = field(default_factory=list)
|
||||
|
||||
def enter(self, stage: str) -> None:
|
||||
self.stages.append(stage)
|
||||
|
||||
|
||||
def clarify_for(
|
||||
reason: str, options: tuple[str, ...] = ()
|
||||
) -> Clarification:
|
||||
questions = {
|
||||
ClarifyReason.NO_ATTRIBUTE: (
|
||||
"Anh/chị muốn tra thuộc tính nào của thuốc này?"
|
||||
),
|
||||
ClarifyReason.AMBIGUOUS_DRUG: (
|
||||
"Câu hỏi có thể ứng với nhiều thuốc. Anh/chị muốn tra thuốc nào?"
|
||||
),
|
||||
ClarifyReason.MULTI_ATTRIBUTE: (
|
||||
"Câu hỏi nhắc tới nhiều mục. Anh/chị muốn xem mục nào trước?"
|
||||
),
|
||||
ClarifyReason.STILL_INSUFFICIENT: (
|
||||
"Chưa tìm đủ căn cứ trong Dược thư cho ý này. "
|
||||
"Anh/chị có thể nêu rõ hơn điều cần tra không?"
|
||||
),
|
||||
}
|
||||
return Clarification(reason, questions[reason], options)
|
||||
|
||||
|
||||
class Retrieve(Protocol):
|
||||
def __call__(self, resolved: ResolvedQuestion) -> tuple[str, ...]: ...
|
||||
|
||||
|
||||
class Generate(Protocol):
|
||||
def __call__(
|
||||
self, resolved: ResolvedQuestion, evidence: tuple[str, ...], state: ConversationState
|
||||
) -> str | None: ...
|
||||
|
||||
|
||||
def run_turn(
|
||||
state: ConversationState,
|
||||
resolved: ResolvedQuestion,
|
||||
retrieve: Retrieve,
|
||||
generate: Generate,
|
||||
clarify_signals: tuple[str, ...] = (),
|
||||
assessor: SufficiencyAssessor | None = None,
|
||||
budget: TurnBudget | None = None,
|
||||
metrics: Metrics | None = None,
|
||||
trace: LoopTrace | None = None,
|
||||
) -> LoopOutcome:
|
||||
"""One conversational turn through the bounded loop.
|
||||
|
||||
`clarify_signals` comes from the existing resolvers — ambiguous drug,
|
||||
unresolved section, multi-attribute. They short-circuit before any spend,
|
||||
because a question worth asking is cheaper and safer than a guess.
|
||||
"""
|
||||
budget = budget or TurnBudget()
|
||||
assessor = assessor or DeterministicAssessor()
|
||||
metrics = metrics or NullMetrics()
|
||||
trace = trace or LoopTrace()
|
||||
|
||||
trace.enter("understand")
|
||||
if clarify_signals:
|
||||
reason = clarify_signals[0]
|
||||
metrics.increment(metric_names.CLARIFY_ASKED, reason=reason)
|
||||
trace.enter("clarify")
|
||||
return LoopOutcome(
|
||||
answer=None,
|
||||
clarification=clarify_for(reason),
|
||||
evidence_texts=(),
|
||||
retrieval_rounds_used=0,
|
||||
repairs_used=0,
|
||||
stopped_because="clarify_signal",
|
||||
)
|
||||
|
||||
evidence: tuple[str, ...] = ()
|
||||
rounds_used = 0
|
||||
stopped = "sufficient"
|
||||
|
||||
while True:
|
||||
try:
|
||||
budget.spend_retrieval()
|
||||
except BudgetExhausted:
|
||||
stopped = "retrieval_budget"
|
||||
break
|
||||
trace.enter("retrieve")
|
||||
evidence = retrieve(resolved)
|
||||
rounds_used += 1
|
||||
|
||||
trace.enter("assess")
|
||||
verdict = assessor.assess(resolved, evidence)
|
||||
if verdict.sufficient:
|
||||
break
|
||||
if budget.retrieval_rounds <= 0 or budget.out_of_time():
|
||||
stopped = "retrieval_budget"
|
||||
break
|
||||
# A round is spent only on a named gap with a genuinely new query.
|
||||
if not verdict.missing or not verdict.refined_query:
|
||||
stopped = "no_actionable_gap"
|
||||
break
|
||||
if verdict.refined_query == resolved.text:
|
||||
stopped = "query_unchanged"
|
||||
break
|
||||
trace.enter("refine")
|
||||
metrics.increment(metric_names.LOOP_REFINED, missing=verdict.missing)
|
||||
resolved = replace(resolved, text=verdict.refined_query)
|
||||
|
||||
metrics.increment(metric_names.LOOP_ROUNDS, rounds=str(rounds_used))
|
||||
|
||||
if not evidence:
|
||||
trace.enter("clarify")
|
||||
metrics.increment(
|
||||
metric_names.CLARIFY_ASKED, reason=ClarifyReason.STILL_INSUFFICIENT
|
||||
)
|
||||
return LoopOutcome(
|
||||
answer=None,
|
||||
clarification=clarify_for(ClarifyReason.STILL_INSUFFICIENT),
|
||||
evidence_texts=(),
|
||||
retrieval_rounds_used=rounds_used,
|
||||
repairs_used=0,
|
||||
stopped_because="no_evidence",
|
||||
)
|
||||
|
||||
repairs_used = 0
|
||||
answer: str | None = None
|
||||
while True:
|
||||
trace.enter("generate")
|
||||
try:
|
||||
budget.spend_llm()
|
||||
except BudgetExhausted:
|
||||
stopped = "llm_budget"
|
||||
break
|
||||
answer = generate(resolved, evidence, state)
|
||||
if answer is not None:
|
||||
break
|
||||
# `generate` returning None means verification already refused it.
|
||||
try:
|
||||
budget.spend_repair()
|
||||
except BudgetExhausted:
|
||||
stopped = "repair_budget"
|
||||
break
|
||||
repairs_used += 1
|
||||
trace.enter("repair")
|
||||
metrics.increment(metric_names.LOOP_REPAIRED)
|
||||
|
||||
return LoopOutcome(
|
||||
answer=answer,
|
||||
clarification=None,
|
||||
evidence_texts=evidence,
|
||||
retrieval_rounds_used=rounds_used,
|
||||
repairs_used=repairs_used,
|
||||
stopped_because=stopped,
|
||||
generated=answer is not None,
|
||||
)
|
||||
@@ -1,5 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import functools
|
||||
import re
|
||||
from dataclasses import dataclass, replace
|
||||
from difflib import SequenceMatcher
|
||||
@@ -55,6 +56,20 @@ class CatalogDrugResolver:
|
||||
self._fuzzy_threshold = fuzzy_threshold
|
||||
self._ambiguity_margin = ambiguity_margin
|
||||
|
||||
# Measured live 2026-08-07: a single `resolve()` call over the real
|
||||
# ~10,164-alias catalog costs ~0.65-0.7s, `suggest()` ~0.94-0.97s — both
|
||||
# O(aliases) regex/SequenceMatcher work, pure functions of their
|
||||
# arguments (only `self._aliases` et al, fixed at construction, feed
|
||||
# them). `understanding.py`'s `_candidate_ids` calls both PER HISTORY
|
||||
# LINE on every single turn — so the SAME already-seen history lines
|
||||
# were being re-resolved from scratch every turn a conversation grew,
|
||||
# ~1.6-1.7s of pure CPU per repeated line. A real user's ordinary
|
||||
# multi-turn conversation was enough to exceed the 20s F-08 budget
|
||||
# before the first Bedrock call ever ran, surfacing as a false
|
||||
# "Dịch vụ đang gặp sự cố" — not a provider outage at all. Caching by
|
||||
# exact input turns all but the newest turn's own text into a dict
|
||||
# lookup on every subsequent call.
|
||||
@functools.lru_cache(maxsize=4096)
|
||||
def resolve(self, query: str) -> DrugResolution:
|
||||
normalized_query = normalize_name(query)
|
||||
query_tokens = normalized_query.split()
|
||||
@@ -140,6 +155,9 @@ class CatalogDrugResolver:
|
||||
break
|
||||
return ordered
|
||||
|
||||
# See the comment on `resolve` above — same cost, same fix, same
|
||||
# single-caller read-only usage (safe to hand back a cached list).
|
||||
@functools.lru_cache(maxsize=4096)
|
||||
def suggest(
|
||||
self, query: str, k: int = 3, min_score: float = 0.5
|
||||
) -> list[tuple[str, float]]:
|
||||
|
||||
@@ -32,6 +32,9 @@ class EvidencePolicy:
|
||||
# A free-form question about a resolved drug otherwise hands the LLM the
|
||||
# entire monograph; rerank trims it to the sections that actually answer.
|
||||
rerank_top_k: int = 6
|
||||
# symptom_to_drug: a common symptom can match far more drugs than is
|
||||
# useful to show in one answer.
|
||||
indication_candidate_limit: int = 8
|
||||
|
||||
|
||||
class RetrievalService:
|
||||
@@ -157,6 +160,47 @@ class RetrievalService:
|
||||
return RetrievalResult(EvidenceDecision.ABSTAIN, "insufficient_retrieval_score")
|
||||
return self._decide(self._hydrate(self._rerank(query, hits)))
|
||||
|
||||
def retrieve_by_indication(self, indication_text: str) -> RetrievalResult:
|
||||
"""Reverse lookup: symptom/indication -> candidate drugs.
|
||||
|
||||
Keyword match first (deterministic, precise — nothing here can
|
||||
fabricate a drug that doesn't genuinely mention the indication).
|
||||
Dense-vector search over `chi_dinh` only is the fallback, tried
|
||||
only when the keyword pass finds nothing, to catch a paraphrase the
|
||||
book's own wording doesn't share. This is the one place in the live
|
||||
path dense search is actually used — see ADR 0008.
|
||||
"""
|
||||
if not indication_text.strip():
|
||||
return RetrievalResult(EvidenceDecision.ABSTAIN, "missing_indication")
|
||||
|
||||
find_by_indication = getattr(self._retriever, "find_by_indication", None)
|
||||
hits = (
|
||||
find_by_indication(indication_text, self._policy.indication_candidate_limit)
|
||||
if find_by_indication is not None
|
||||
else []
|
||||
)
|
||||
if not hits:
|
||||
search_indication = getattr(self._retriever, "search_indication", None)
|
||||
if search_indication is not None:
|
||||
try:
|
||||
hits = search_indication(
|
||||
indication_text, self._policy.indication_candidate_limit
|
||||
)
|
||||
except QueryEmbeddingUnavailable:
|
||||
hits = []
|
||||
# Dense search always returns its nearest neighbours, even for
|
||||
# an indication the corpus has nothing on — verified live: a
|
||||
# made-up phrase still got 8 unrelated "matches". A weak top
|
||||
# score means those neighbours aren't really about the
|
||||
# question, so don't spend a generation call finding that out
|
||||
# the slow way; abstain here, the same bar `retrieve()`'s own
|
||||
# dense fallback already applies.
|
||||
if hits and hits[0].score < self._policy.minimum_score:
|
||||
hits = []
|
||||
if not hits:
|
||||
return RetrievalResult(EvidenceDecision.ABSTAIN, "no_indication_match")
|
||||
return self._decide(self._hydrate(hits, limit=None))
|
||||
|
||||
@staticmethod
|
||||
def _is_question(query: str) -> bool:
|
||||
"""A bare drug name (one or two tokens) wants the whole monograph; more
|
||||
@@ -206,8 +250,29 @@ class RetrievalService:
|
||||
if match is None:
|
||||
return None
|
||||
hits = find_by_section(drug_id, match.section_key)
|
||||
if match.section_key == "than_trong":
|
||||
# A "thận trọng" question about a specific condition sometimes has
|
||||
# its real answer filed under "chống chỉ định" instead — found live
|
||||
# 2026-08-10: Aspirin's own "thận trọng" text never says "loét dạ
|
||||
# dày", the fact only exists in its "chống chỉ định" text ("loét
|
||||
# dạ dày hoặc tá tràng đang hoạt động"). The two are the closest
|
||||
# pair of "is this safe for my patient" categories the book has,
|
||||
# and chống chỉ định text is short — pooling it costs nothing on
|
||||
# a drug where than_trong already answers, and prevents a false
|
||||
# "not in this source" clarify/abstain on one where it doesn't.
|
||||
hits = hits + find_by_section(drug_id, "chong_chi_dinh")
|
||||
return hits or None
|
||||
|
||||
def decide(self, evidence: tuple[Evidence, ...]) -> RetrievalResult:
|
||||
"""Public entry point for a caller that assembles its own evidence
|
||||
pool across several `retrieve_framed` calls — e.g. `RagAgent`'s
|
||||
2-drug interaction path — and needs the same quarantine/provenance
|
||||
policy applied to the combined pool that a single call already gets.
|
||||
Bypassing this (hand-rolling `RetrievalResult(ANSWERABLE, ...)`) is
|
||||
exactly how the interaction path silently dropped a quarantined
|
||||
drug's evidence instead of surfacing VERIFY_PDF for it."""
|
||||
return self._decide(evidence)
|
||||
|
||||
def _decide(
|
||||
self, evidence: tuple[Evidence, ...], is_drug_overview: bool = False
|
||||
) -> RetrievalResult:
|
||||
|
||||
@@ -35,9 +35,15 @@ stub runs the whole path offline in tests.
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from dataclasses import dataclass, field
|
||||
import logging
|
||||
from dataclasses import dataclass, field, replace
|
||||
from typing import Protocol, Sequence
|
||||
|
||||
from .budget import RequestBudget
|
||||
from .ports import AnswerGenerationUnavailable
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# The 19 monograph section keys, kept here as the closed vocabulary the model may
|
||||
# use for `attribute`. Adding a new section is one entry, not a code change.
|
||||
SECTION_KEYS = (
|
||||
@@ -126,8 +132,22 @@ class QueryFrame:
|
||||
weight_kg: float | None = None
|
||||
age_text: str | None = None
|
||||
indication: str | None = None # symptom/disease, for symptom_to_drug
|
||||
route: str | None = None # e.g. "uong", "tiem_tinh_mach", "dat_truc_trang"
|
||||
needs_clarify: bool = False
|
||||
clarify_reason: str | None = None
|
||||
# Short suggested replies for `clarify_reason` (e.g. ("Người lớn", "Trẻ
|
||||
# em")) — only when the model judged the missing detail has a few
|
||||
# natural discrete answers; often empty (e.g. a question needing a
|
||||
# specific weight has no clean short options).
|
||||
quick_replies: tuple[str, ...] = ()
|
||||
# Set only when `needs_clarify` fired because of a real technical
|
||||
# failure (provider outage, malformed model output) rather than the
|
||||
# model genuinely judging the turn under-specified. Found live
|
||||
# 2026-08-07: both cases produced the exact same `reason="needs_more_info"`
|
||||
# downstream, making a real, diagnosable outage indistinguishable from an
|
||||
# ordinary clarifying question in the API response and trace — this lets
|
||||
# `RagAgent._route` surface the real cause instead.
|
||||
system_error: str | None = None
|
||||
raw: dict = field(default_factory=dict, compare=False)
|
||||
|
||||
|
||||
@@ -147,8 +167,24 @@ FRAME_SCHEMA = {
|
||||
),
|
||||
"age_text": "the age exactly as stated (e.g. '3 tuổi', '5 tháng'), else null",
|
||||
"indication": "the symptom or disease if turn_type is symptom_to_drug, else null",
|
||||
"route": (
|
||||
"route of administration if stated or implied, normalized to one of: "
|
||||
"uong | tiem_tinh_mach | tiem_bap | tiem_duoi_da | dat_truc_trang | "
|
||||
"boi_ngoai_da | nho_mat | nho_mui | khac, else null. A bare reply like "
|
||||
"'uống' or 'tiêm' to your own prior clarify question about route IS "
|
||||
"this field — read it here, do not leave it null and re-ask."
|
||||
),
|
||||
"needs_clarify": "true only if the turn cannot be acted on without more info",
|
||||
"clarify_reason": "short Vietnamese question to ask, or null",
|
||||
"quick_replies": (
|
||||
"2-4 short suggested replies (each under ~20 chars) to your OWN "
|
||||
"clarify_reason, for the user to tap instead of typing — ONLY when "
|
||||
"clarify_reason genuinely has a few natural discrete answers (e.g. "
|
||||
"['Người lớn', 'Trẻ em'] or ['Uống', 'Tiêm']). Empty list [] if the "
|
||||
"missing detail needs a specific free-form value (e.g. an exact "
|
||||
"weight) with no clean short options — never invent numeric-ish "
|
||||
"options."
|
||||
),
|
||||
}
|
||||
|
||||
_SYSTEM = """Bạn là bộ HIỂU CÂU HỎI cho một chatbot tra cứu Dược thư Quốc gia Việt Nam.
|
||||
@@ -169,7 +205,39 @@ Quy tắc bắt buộc:
|
||||
không kèm đơn vị khác) NGHĨA LÀ 30 kg -> điền weight_kg=30, không bỏ trống.
|
||||
- Lượt nối tiếp ("còn liều thì sao", "nó dùng cho trẻ em?") -> dùng LỊCH SỬ để biết
|
||||
thuốc đang nói tới và điền vào "drugs".
|
||||
- Chào hỏi/vu vơ -> "smalltalk". Ngoài phạm vi chuyên luận thuốc -> "out_of_scope"."""
|
||||
- QUAN TRỌNG — lượt hiện tại trả lời câu hỏi bạn VỪA hỏi: nếu dòng "Trợ lý:" cuối
|
||||
cùng trong LỊCH SỬ là một câu hỏi làm rõ (vd "Người lớn hay trẻ em?", "Uống hay
|
||||
tiêm?", "Cân nặng bao nhiêu kg?"), và CÂU HỎI HIỆN TẠI là một câu trả lời ngắn
|
||||
hợp lý cho đúng câu đó (vd "Uống", "Người lớn", "30kg") — hãy đọc nó là câu trả
|
||||
lời, điền vào field tương ứng (route/population/weight_kg/age_text), giữ lại các
|
||||
field đã biết từ các lượt trước đó trong LỊCH SỬ (đừng bỏ trống lại), và CHỈ đặt
|
||||
needs_clarify=true với PHẦN THÔNG TIN CÒN THIẾU KHÁC (nếu có) — TUYỆT ĐỐI KHÔNG
|
||||
lặp lại nguyên văn clarify_reason đã được trả lời. Nếu sau khi điền, đã đủ dữ
|
||||
kiện (đối tượng + đường dùng, và tuổi/cân nặng nếu là trẻ em) thì needs_clarify=false.
|
||||
Nếu có khối "THÔNG TIN ĐÃ XÁC ĐỊNH TỪ CÁC LƯỢT TRƯỚC" bên dưới, các mục đó ĐÃ
|
||||
ĐƯỢC XÁC NHẬN — không hỏi lại, không đặt needs_clarify=true vì thiếu đúng mục
|
||||
đã liệt kê ở đó.
|
||||
- NGƯỢC LẠI — nếu CÂU HỎI HIỆN TẠI là một câu hỏi y khoa MỚI, KHÔNG phải một câu
|
||||
trả lời ngắn cho câu hỏi làm rõ gần nhất (không khớp loại thông tin vừa hỏi) và
|
||||
KHÔNG nhắc lại thuốc/triệu chứng nào đã có trong LỊCH SỬ hay khối "THÔNG TIN ĐÃ
|
||||
XÁC ĐỊNH": đây là LƯỢT MỚI HOÀN TOÀN — TUYỆT ĐỐI KHÔNG mang "drugs"/
|
||||
"population"/"weight_kg"/"age_text"/"route"/"indication" của lượt trước sang lượt
|
||||
này, chỉ điền những gì thực sự có trong CÂU HỎI HIỆN TẠI. Ví dụ: lượt trước đang
|
||||
hỏi về Omeprazol nhưng câu hiện tại là "tôi bị đau đầu nên uống thuốc gì" (không
|
||||
nhắc Omeprazol) -> chủ đề mới, "drugs" phải để trống trừ khi có thuốc thực sự
|
||||
được nhắc trong câu này.
|
||||
- Nếu CÂU HỎI HIỆN TẠI là một lời PHỦ ĐỊNH/SỬA LẠI câu trả lời vừa rồi (vd "tôi
|
||||
có hỏi X đâu", "tôi không hỏi vậy", "đâu phải thế", "ý tôi không phải vậy",
|
||||
"sai rồi") — đây là dấu hiệu bạn vừa hiểu SAI ý người dùng ở lượt trước.
|
||||
TUYỆT ĐỐI KHÔNG lặp lại đúng route/population/thuộc tính vừa trả lời (đã bị
|
||||
từ chối): đặt needs_clarify=true và hỏi lại thật ngắn gọn, cụ thể người dùng
|
||||
thực sự muốn hỏi điều gì (vd "Anh/chị muốn hỏi đường dùng nào ạ?"), không tự
|
||||
suy đoán lại giá trị cũ.
|
||||
- Chào hỏi/vu vơ -> "smalltalk". Ngoài phạm vi chuyên luận thuốc -> "out_of_scope".
|
||||
- Khi needs_clarify=true, kèm "quick_replies": 2-4 phương án NGẮN cho câu hỏi lại
|
||||
đó, CHỈ khi nó thực sự có vài lựa chọn rời rạc tự nhiên (vd đối tượng: "Người
|
||||
lớn"/"Trẻ em"). Để mảng rỗng nếu cần một giá trị cụ thể không có lựa chọn ngắn
|
||||
(vd hỏi cân nặng chính xác) — không bịa phương án dạng số."""
|
||||
|
||||
|
||||
class JsonLlm(Protocol):
|
||||
@@ -181,7 +249,11 @@ class JsonLlm(Protocol):
|
||||
|
||||
class QueryUnderstander(Protocol):
|
||||
def understand(
|
||||
self, turn: str, history: Sequence[str] = ()
|
||||
self,
|
||||
turn: str,
|
||||
history: Sequence[str] = (),
|
||||
budget: RequestBudget | None = None,
|
||||
prior_frame: QueryFrame | None = None,
|
||||
) -> QueryFrame: ...
|
||||
|
||||
|
||||
@@ -240,7 +312,13 @@ class LlmQueryUnderstander:
|
||||
ids.add(drug_id)
|
||||
return ids
|
||||
|
||||
def understand(self, turn: str, history: Sequence[str] = ()) -> QueryFrame:
|
||||
def understand(
|
||||
self,
|
||||
turn: str,
|
||||
history: Sequence[str] = (),
|
||||
budget: RequestBudget | None = None,
|
||||
prior_frame: QueryFrame | None = None,
|
||||
) -> QueryFrame:
|
||||
shown = {
|
||||
drug_id: self._catalog[drug_id]
|
||||
for drug_id in self._candidate_ids(turn, history)
|
||||
@@ -255,6 +333,7 @@ class LlmQueryUnderstander:
|
||||
"LỊCH SỬ HỘI THOẠI (cũ -> mới):\n" + "\n".join(history)
|
||||
if history else "LỊCH SỬ HỘI THOẠI: (chưa có)"
|
||||
)
|
||||
known_block = _known_facts_block(prior_frame)
|
||||
user = (
|
||||
f"DANH SÁCH THUỐC ỨNG VIÊN cho lượt này (drug_id\\ttên) — CHỈ được chọn "
|
||||
f"drug_id từ đây, đây KHÔNG phải toàn bộ Dược thư, chỉ là các thuốc khớp "
|
||||
@@ -263,11 +342,39 @@ class LlmQueryUnderstander:
|
||||
"CÁC SECTION KEY hợp lệ cho 'attribute' (key: ý nghĩa):\n"
|
||||
+ "\n".join(f"{key}: {SECTION_KEY_HINTS[key]}" for key in SECTION_KEYS)
|
||||
+ "\n\n"
|
||||
f"{known_block}"
|
||||
f"{history_block}\n\n"
|
||||
f"CÂU HỎI HIỆN TẠI: {turn}"
|
||||
)
|
||||
raw_text = self._llm.generate(_SYSTEM, user, FRAME_SCHEMA)
|
||||
return self._parse(raw_text, shown)
|
||||
# Found live 2026-08-07 (F-10 adversarial pass): unlike every other
|
||||
# LLM call site in this product (`answer.py`'s sufficiency/generate/
|
||||
# entailment all catch this), this one call had no error handling at
|
||||
# all — a provider outage here propagated straight through
|
||||
# `RagAgent.handle()` and `routers/rag.py` (which only wraps the
|
||||
# trace-save call, not `agent.handle()`) into an unhandled 500,
|
||||
# rather than the graceful abstain every other failure mode gets.
|
||||
try:
|
||||
if budget is not None:
|
||||
budget.require()
|
||||
raw_text = self._llm.generate(_SYSTEM, user, FRAME_SCHEMA)
|
||||
except AnswerGenerationUnavailable as exc:
|
||||
# Found live 2026-08-07: this except block silently swallowed
|
||||
# the real exception entirely — no log line anywhere — so a
|
||||
# genuine provider outage (throttling, timeout, IAM, whatever)
|
||||
# left zero trace to diagnose from. Now logged with the actual
|
||||
# exception, and tagged with a `system_error` code distinct from
|
||||
# an ordinary clarify (see `QueryFrame.system_error`).
|
||||
logger.warning(
|
||||
"understanding call failed (%s): %s", type(exc).__name__, exc
|
||||
)
|
||||
return QueryFrame(
|
||||
turn_type="out_of_scope",
|
||||
needs_clarify=True,
|
||||
clarify_reason="Dịch vụ đang gặp sự cố tạm thời, vui lòng thử lại "
|
||||
"sau ít phút.",
|
||||
system_error="understanding_provider_unavailable",
|
||||
)
|
||||
return _merge_with_prior_frame(self._parse(raw_text, shown), prior_frame)
|
||||
|
||||
@staticmethod
|
||||
def _resolve_id(value: str, shown: dict[str, str]) -> str | None:
|
||||
@@ -291,11 +398,15 @@ class LlmQueryUnderstander:
|
||||
try:
|
||||
data = json.loads(raw_text)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
# Fail closed to a clarify rather than to a wrong reading.
|
||||
# Fail closed to a clarify rather than to a wrong reading. Not a
|
||||
# provider outage (the call succeeded) — the model's own output
|
||||
# didn't parse, a distinct, separately diagnosable cause.
|
||||
logger.warning("understanding call returned unparseable JSON: %r", raw_text)
|
||||
return QueryFrame(
|
||||
turn_type="out_of_scope",
|
||||
needs_clarify=True,
|
||||
clarify_reason="Xin lỗi, tôi chưa hiểu rõ câu hỏi. Anh/chị hỏi lại giúp nhé?",
|
||||
system_error="understanding_malformed_output",
|
||||
)
|
||||
resolved = [
|
||||
(d, self._resolve_id(d, shown)) for d in _as_list(data.get("drugs"))
|
||||
@@ -322,12 +433,86 @@ class LlmQueryUnderstander:
|
||||
weight_kg=_clean_float(data.get("weight_kg")),
|
||||
age_text=_clean_str(data.get("age_text")),
|
||||
indication=_clean_str(data.get("indication")),
|
||||
route=_clean_str(data.get("route")),
|
||||
needs_clarify=bool(data.get("needs_clarify")),
|
||||
clarify_reason=_clean_str(data.get("clarify_reason")),
|
||||
quick_replies=tuple(_as_list(data.get("quick_replies"))),
|
||||
raw=data if isinstance(data, dict) else {},
|
||||
)
|
||||
|
||||
|
||||
_KNOWN_FACT_LABELS: tuple[tuple[str, str], ...] = (
|
||||
("population", "Đối tượng"),
|
||||
("age_text", "Tuổi"),
|
||||
("route", "Đường dùng"),
|
||||
("indication", "Chỉ định/triệu chứng"),
|
||||
("attribute", "Thuộc tính đang tra"),
|
||||
)
|
||||
|
||||
|
||||
def _known_facts_block(prior_frame: QueryFrame | None) -> str:
|
||||
"""The structured "already established" summary shown to the model on a
|
||||
clarify-continuation turn.
|
||||
|
||||
Found live 2026-08-07 (50-question hand-typed browser audit): relying on
|
||||
the model to re-derive the WHOLE frame from raw text history every turn
|
||||
is fragile — reproduced 3 times independently (Insulin storage, weight-
|
||||
based Azithromycin dosing, a headache question mislabeled OMEPRAZOL) as
|
||||
either a non-terminating re-ask of an already-answered clarify question,
|
||||
or a stale drug bleeding into an unrelated new topic. Stating the known
|
||||
fields explicitly, as data rather than asking the model to infer them
|
||||
from a growing text transcript, removes most of the guesswork; `_merge_
|
||||
with_prior_frame` below is the code-level backstop for whatever the
|
||||
model still drops.
|
||||
"""
|
||||
if prior_frame is None or not prior_frame.needs_clarify:
|
||||
return ""
|
||||
parts = []
|
||||
if prior_frame.drugs:
|
||||
parts.append(f"Thuốc: {', '.join(prior_frame.drugs)}")
|
||||
if prior_frame.weight_kg is not None:
|
||||
parts.append(f"Cân nặng: {prior_frame.weight_kg:g} kg")
|
||||
for field_name, label in _KNOWN_FACT_LABELS:
|
||||
value = getattr(prior_frame, field_name)
|
||||
if value:
|
||||
parts.append(f"{label}: {value}")
|
||||
if not parts:
|
||||
return ""
|
||||
return (
|
||||
"THÔNG TIN ĐÃ XÁC ĐỊNH TỪ CÁC LƯỢT TRƯỚC (dữ liệu CÓ THẬT, đã xác nhận "
|
||||
"— KHÔNG hỏi lại các mục này; nếu câu hỏi hiện tại là một chủ đề mới "
|
||||
"không liên quan, hãy bỏ qua khối này thay vì gán nhầm vào lượt mới):\n"
|
||||
+ "\n".join(parts) + "\n\n"
|
||||
)
|
||||
|
||||
|
||||
def _merge_with_prior_frame(frame: QueryFrame, prior_frame: QueryFrame | None) -> QueryFrame:
|
||||
"""Code-level backstop for the model dropping an already-known field.
|
||||
|
||||
Only fires when the prior turn was itself a clarify (there is something
|
||||
to continue) and this turn's own `drugs` extraction agrees with it (empty,
|
||||
meaning a short reply like "20kg"/"Uống" that names no drug of its own, or
|
||||
an exact match) — a turn that resolves a DIFFERENT drug is a genuine topic
|
||||
change and must not inherit stale population/weight/route/indication from
|
||||
the old one (the headache/OMEPRAZOL bleed this guards against runs the
|
||||
other way: don't let old fields survive into an unrelated new drug either).
|
||||
"""
|
||||
if prior_frame is None or not prior_frame.needs_clarify:
|
||||
return frame
|
||||
if frame.drugs and frame.drugs != prior_frame.drugs:
|
||||
return frame
|
||||
return replace(
|
||||
frame,
|
||||
drugs=frame.drugs or prior_frame.drugs,
|
||||
population=frame.population or prior_frame.population,
|
||||
age_text=frame.age_text or prior_frame.age_text,
|
||||
weight_kg=frame.weight_kg if frame.weight_kg is not None else prior_frame.weight_kg,
|
||||
route=frame.route or prior_frame.route,
|
||||
indication=frame.indication or prior_frame.indication,
|
||||
attribute=frame.attribute or prior_frame.attribute,
|
||||
)
|
||||
|
||||
|
||||
def _as_list(value) -> list[str]:
|
||||
if isinstance(value, str):
|
||||
return [value] if value.strip() else []
|
||||
|
||||
@@ -35,6 +35,9 @@ class CitationResponse(BaseModel):
|
||||
bbox: tuple[float, float, float, float] | None = None
|
||||
source_crop: str | None = None
|
||||
attachment: str | None = None
|
||||
# The exact retrieved chunk text this citation stands for — lets the UI
|
||||
# show precisely what was retrieved, not a client-side guess at it.
|
||||
evidence_text: str = ""
|
||||
|
||||
|
||||
class RagQueryResponse(BaseModel):
|
||||
@@ -44,6 +47,16 @@ class RagQueryResponse(BaseModel):
|
||||
answer: str | None
|
||||
resolved_drug_id: str | None
|
||||
citations: list[CitationResponse]
|
||||
# Whether `answer` is an LLM paraphrase (verified by grounding +
|
||||
# entailment) or a verbatim extractive quote of the retrieved source —
|
||||
# the UI shows these differently so a clinician knows which they're
|
||||
# reading.
|
||||
generated: bool = False
|
||||
# Short suggested replies for a `decision == "clarify"` turn (e.g.
|
||||
# ["Người lớn", "Trẻ em"]) — only populated by the sufficiency-check
|
||||
# clarify path today; other clarify sources (no_drug, dosing_calc's
|
||||
# needs_clarify) leave this empty rather than fabricate options.
|
||||
quick_replies: list[str] = []
|
||||
|
||||
|
||||
def _answer_service(request: Request) -> GroundedAnswerService:
|
||||
@@ -91,6 +104,7 @@ def _map_citations(items) -> list[CitationResponse]:
|
||||
bbox=item.bbox,
|
||||
source_crop=item.source_crop,
|
||||
attachment=item.attachment,
|
||||
evidence_text=item.evidence_text,
|
||||
)
|
||||
for item in items
|
||||
]
|
||||
@@ -128,6 +142,8 @@ def query_rag(
|
||||
answer = reply.clarification if reply.clarification is not None else reply.answer
|
||||
resolved_drug_id = ", ".join(reply.drugs) if reply.drugs else None
|
||||
citations = _map_citations(reply.citations)
|
||||
generated = reply.generated
|
||||
quick_replies = list(reply.quick_replies)
|
||||
else:
|
||||
# No generator configured (ANSWER_PROVIDER=disabled): there is no LLM
|
||||
# to understand a turn with, so this is retrieval-only, single-turn,
|
||||
@@ -138,12 +154,16 @@ def query_rag(
|
||||
answer = grounded.clarification
|
||||
resolved_drug_id = grounded.result.resolved_drug_id
|
||||
citations = []
|
||||
generated = False
|
||||
quick_replies = list(grounded.quick_replies)
|
||||
else:
|
||||
decision = grounded.result.decision.value
|
||||
reason = grounded.result.reason
|
||||
answer = grounded.answer
|
||||
resolved_drug_id = grounded.result.resolved_drug_id
|
||||
citations = _map_citations(grounded.citations)
|
||||
generated = grounded.generated
|
||||
quick_replies = []
|
||||
|
||||
# Trace persistence is fail-open (F-09): an already-computed, safe answer
|
||||
# must reach the caller even if Postgres is unreachable. `save()` opens a
|
||||
@@ -175,4 +195,6 @@ def query_rag(
|
||||
answer=answer,
|
||||
resolved_drug_id=resolved_drug_id,
|
||||
citations=citations,
|
||||
generated=generated,
|
||||
quick_replies=quick_replies,
|
||||
)
|
||||
|
||||
@@ -8,18 +8,20 @@ Qdrant/Bedrock fixtures).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from rag.agent import RagAgent
|
||||
from rag.agent import MAX_CONSECUTIVE_CLARIFY, RagAgent
|
||||
from rag.answer import GroundedAnswerService
|
||||
from rag.metrics import GENERATION_REJECTED, InMemoryMetrics
|
||||
from rag.models import Evidence, EvidenceDecision, RetrievalResult, SourceRef
|
||||
from rag.understanding import QueryFrame
|
||||
|
||||
SOURCE = SourceRef(physical_page=100, precision="region", printed_page=100)
|
||||
|
||||
|
||||
def _evidence(text: str) -> Evidence:
|
||||
def _evidence(text: str, requires_visual_check: bool = False) -> Evidence:
|
||||
return Evidence(
|
||||
evidence_id="e0", matched_doc_id="e0", kind="prose", text=text, score=1.0,
|
||||
source_refs=(SOURCE,), hydrated_from_parent=False, requires_visual_check=False,
|
||||
source_refs=(SOURCE,), hydrated_from_parent=False,
|
||||
requires_visual_check=requires_visual_check,
|
||||
)
|
||||
|
||||
|
||||
@@ -27,7 +29,7 @@ class _FixedUnderstander:
|
||||
def __init__(self, frame: QueryFrame) -> None:
|
||||
self._frame = frame
|
||||
|
||||
def understand(self, turn, history=()):
|
||||
def understand(self, turn, history=(), budget=None, prior_frame=None):
|
||||
return self._frame
|
||||
|
||||
|
||||
@@ -36,20 +38,50 @@ class _FixedRetrieval:
|
||||
drug_id regardless of section/query, so these tests assert routing, not
|
||||
retrieval (that's `test_retrieval_service.py`'s job)."""
|
||||
|
||||
def __init__(self, results: dict[str, RetrievalResult]) -> None:
|
||||
def __init__(
|
||||
self,
|
||||
results: dict[str, RetrievalResult],
|
||||
indication_results: dict[str, RetrievalResult] | None = None,
|
||||
) -> None:
|
||||
self._results = results
|
||||
self._indication_results = indication_results or {}
|
||||
self.calls: list[tuple[str, str | None, str]] = []
|
||||
|
||||
def retrieve_framed(self, drug_id, section_key, query, is_overview=False):
|
||||
self.calls.append((drug_id, section_key, query))
|
||||
return self._results.get(
|
||||
drug_id, RetrievalResult(EvidenceDecision.ABSTAIN, "not_configured")
|
||||
)
|
||||
|
||||
def retrieve_by_indication(self, indication_text):
|
||||
return self._indication_results.get(
|
||||
indication_text, RetrievalResult(EvidenceDecision.ABSTAIN, "no_indication_match")
|
||||
)
|
||||
|
||||
def _agent(frame: QueryFrame, results: dict[str, RetrievalResult] | None = None) -> RagAgent:
|
||||
def decide(self, evidence):
|
||||
# Mirrors `RetrievalService.decide`'s real policy (not a stub that
|
||||
# always says ANSWERABLE) so `_interaction`'s use of it is actually
|
||||
# under test, not just its own call site.
|
||||
if not evidence:
|
||||
return RetrievalResult(EvidenceDecision.ABSTAIN, "parent_hydration_failed")
|
||||
if any(item.requires_visual_check for item in evidence):
|
||||
return RetrievalResult(EvidenceDecision.VERIFY_PDF, "visual_verification_required", evidence)
|
||||
return RetrievalResult(EvidenceDecision.ANSWERABLE, "grounded_evidence_available", evidence)
|
||||
|
||||
|
||||
def _agent(
|
||||
frame: QueryFrame,
|
||||
results: dict[str, RetrievalResult] | None = None,
|
||||
indication_results: dict[str, RetrievalResult] | None = None,
|
||||
) -> RagAgent:
|
||||
# `routing=None`: `answer_from_result` (the only method this path calls)
|
||||
# never touches it — see `rag/answer.py`.
|
||||
answers = GroundedAnswerService(routing=None)
|
||||
return RagAgent(_FixedUnderstander(frame), _FixedRetrieval(results or {}), answers)
|
||||
return RagAgent(
|
||||
_FixedUnderstander(frame),
|
||||
_FixedRetrieval(results or {}, indication_results),
|
||||
answers,
|
||||
)
|
||||
|
||||
|
||||
def test_smalltalk_does_not_touch_retrieval():
|
||||
@@ -100,6 +132,25 @@ def test_needs_clarify_frame_is_surfaced_directly():
|
||||
reply = agent.handle("liều paracetamol cho trẻ em")
|
||||
assert reply.decision == "clarify"
|
||||
assert reply.clarification == "Bé mấy tuổi, cân nặng bao nhiêu kg?"
|
||||
# No quick_replies on the frame -> none surfaced (a specific weight has
|
||||
# no clean short options; must not be fabricated downstream).
|
||||
assert reply.quick_replies == ()
|
||||
|
||||
|
||||
def test_needs_clarify_frame_carries_quick_replies_through():
|
||||
"""This is the path real traffic actually hits (checked live): the
|
||||
understanding LLM call itself sets needs_clarify/clarify_reason before
|
||||
retrieval ever runs, short-circuiting `_route` — quick_replies must
|
||||
survive that same short-circuit, not just the sufficiency-check path
|
||||
inside `GroundedAnswerService`."""
|
||||
agent = _agent(QueryFrame(
|
||||
turn_type="drug_attribute", drugs=("paracetamol",),
|
||||
needs_clarify=True, clarify_reason="Người lớn hay trẻ em?",
|
||||
quick_replies=("Người lớn", "Trẻ em"),
|
||||
))
|
||||
reply = agent.handle("liều paracetamol")
|
||||
assert reply.decision == "clarify"
|
||||
assert reply.quick_replies == ("Người lớn", "Trẻ em")
|
||||
|
||||
|
||||
def test_single_drug_attribute_retrieves_and_answers():
|
||||
@@ -118,6 +169,58 @@ def test_single_drug_attribute_retrieves_and_answers():
|
||||
assert "500 mg" in reply.answer
|
||||
|
||||
|
||||
def test_context_resolved_across_turns_is_folded_into_the_query():
|
||||
"""The 2026-08-07 P0 (named in the 2026-08-06 audit): population/weight/
|
||||
age/route are extracted by understanding.py but were never passed into
|
||||
retrieval or generation — so a reply like "Uống" three turns into a dose
|
||||
conversation reached `GroundedAnswerService` as literally just "Uống",
|
||||
with no notion that population=adult was already established. Fixed via
|
||||
`_synthesize_query`; this asserts the synthesized text — not the bare
|
||||
turn — is what retrieval and generation actually see."""
|
||||
result = RetrievalResult(
|
||||
EvidenceDecision.ANSWERABLE, "grounded_evidence_available",
|
||||
(_evidence("Liều uống người lớn: 500 mg."),
|
||||
_evidence("Liều tiêm người lớn: 1 g.")),
|
||||
resolved_drug_id="paracetamol_acetaminophen",
|
||||
)
|
||||
retrieval = _FixedRetrieval({"paracetamol_acetaminophen": result})
|
||||
agent = RagAgent(
|
||||
_FixedUnderstander(QueryFrame(
|
||||
turn_type="dosing_calc", drugs=("paracetamol_acetaminophen",),
|
||||
attribute="lieu_luong_va_cach_dung", population="nguoi_lon",
|
||||
route="uong", needs_clarify=False,
|
||||
)),
|
||||
retrieval,
|
||||
GroundedAnswerService(routing=None),
|
||||
)
|
||||
agent.handle("Uống")
|
||||
assert len(retrieval.calls) == 1
|
||||
_, _, query = retrieval.calls[0]
|
||||
assert query.startswith("Uống")
|
||||
assert "người lớn" in query
|
||||
assert "uống" in query.lower()
|
||||
|
||||
|
||||
def test_context_synthesis_is_a_no_op_when_the_frame_has_no_resolved_fields():
|
||||
"""A fresh, fully-specified single-shot question already states its own
|
||||
context — synthesis must not alter it or introduce redundant noise."""
|
||||
result = RetrievalResult(
|
||||
EvidenceDecision.ANSWERABLE, "grounded_evidence_available",
|
||||
(_evidence("x"), _evidence("y")), resolved_drug_id="metformin",
|
||||
)
|
||||
retrieval = _FixedRetrieval({"metformin": result})
|
||||
agent = RagAgent(
|
||||
_FixedUnderstander(QueryFrame(
|
||||
turn_type="drug_attribute", drugs=("metformin",),
|
||||
attribute="chong_chi_dinh",
|
||||
)),
|
||||
retrieval,
|
||||
GroundedAnswerService(routing=None),
|
||||
)
|
||||
agent.handle("Chống chỉ định của metformin là gì?")
|
||||
assert retrieval.calls[0][2] == "Chống chỉ định của metformin là gì?"
|
||||
|
||||
|
||||
def test_interaction_combines_both_drugs_evidence():
|
||||
warfarin = RetrievalResult(
|
||||
EvidenceDecision.ANSWERABLE, "grounded_evidence_available",
|
||||
@@ -137,6 +240,34 @@ def test_interaction_combines_both_drugs_evidence():
|
||||
assert "chảy máu" in reply.answer
|
||||
|
||||
|
||||
def test_interaction_with_one_drug_quarantined_never_answers_confidently():
|
||||
"""The P0 the 2026-08-06 audit found: `_interaction` used to keep only
|
||||
`ANSWERABLE` parts, so a quarantined drug's evidence (and the "table
|
||||
exists, verify PDF" notice the quarantine contract requires) was
|
||||
silently dropped — a confident interaction answer could omit a real
|
||||
unverified contraindication table for one of the two drugs. Fixed via
|
||||
`RetrievalService.decide` applied to the combined pool, the same policy
|
||||
the single-drug path already uses."""
|
||||
warfarin = RetrievalResult(
|
||||
EvidenceDecision.ANSWERABLE, "grounded_evidence_available",
|
||||
(_evidence("Tương tác với aspirin làm tăng nguy cơ chảy máu."),),
|
||||
)
|
||||
aspirin = RetrievalResult(
|
||||
EvidenceDecision.VERIFY_PDF, "visual_verification_required",
|
||||
(_evidence("Bảng tương tác cần đối chiếu PDF.", requires_visual_check=True),),
|
||||
)
|
||||
agent = _agent(
|
||||
QueryFrame(turn_type="interaction", drugs=("warfarin", "aspirin")),
|
||||
{"warfarin": warfarin, "aspirin": aspirin},
|
||||
)
|
||||
reply = agent.handle("warfarin với aspirin có dùng chung được không")
|
||||
# Must NOT be a confident "answerable" that silently omits aspirin's
|
||||
# quarantined table — must ask for PDF verification instead.
|
||||
assert reply.decision == "verify_pdf"
|
||||
# Both drugs' evidence must still be present (as citations), not dropped.
|
||||
assert len(reply.citations) == 2
|
||||
|
||||
|
||||
def test_interaction_with_no_evidence_abstains_and_does_not_imply_safety():
|
||||
agent = _agent(
|
||||
QueryFrame(turn_type="interaction", drugs=("drug_a", "drug_b")), {},
|
||||
@@ -147,18 +278,65 @@ def test_interaction_with_no_evidence_abstains_and_does_not_imply_safety():
|
||||
assert "KHÔNG có nghĩa là an toàn" in reply.answer
|
||||
|
||||
|
||||
def test_symptom_to_drug_without_a_drug_name_asks_honestly_not_wired_yet():
|
||||
agent = _agent(QueryFrame(turn_type="symptom_to_drug", indication="sốt cao"))
|
||||
reply = agent.handle("sốt cao uống thuốc gì")
|
||||
def test_symptom_to_drug_with_no_indication_extracted_asks_for_one():
|
||||
agent = _agent(QueryFrame(turn_type="symptom_to_drug"))
|
||||
reply = agent.handle("có thuốc gì không")
|
||||
assert reply.decision == "clarify"
|
||||
assert reply.reason == "reverse_lookup_not_ready"
|
||||
assert reply.reason == "no_indication"
|
||||
|
||||
|
||||
def test_symptom_to_drug_with_no_match_abstains_not_silently_safe():
|
||||
agent = _agent(
|
||||
QueryFrame(turn_type="symptom_to_drug", indication="bệnh hiếm gặp x"),
|
||||
indication_results={
|
||||
"bệnh hiếm gặp x": RetrievalResult(EvidenceDecision.ABSTAIN, "no_indication_match"),
|
||||
},
|
||||
)
|
||||
reply = agent.handle("thuốc gì trị bệnh hiếm gặp x")
|
||||
assert reply.decision == "abstain"
|
||||
assert "bệnh hiếm gặp x" in reply.answer
|
||||
assert "KHÔNG" in reply.answer
|
||||
|
||||
|
||||
def test_symptom_to_drug_returns_the_matched_drugs_not_frame_drugs():
|
||||
"""`frame.drugs` is empty by construction for this turn_type (the router
|
||||
only reaches `_symptom_to_drug` with no named drug) — the reply's
|
||||
`drugs` must come from what retrieval actually found."""
|
||||
result = RetrievalResult(
|
||||
EvidenceDecision.ANSWERABLE, "grounded_evidence_available",
|
||||
(_evidence("Paracetamol chỉ định hạ sốt."),
|
||||
_evidence("Ibuprofen chỉ định hạ sốt, giảm đau.")),
|
||||
)
|
||||
# Overwrite matched_doc_id per evidence to simulate two different drugs
|
||||
# (the shared `_evidence` helper always uses "e0" — construct directly).
|
||||
ev_a = Evidence(
|
||||
evidence_id="paracetamol_acetaminophen__chi_dinh__0",
|
||||
matched_doc_id="paracetamol_acetaminophen__chi_dinh__0",
|
||||
kind="prose", text="Paracetamol chỉ định hạ sốt.", score=1.0,
|
||||
source_refs=(SOURCE,), hydrated_from_parent=False, requires_visual_check=False,
|
||||
)
|
||||
ev_b = Evidence(
|
||||
evidence_id="ibuprofen__chi_dinh__0", matched_doc_id="ibuprofen__chi_dinh__0",
|
||||
kind="prose", text="Ibuprofen chỉ định hạ sốt, giảm đau.", score=1.0,
|
||||
source_refs=(SOURCE,), hydrated_from_parent=False, requires_visual_check=False,
|
||||
)
|
||||
result = RetrievalResult(
|
||||
EvidenceDecision.ANSWERABLE, "grounded_evidence_available", (ev_a, ev_b),
|
||||
)
|
||||
agent = _agent(
|
||||
QueryFrame(turn_type="symptom_to_drug", indication="sốt"),
|
||||
indication_results={"sốt": result},
|
||||
)
|
||||
reply = agent.handle("sốt thì uống thuốc gì")
|
||||
assert reply.decision == "answerable"
|
||||
assert reply.drugs == ("paracetamol_acetaminophen", "ibuprofen")
|
||||
|
||||
|
||||
def test_history_is_passed_to_the_understander_on_the_next_turn():
|
||||
received_history: list[tuple[str, ...]] = []
|
||||
|
||||
class _RecordingUnderstander:
|
||||
def understand(self, turn, history=()):
|
||||
def understand(self, turn, history=(), budget=None, prior_frame=None):
|
||||
received_history.append(tuple(history))
|
||||
return QueryFrame(turn_type="smalltalk")
|
||||
|
||||
@@ -176,7 +354,7 @@ def test_history_is_isolated_per_conversation_id():
|
||||
received_history: list[tuple[str, ...]] = []
|
||||
|
||||
class _RecordingUnderstander:
|
||||
def understand(self, turn, history=()):
|
||||
def understand(self, turn, history=(), budget=None, prior_frame=None):
|
||||
received_history.append(tuple(history))
|
||||
return QueryFrame(turn_type="smalltalk")
|
||||
|
||||
@@ -190,6 +368,26 @@ def test_history_is_isolated_per_conversation_id():
|
||||
assert received_history[1] == ()
|
||||
|
||||
|
||||
# --- F-10: `conversation_id` presence/absence must not change the safety
|
||||
# decision on a fresh (first) turn — only whether the turn is remembered
|
||||
# afterward -------------------------------------------------------------
|
||||
|
||||
|
||||
def test_conversation_id_presence_or_absence_reaches_the_same_decision():
|
||||
"""A single-turn call (no `conversation_id`) and the first turn of a
|
||||
fresh multi-turn conversation must resolve identically — both see empty
|
||||
history, so nothing about `conversation_id` itself may become a second,
|
||||
undocumented safety signal."""
|
||||
agent = _agent(QueryFrame(
|
||||
turn_type="drug_attribute", unknown_drugs=("aspirinol",),
|
||||
))
|
||||
without_id = agent.handle("liều aspirinol")
|
||||
with_id = agent.handle("liều aspirinol", conversation_id="fresh-conv")
|
||||
|
||||
assert without_id.decision == with_id.decision == "abstain"
|
||||
assert without_id.reason == with_id.reason == "drug_not_in_formulary"
|
||||
|
||||
|
||||
def test_autocomplete_delegates_to_the_configured_source():
|
||||
class _Source:
|
||||
def complete(self, prefix, k):
|
||||
@@ -210,3 +408,291 @@ def test_autocomplete_with_no_source_configured_returns_empty():
|
||||
_FixedRetrieval({}), answers,
|
||||
)
|
||||
assert agent.complete("met") == []
|
||||
|
||||
|
||||
# --- F-08: a per-turn budget actually stops real provider calls once spent,
|
||||
# it isn't just bookkeeping ---------------------------------------------
|
||||
|
||||
|
||||
def test_a_budget_exhausted_during_understand_blocks_every_later_call():
|
||||
"""`max_llm_calls_per_turn=1` means the (simulated) `understand()` call
|
||||
spends the entire turn's budget — sufficiency and generate must never
|
||||
reach the generator at all, not just receive an error from it. Proves
|
||||
the budget is threaded end to end through `RagAgent`, not only present
|
||||
at the one call site each unit test exercises in isolation."""
|
||||
class _BudgetSpendingUnderstander:
|
||||
"""Stands in for the real `LlmQueryUnderstander`, which calls
|
||||
`budget.require()` once before its own LLM call — simulated here so
|
||||
this test doesn't need a live-shaped LLM fake for the understand
|
||||
step, only for the reply's calls to fail budget."""
|
||||
|
||||
def understand(self, turn, history=(), budget=None, prior_frame=None):
|
||||
if budget is not None:
|
||||
budget.require()
|
||||
return QueryFrame(
|
||||
turn_type="drug_attribute", drugs=("metformin",),
|
||||
attribute="chi_dinh",
|
||||
)
|
||||
|
||||
result = RetrievalResult(
|
||||
EvidenceDecision.ANSWERABLE, "grounded_evidence_available",
|
||||
(_evidence("đoạn 0"), _evidence("đoạn 1")),
|
||||
)
|
||||
|
||||
generator_calls: list[dict] = []
|
||||
|
||||
class _CountingGenerator:
|
||||
def generate(self, system, user, schema):
|
||||
generator_calls.append(schema)
|
||||
return '{"answer": "unused", "evidence_sufficient": true, "clarifying_question": null}'
|
||||
|
||||
metrics = InMemoryMetrics()
|
||||
answers = GroundedAnswerService(
|
||||
routing=None, generator=_CountingGenerator(), metrics=metrics
|
||||
)
|
||||
agent = RagAgent(
|
||||
_BudgetSpendingUnderstander(), _FixedRetrieval({"metformin": result}), answers,
|
||||
max_llm_calls_per_turn=1,
|
||||
)
|
||||
|
||||
reply = agent.handle("liều metformin")
|
||||
|
||||
# The real point of F-08: no more actual provider calls happen once the
|
||||
# budget is spent — not "the generator returned an error", the generator
|
||||
# is never invoked at all.
|
||||
assert generator_calls == []
|
||||
assert reply.decision == "abstain"
|
||||
assert reply.answer is None
|
||||
assert metrics.total(GENERATION_REJECTED, reason="request_budget_exhausted") >= 1
|
||||
|
||||
|
||||
def test_a_generous_budget_does_not_change_normal_behaviour():
|
||||
"""Control for the test above: with the default (generous) budget, the
|
||||
same setup answers normally — proves the previous test's tiny budget is
|
||||
what caused the block, not some other change to the fixtures."""
|
||||
class _BudgetSpendingUnderstander:
|
||||
def understand(self, turn, history=(), budget=None, prior_frame=None):
|
||||
if budget is not None:
|
||||
budget.require()
|
||||
return QueryFrame(
|
||||
turn_type="drug_attribute", drugs=("metformin",),
|
||||
attribute="chi_dinh",
|
||||
)
|
||||
|
||||
result = RetrievalResult(
|
||||
EvidenceDecision.ANSWERABLE, "grounded_evidence_available",
|
||||
(_evidence("Liều 500 mg mỗi ngày."), _evidence("đoạn 1")),
|
||||
)
|
||||
|
||||
class _Generator:
|
||||
def generate(self, system, user, schema):
|
||||
if "sufficient" in schema.get("properties", {}):
|
||||
return '{"sufficient": true, "clarifying_question": null, "quick_replies": []}'
|
||||
if "entailed" in schema.get("properties", {}):
|
||||
return '{"entailed": true, "unsupported": []}'
|
||||
return '{"answer": "Liều 500 mg [1].", "evidence_sufficient": true, "clarifying_question": null}'
|
||||
|
||||
answers = GroundedAnswerService(routing=None, generator=_Generator())
|
||||
agent = RagAgent(
|
||||
_BudgetSpendingUnderstander(), _FixedRetrieval({"metformin": result}), answers,
|
||||
)
|
||||
|
||||
reply = agent.handle("liều metformin")
|
||||
|
||||
assert reply.decision == "answerable"
|
||||
assert reply.answer == "Liều 500 mg [1]."
|
||||
|
||||
|
||||
# --- durable conversation history (ADR 0008's named gap): an optional
|
||||
# `ConversationStore` replaces the in-process dict when configured --------
|
||||
|
||||
|
||||
class _FakeStore:
|
||||
def __init__(self) -> None:
|
||||
self.lines: dict[str, list[str]] = {}
|
||||
self.fail_reads = False
|
||||
self.fail_writes = False
|
||||
|
||||
def recent(self, conversation_id, limit):
|
||||
if self.fail_reads:
|
||||
raise ConnectionError("store unreachable")
|
||||
return self.lines.get(conversation_id, [])[-limit:]
|
||||
|
||||
def append(self, conversation_id, line):
|
||||
if self.fail_writes:
|
||||
raise ConnectionError("store unreachable")
|
||||
self.lines.setdefault(conversation_id, []).append(line)
|
||||
|
||||
|
||||
def test_history_round_trips_through_a_configured_store():
|
||||
received_history: list[tuple[str, ...]] = []
|
||||
|
||||
class _RecordingUnderstander:
|
||||
def understand(self, turn, history=(), budget=None, prior_frame=None):
|
||||
received_history.append(tuple(history))
|
||||
return QueryFrame(turn_type="smalltalk")
|
||||
|
||||
store = _FakeStore()
|
||||
answers = GroundedAnswerService(routing=None)
|
||||
agent = RagAgent(_RecordingUnderstander(), _FixedRetrieval({}), answers, store=store)
|
||||
|
||||
agent.handle("chào bạn", conversation_id="c1")
|
||||
agent.handle("còn liều thì sao?", conversation_id="c1")
|
||||
|
||||
assert received_history[0] == ()
|
||||
assert any("chào bạn" in line for line in received_history[1])
|
||||
assert store.lines["c1"] # actually persisted, not just read back in-process
|
||||
|
||||
|
||||
def test_store_read_failure_fails_open_to_fresh_history_not_a_crash():
|
||||
store = _FakeStore()
|
||||
store.lines["c1"] = ["Người dùng: câu cũ", "Trợ lý: trả lời cũ"]
|
||||
store.fail_reads = True
|
||||
|
||||
answers = GroundedAnswerService(routing=None)
|
||||
agent = RagAgent(
|
||||
_FixedUnderstander(QueryFrame(turn_type="smalltalk")),
|
||||
_FixedRetrieval({}), answers, store=store,
|
||||
)
|
||||
|
||||
# Must not raise -- degrades to no history for this turn.
|
||||
reply = agent.handle("chào bạn", conversation_id="c1")
|
||||
assert reply.decision == "answerable"
|
||||
|
||||
|
||||
def test_store_write_failure_fails_open_the_response_still_returns():
|
||||
store = _FakeStore()
|
||||
store.fail_writes = True
|
||||
|
||||
answers = GroundedAnswerService(routing=None)
|
||||
agent = RagAgent(
|
||||
_FixedUnderstander(QueryFrame(turn_type="smalltalk")),
|
||||
_FixedRetrieval({}), answers, store=store,
|
||||
)
|
||||
|
||||
# Must not raise, even though persisting this turn silently fails.
|
||||
reply = agent.handle("chào bạn", conversation_id="c1")
|
||||
assert reply.decision == "answerable"
|
||||
assert reply.answer is not None
|
||||
|
||||
|
||||
# --- F-11: the clarify-loop circuit breaker. Found live 2026-08-07
|
||||
# (50-question hand-typed browser audit): the understanding LLM can keep
|
||||
# deciding needs_clarify=true forever with no natural exit — reproduced 3
|
||||
# times independently, one case never converged after 5 real answered turns.
|
||||
# `understanding.py`'s prior-frame merge fixes most of the underlying cause,
|
||||
# but this is the code-level bound that guarantees a user is never stuck. --
|
||||
|
||||
|
||||
class _AlwaysClarifyUnderstander:
|
||||
"""Simulates a model stuck re-asking regardless of what the user
|
||||
answers — exactly the observed live failure, isolated from any real
|
||||
LLM's actual (variable) behaviour so this test is deterministic."""
|
||||
|
||||
def understand(self, turn, history=(), budget=None, prior_frame=None):
|
||||
return QueryFrame(
|
||||
turn_type="dosing_calc", drugs=("paracetamol_acetaminophen",),
|
||||
needs_clarify=True, clarify_reason="Bé nặng bao nhiêu kg?",
|
||||
)
|
||||
|
||||
|
||||
def test_clarify_loop_is_hard_stopped_after_max_consecutive_turns():
|
||||
answers = GroundedAnswerService(routing=None)
|
||||
agent = RagAgent(_AlwaysClarifyUnderstander(), _FixedRetrieval({}), answers)
|
||||
|
||||
replies = [
|
||||
agent.handle(f"turn {i}", conversation_id="stuck")
|
||||
for i in range(MAX_CONSECUTIVE_CLARIFY)
|
||||
]
|
||||
|
||||
# Every turn up to the last stays a genuine clarify...
|
||||
for reply in replies[:-1]:
|
||||
assert reply.decision == "clarify"
|
||||
# ...the Nth forces a hard stop instead of asking again.
|
||||
assert replies[-1].decision == "abstain"
|
||||
assert replies[-1].reason == "clarify_loop_exhausted"
|
||||
assert replies[-1].answer is not None
|
||||
|
||||
|
||||
def test_clarify_streak_resets_after_the_hard_stop_so_a_new_attempt_can_proceed():
|
||||
"""Confirms the breaker is a bounded pause, not a permanent lockout of
|
||||
the conversation id — the very next turn gets a fresh streak."""
|
||||
answers = GroundedAnswerService(routing=None)
|
||||
agent = RagAgent(_AlwaysClarifyUnderstander(), _FixedRetrieval({}), answers)
|
||||
|
||||
for i in range(MAX_CONSECUTIVE_CLARIFY):
|
||||
agent.handle(f"turn {i}", conversation_id="stuck")
|
||||
reply = agent.handle("one more try", conversation_id="stuck")
|
||||
|
||||
assert reply.decision == "clarify"
|
||||
|
||||
|
||||
def test_a_resolved_turn_resets_the_clarify_streak():
|
||||
"""An answerable turn in between two clarify runs must not let their
|
||||
streaks combine — only genuinely consecutive clarifies count."""
|
||||
result = RetrievalResult(
|
||||
EvidenceDecision.ANSWERABLE, "grounded_evidence_available",
|
||||
(_evidence("Liều 500 mg."),),
|
||||
)
|
||||
clarify_frame = QueryFrame(
|
||||
turn_type="dosing_calc", drugs=("paracetamol_acetaminophen",),
|
||||
needs_clarify=True, clarify_reason="Bé nặng bao nhiêu kg?",
|
||||
)
|
||||
resolved_frame = QueryFrame(
|
||||
turn_type="drug_attribute", drugs=("metformin",),
|
||||
attribute="lieu_luong_va_cach_dung",
|
||||
)
|
||||
# (MAX-1) clarifies, one resolved turn, then (MAX-1) clarifies again —
|
||||
# scripted explicitly so the test asserts the streak reset, not an
|
||||
# incidental side effect of some other call-counting scheme.
|
||||
script = (
|
||||
[clarify_frame] * (MAX_CONSECUTIVE_CLARIFY - 1)
|
||||
+ [resolved_frame]
|
||||
+ [clarify_frame] * (MAX_CONSECUTIVE_CLARIFY - 1)
|
||||
)
|
||||
|
||||
class _ScriptedUnderstander:
|
||||
def __init__(self, frames):
|
||||
self._frames = iter(frames)
|
||||
|
||||
def understand(self, turn, history=(), budget=None, prior_frame=None):
|
||||
return next(self._frames)
|
||||
|
||||
answers = GroundedAnswerService(routing=None)
|
||||
agent = RagAgent(
|
||||
_ScriptedUnderstander(script), _FixedRetrieval({"metformin": result}), answers,
|
||||
)
|
||||
|
||||
for i in range(MAX_CONSECUTIVE_CLARIFY - 1):
|
||||
reply = agent.handle(f"clarify {i}", conversation_id="c1")
|
||||
assert reply.decision == "clarify"
|
||||
resolved = agent.handle("answerable turn", conversation_id="c1")
|
||||
assert resolved.decision == "answerable"
|
||||
|
||||
# Streak was reset by the resolved turn -- this run of clarifies must
|
||||
# not be treated as a continuation of the earlier (pre-reset) run.
|
||||
for i in range(MAX_CONSECUTIVE_CLARIFY - 1):
|
||||
reply = agent.handle(f"clarify again {i}", conversation_id="c1")
|
||||
assert reply.decision == "clarify"
|
||||
|
||||
|
||||
def test_prior_frame_is_threaded_from_the_last_turn_to_the_understander():
|
||||
received: list[QueryFrame | None] = []
|
||||
|
||||
class _RecordingUnderstander:
|
||||
def understand(self, turn, history=(), budget=None, prior_frame=None):
|
||||
received.append(prior_frame)
|
||||
return QueryFrame(
|
||||
turn_type="dosing_calc", drugs=("paracetamol_acetaminophen",),
|
||||
needs_clarify=True, clarify_reason="Bé nặng bao nhiêu kg?",
|
||||
)
|
||||
|
||||
answers = GroundedAnswerService(routing=None)
|
||||
agent = RagAgent(_RecordingUnderstander(), _FixedRetrieval({}), answers)
|
||||
|
||||
agent.handle("liều paracetamol cho bé", conversation_id="c1")
|
||||
agent.handle("20kg", conversation_id="c1")
|
||||
|
||||
assert received[0] is None
|
||||
assert received[1] is not None
|
||||
assert received[1].clarify_reason == "Bé nặng bao nhiêu kg?"
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
"""`rag/budget.py::RequestBudget` — F-08."""
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
from rag.budget import RequestBudget, RequestBudgetExhausted
|
||||
from rag.ports import AnswerGenerationUnavailable
|
||||
|
||||
|
||||
def test_budget_exhausted_is_a_subclass_of_answer_generation_unavailable():
|
||||
"""Deliberate: every existing `except AnswerGenerationUnavailable:`
|
||||
fail-open/fail-closed handler in the codebase must catch this with zero
|
||||
changes, since it predates F-08 and already encodes the right behaviour
|
||||
for "the provider is unavailable to us right now"."""
|
||||
assert issubclass(RequestBudgetExhausted, AnswerGenerationUnavailable)
|
||||
|
||||
|
||||
def test_fresh_budget_has_budget():
|
||||
budget = RequestBudget.start(max_wall_clock_ms=20_000, max_calls=5)
|
||||
assert budget.has_budget() is True
|
||||
|
||||
|
||||
def test_require_spends_one_call():
|
||||
budget = RequestBudget.start(max_wall_clock_ms=20_000, max_calls=2)
|
||||
budget.require()
|
||||
assert budget.calls_remaining == 1
|
||||
budget.require()
|
||||
assert budget.calls_remaining == 0
|
||||
assert budget.has_budget() is False
|
||||
|
||||
|
||||
def test_require_raises_once_calls_are_exhausted():
|
||||
budget = RequestBudget.start(max_wall_clock_ms=20_000, max_calls=1)
|
||||
budget.require()
|
||||
with pytest.raises(RequestBudgetExhausted):
|
||||
budget.require()
|
||||
|
||||
|
||||
def test_require_raises_once_the_deadline_has_passed():
|
||||
budget = RequestBudget.start(max_wall_clock_ms=0, max_calls=100)
|
||||
time.sleep(0.01)
|
||||
assert budget.has_budget() is False
|
||||
with pytest.raises(RequestBudgetExhausted):
|
||||
budget.require()
|
||||
|
||||
|
||||
def test_a_failed_require_does_not_spend_a_call():
|
||||
"""`require()` raises before decrementing when there's no budget left —
|
||||
`calls_remaining` must not go negative, which would otherwise make a
|
||||
budget that's already exhausted look like it has "negative debt" instead
|
||||
of cleanly `0`."""
|
||||
budget = RequestBudget.start(max_wall_clock_ms=20_000, max_calls=1)
|
||||
budget.require()
|
||||
for _ in range(3):
|
||||
with pytest.raises(RequestBudgetExhausted):
|
||||
budget.require()
|
||||
assert budget.calls_remaining == 0
|
||||
@@ -16,6 +16,7 @@ from rag.models import (
|
||||
SourceRef,
|
||||
SubjectScope,
|
||||
)
|
||||
from rag.ports import AnswerGenerationUnavailable
|
||||
from rag.prompt import build_request
|
||||
|
||||
|
||||
@@ -41,22 +42,31 @@ class _Routing:
|
||||
|
||||
|
||||
class _Generator:
|
||||
def __init__(self, payload: dict, entailment_payload: dict | None = None) -> None:
|
||||
def __init__(
|
||||
self, payload: dict, entailment_payload: dict | None = None,
|
||||
sufficiency_payload: dict | None = None,
|
||||
) -> None:
|
||||
self._payload = payload
|
||||
self._entailment_payload = entailment_payload or {
|
||||
"entailed": True,
|
||||
"unsupported": [],
|
||||
}
|
||||
self._sufficiency_payload = sufficiency_payload
|
||||
|
||||
def generate(self, system: str, user: str, schema: dict) -> str: # noqa: ARG002
|
||||
# `_generate` also runs a post-generation entailment check; tell the
|
||||
# two request shapes apart by schema so callers here only need to
|
||||
# fake the main answer, not both.
|
||||
payload = (
|
||||
self._entailment_payload
|
||||
if "entailed" in schema.get("properties", {})
|
||||
else self._payload
|
||||
)
|
||||
# `_generate` also runs a post-generation entailment check, and
|
||||
# `_check_sufficiency` runs its own separate call before that — tell
|
||||
# the three request shapes apart by schema so a test asserting on one
|
||||
# doesn't have to also shape a payload for the others.
|
||||
props = schema.get("properties", {})
|
||||
if "entailed" in props:
|
||||
payload = self._entailment_payload
|
||||
elif "sufficient" in props and self._sufficiency_payload is not None:
|
||||
payload = self._sufficiency_payload
|
||||
else:
|
||||
payload = self._payload
|
||||
if isinstance(payload, BaseException):
|
||||
raise payload
|
||||
return json.dumps(payload, ensure_ascii=False)
|
||||
|
||||
|
||||
@@ -122,6 +132,42 @@ def test_underspecified_dose_asks_instead_of_dumping():
|
||||
assert "tuổi" in g.clarification
|
||||
assert g.answer == g.clarification
|
||||
assert g.generated is False
|
||||
assert g.quick_replies == ()
|
||||
|
||||
|
||||
def test_underspecified_dose_carries_quick_replies_when_the_model_offers_them():
|
||||
result = _answerable(_evidence(0, 100), _evidence(1, 200))
|
||||
gen = _Generator(
|
||||
{"sufficient": False,
|
||||
"clarifying_question": "Người lớn hay trẻ em?",
|
||||
"quick_replies": ["Người lớn", "Trẻ em"]}
|
||||
)
|
||||
service = GroundedAnswerService(_Routing(result), gen)
|
||||
|
||||
g = service.answer("paracetamol", SubjectScope.HUMAN, QueryIntent.FACT_LOOKUP)
|
||||
|
||||
assert g.quick_replies == ("Người lớn", "Trẻ em")
|
||||
|
||||
|
||||
def test_sufficiency_check_outage_fails_open_to_generation_not_abstain():
|
||||
"""F-10: `_check_sufficiency` documents (and this pins) a deliberate
|
||||
fail-OPEN on provider outage — unlike every other failure mode in this
|
||||
service, a sufficiency-check outage does not abstain, it just skips the
|
||||
clarify heuristic and lets grounding/entailment (tested elsewhere) be
|
||||
the real safety net on whatever gets generated next."""
|
||||
result = _answerable(_evidence(0, 100), _evidence(1, 200))
|
||||
gen = _Generator(
|
||||
{"answer": "Đoạn bằng chứng 0 [1].", "evidence_sufficient": True,
|
||||
"clarifying_question": None},
|
||||
sufficiency_payload=AnswerGenerationUnavailable("Bedrock unreachable"),
|
||||
)
|
||||
service = GroundedAnswerService(_Routing(result), gen)
|
||||
|
||||
g = service.answer("liều người lớn", SubjectScope.HUMAN, QueryIntent.FACT_LOOKUP)
|
||||
|
||||
assert g.clarification is None
|
||||
assert g.result.decision == EvidenceDecision.ANSWERABLE
|
||||
assert g.generated is True
|
||||
|
||||
|
||||
def test_sufficient_query_is_not_turned_into_a_clarification():
|
||||
@@ -144,3 +190,52 @@ def test_bare_name_builds_an_intro_prompt():
|
||||
normal = build_request("Liều?", ("đoạn A",), intro=False)
|
||||
assert "CÂU HỎI:" in normal.user
|
||||
assert "GIỚI THIỆU" not in normal.user
|
||||
|
||||
|
||||
def test_list_mode_prompt_instructs_enumerating_every_drug_not_ranking():
|
||||
"""2026-08-07 finding: without an explicit instruction, the model picked
|
||||
one drug out of 8 real symptom_to_drug matches and silently dropped the
|
||||
rest — verified live. `list_mode` closes that."""
|
||||
request = build_request("thuốc gì trị sốt", ("chỉ định A", "chỉ định B"), list_mode=True)
|
||||
assert "LIỆT KÊ TẤT CẢ" in request.user
|
||||
assert "không xếp hạng" in request.user or "KHÔNG" in request.user
|
||||
assert "CÂU HỎI:" in request.user
|
||||
|
||||
|
||||
def test_list_mode_skips_the_sufficiency_clarify():
|
||||
"""If the sufficiency call were NOT actually skipped, it would read
|
||||
`sufficiency_payload` (`sufficient=False`) and clarify. `list_mode=True`
|
||||
must never call it at all, so only the real answer payload is ever read."""
|
||||
result = _answerable(_evidence(0, 100), _evidence(1, 200))
|
||||
gen = _Generator(
|
||||
{"answer": "Đoạn bằng chứng 0 [1]. Đoạn bằng chứng 1 [2].",
|
||||
"evidence_sufficient": True, "clarifying_question": None},
|
||||
sufficiency_payload={
|
||||
"sufficient": False, "clarifying_question": "Loại nào?", "quick_replies": [],
|
||||
},
|
||||
)
|
||||
service = GroundedAnswerService(_Routing(result), gen)
|
||||
|
||||
g = service.answer_from_result("thuốc gì trị sốt", result, list_mode=True)
|
||||
|
||||
assert g.clarification is None
|
||||
assert g.answer is not None
|
||||
assert g.generated is True
|
||||
|
||||
|
||||
def test_without_list_mode_the_same_evidence_does_ask_for_clarification():
|
||||
"""Control for the test above: the same sufficiency payload, without
|
||||
`list_mode`, must actually clarify — proving the previous test's "not
|
||||
skipped" branch is reachable and would have failed loudly."""
|
||||
result = _answerable(_evidence(0, 100), _evidence(1, 200))
|
||||
gen = _Generator(
|
||||
{"answer": "unused", "evidence_sufficient": True, "clarifying_question": None},
|
||||
sufficiency_payload={
|
||||
"sufficient": False, "clarifying_question": "Loại nào?", "quick_replies": [],
|
||||
},
|
||||
)
|
||||
service = GroundedAnswerService(_Routing(result), gen)
|
||||
|
||||
g = service.answer_from_result("thuốc gì trị sốt", result, list_mode=False)
|
||||
|
||||
assert g.clarification == "Loại nào?"
|
||||
|
||||
@@ -1,189 +0,0 @@
|
||||
"""Follow-ups must inherit context, and must never inherit it silently.
|
||||
|
||||
The cases here are the ones the owner named on 2026-08-05: "còn trẻ em thì
|
||||
sao?", "giải thích kỹ hơn", and not making the user repeat themselves. The
|
||||
adversarial cases are the ones that make inheritance dangerous in a formulary
|
||||
— a stale drug, and an explicit mention being overridden by context.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from rag.conversation import (
|
||||
FOCUS_TTL_TURNS,
|
||||
ConversationState,
|
||||
Focus,
|
||||
Turn,
|
||||
detect_population,
|
||||
detect_verbosity,
|
||||
looks_like_followup,
|
||||
resolve_against,
|
||||
update_focus,
|
||||
)
|
||||
|
||||
|
||||
def _state(turn_count: int = 1, **focus_fields) -> ConversationState:
|
||||
focus = Focus()
|
||||
for name, value in focus_fields.items():
|
||||
focus = focus.with_field(name, value, turn_count - 1)
|
||||
return ConversationState("c1", focus=focus, turn_count=turn_count)
|
||||
|
||||
|
||||
# --- the follow-ups the owner asked for --------------------------------------
|
||||
|
||||
|
||||
def test_con_tre_em_thi_sao_inherits_drug_and_section():
|
||||
state = _state(drug_id="metformin", section_key="lieu_luong_va_cach_dung")
|
||||
|
||||
resolved = resolve_against(state, "còn trẻ em thì sao?", None, None)
|
||||
|
||||
assert resolved.drug_id == "metformin"
|
||||
assert resolved.section_key == "lieu_luong_va_cach_dung"
|
||||
assert resolved.population == "tre_em"
|
||||
assert resolved.inherited_drug is True
|
||||
|
||||
|
||||
def test_giai_thich_ky_hon_sets_verbosity_and_keeps_the_topic():
|
||||
state = _state(drug_id="warfarin", section_key="tuong_tac_thuoc")
|
||||
|
||||
resolved = resolve_against(state, "giải thích kỹ hơn", None, None)
|
||||
|
||||
assert resolved.drug_id == "warfarin"
|
||||
assert resolved.verbosity == "detailed"
|
||||
|
||||
|
||||
def test_the_user_is_not_made_to_repeat_the_drug():
|
||||
state = _state(drug_id="metformin")
|
||||
|
||||
resolved = resolve_against(state, "chống chỉ định", None, "chong_chi_dinh")
|
||||
|
||||
assert resolved.drug_id == "metformin"
|
||||
assert resolved.section_key == "chong_chi_dinh"
|
||||
|
||||
|
||||
# --- what makes inheritance safe ---------------------------------------------
|
||||
|
||||
|
||||
def test_an_explicit_drug_always_beats_context():
|
||||
"""Naming a drug must override whatever the conversation was about, or a
|
||||
deliberate topic change silently answers about the previous medicine."""
|
||||
state = _state(drug_id="metformin", section_key="lieu_luong_va_cach_dung")
|
||||
|
||||
resolved = resolve_against(state, "liều dùng warfarin", "warfarin", None)
|
||||
|
||||
assert resolved.drug_id == "warfarin"
|
||||
assert resolved.inherited_drug is False
|
||||
|
||||
|
||||
def test_a_stale_drug_is_dropped_rather_than_inherited():
|
||||
"""Beyond the TTL the drug is not context, it is a hazard."""
|
||||
state = _state(turn_count=FOCUS_TTL_TURNS + 3, drug_id="metformin")
|
||||
# `_state` stamps at turn_count - 1, so age is 1; age it past the TTL.
|
||||
aged = ConversationState(
|
||||
"c1",
|
||||
focus=Focus(drug_id="metformin", set_at_turn={"drug_id": 0}),
|
||||
turn_count=FOCUS_TTL_TURNS + 2,
|
||||
)
|
||||
|
||||
assert state.inherited("drug_id") == "metformin"
|
||||
assert aged.inherited("drug_id") is None
|
||||
|
||||
resolved = resolve_against(aged, "còn trẻ em thì sao?", None, None)
|
||||
assert resolved.drug_id is None
|
||||
|
||||
|
||||
def test_an_inherited_drug_must_be_named_in_the_answer():
|
||||
state = _state(drug_id="metformin")
|
||||
|
||||
inherited = resolve_against(state, "còn trẻ em thì sao?", None, None)
|
||||
explicit = resolve_against(state, "liều warfarin", "warfarin", None)
|
||||
|
||||
assert inherited.needs_carry_over_notice is True
|
||||
assert explicit.needs_carry_over_notice is False
|
||||
|
||||
|
||||
# --- phrase detection ---------------------------------------------------------
|
||||
|
||||
|
||||
def test_longest_population_phrase_wins():
|
||||
"""`phụ nữ cho con bú` must not be read as `phụ nữ`, and `trẻ sơ sinh`
|
||||
must not be read as `trẻ em` — the same rule `sections.py` relies on."""
|
||||
assert detect_population("phụ nữ cho con bú") == "phu_nu_cho_con_bu"
|
||||
assert detect_population("trẻ sơ sinh dùng sao") == "tre_so_sinh"
|
||||
assert detect_population("bà bầu uống được không") == "phu_nu_co_thai"
|
||||
assert detect_population("liều cho người lớn") == "nguoi_lon"
|
||||
|
||||
|
||||
def test_no_population_named_is_none_not_a_guess():
|
||||
assert detect_population("liều dùng paracetamol") is None
|
||||
assert detect_verbosity("liều dùng paracetamol") is None
|
||||
|
||||
|
||||
def test_followup_markers():
|
||||
assert looks_like_followup("còn trẻ em thì sao?") is True
|
||||
assert looks_like_followup("so với metformin thì sao") is True
|
||||
assert looks_like_followup("liều dùng paracetamol") is False
|
||||
|
||||
|
||||
# --- window and focus update --------------------------------------------------
|
||||
|
||||
|
||||
def test_recent_window_evicts_oldest():
|
||||
state = ConversationState("c1")
|
||||
for index in range(8):
|
||||
state = state.append(Turn("user", f"q{index}", "2026-08-05"), window=6)
|
||||
|
||||
assert len(state.recent) == 6
|
||||
assert state.recent[0].text == "q2"
|
||||
assert state.turn_count == 8
|
||||
|
||||
|
||||
def test_evicted_turns_reach_overflow_not_silently_dropped():
|
||||
"""Bug fixed 2026-08-06 (Codex review, F-06): `overflow()` used to check
|
||||
`len(self.recent) > window`, but `append()` already truncates `recent`
|
||||
to `window`, so that comparison could never be true — evicted turns
|
||||
never reached the summariser no matter how long a conversation ran.
|
||||
Exact repro from the review: 8 turns into a window of 6."""
|
||||
state = ConversationState("c1")
|
||||
for index in range(8):
|
||||
state = state.append(Turn("user", f"q{index}", "2026-08-05"), window=6)
|
||||
|
||||
overflow = state.overflow()
|
||||
|
||||
assert [turn.text for turn in overflow] == ["q0", "q1"]
|
||||
|
||||
|
||||
def test_overflow_accumulates_across_the_two_appends_one_turn_makes():
|
||||
"""A live turn typically calls `append()` twice in a row (user, then
|
||||
assistant). Each can evict at most one turn; the second call's overflow
|
||||
must not overwrite, and so lose, the first's."""
|
||||
state = ConversationState("c1")
|
||||
for index in range(6):
|
||||
state = state.append(Turn("user", f"q{index}", "2026-08-05"), window=6)
|
||||
assert state.overflow() == () # window exactly full, nothing evicted yet
|
||||
|
||||
state = state.append(Turn("user", "q6", "2026-08-05"), window=6)
|
||||
state = state.append(Turn("assistant", "a6", "2026-08-05"), window=6)
|
||||
|
||||
assert [turn.text for turn in state.overflow()] == ["q0", "q1"]
|
||||
|
||||
|
||||
def test_overflow_is_empty_again_after_the_caller_clears_it():
|
||||
from dataclasses import replace
|
||||
|
||||
state = ConversationState("c1")
|
||||
for index in range(8):
|
||||
state = state.append(Turn("user", f"q{index}", "2026-08-05"), window=6)
|
||||
assert state.overflow() != ()
|
||||
|
||||
state = replace(state, pending_overflow=())
|
||||
|
||||
assert state.overflow() == ()
|
||||
|
||||
|
||||
def test_focus_update_stamps_the_current_turn():
|
||||
state = _state(turn_count=3)
|
||||
|
||||
resolved = resolve_against(state, "liều dùng metformin", "metformin", "lieu_luong_va_cach_dung")
|
||||
focus = update_focus(state, resolved)
|
||||
|
||||
assert focus.drug_id == "metformin"
|
||||
assert focus.set_at_turn["drug_id"] == 3
|
||||
@@ -1,50 +0,0 @@
|
||||
from rag.conversation import (
|
||||
ConversationState,
|
||||
DeterministicSummariser,
|
||||
InMemoryConversationStore,
|
||||
Turn,
|
||||
)
|
||||
|
||||
|
||||
def test_store_returns_fresh_state_for_unknown_id():
|
||||
store = InMemoryConversationStore()
|
||||
state = store.load("conv-new")
|
||||
assert state.conversation_id == "conv-new"
|
||||
assert state.turn_count == 0
|
||||
assert state.recent == ()
|
||||
|
||||
|
||||
def test_store_round_trips_saved_state():
|
||||
store = InMemoryConversationStore()
|
||||
state = ConversationState("conv-1", summary="s", turn_count=3)
|
||||
store.save(state)
|
||||
assert store.load("conv-1") is state
|
||||
|
||||
|
||||
def test_summariser_records_topic_labels_only():
|
||||
s = DeterministicSummariser()
|
||||
dropped = (
|
||||
Turn("user", "Chống chỉ định của metformin?", "t0",
|
||||
drug_id="metformin", section_key="chong_chi_dinh"),
|
||||
Turn("assistant", "Quá mẫn với metformin, suy thận Clcr < 60...", "t1",
|
||||
drug_id="metformin", section_key="chong_chi_dinh"),
|
||||
)
|
||||
out = s.fold("", dropped)
|
||||
# The label line is present...
|
||||
assert "chong_chi_dinh của metformin" in out
|
||||
# ...and no clinical value leaked from the assistant turn.
|
||||
assert "Clcr" not in out
|
||||
assert "60" not in out
|
||||
|
||||
|
||||
def test_summariser_stays_within_budget_dropping_oldest():
|
||||
s = DeterministicSummariser()
|
||||
dropped = tuple(
|
||||
Turn("user", f"q{i}", f"t{i}", drug_id=f"drug{i}", section_key="lieu_luong")
|
||||
for i in range(400)
|
||||
)
|
||||
out = s.fold("", dropped)
|
||||
assert len(out) <= DeterministicSummariser.MAX_CHARS
|
||||
# Most-recent topic survives, oldest is dropped.
|
||||
assert "drug399" in out
|
||||
assert "drug0 " not in out
|
||||
@@ -1,133 +0,0 @@
|
||||
from rag.answer import GroundedAnswer
|
||||
from rag.conversation import DeterministicSummariser, InMemoryConversationStore
|
||||
from rag.conversational import (
|
||||
SMALLTALK_REPLY,
|
||||
ConversationalLoopService,
|
||||
)
|
||||
from rag.models import (
|
||||
Evidence,
|
||||
EvidenceDecision,
|
||||
QueryIntent,
|
||||
RetrievalResult,
|
||||
SubjectScope,
|
||||
)
|
||||
from rag.reasoning import ClarifyReason
|
||||
from rag.routing import CatalogDrugResolver
|
||||
from rag.sections import SectionResolver
|
||||
|
||||
|
||||
def _grounded(answer, evidence_text):
|
||||
ev = Evidence("e1", "e1", "prose", evidence_text, 1.0, (), False, False)
|
||||
result = RetrievalResult(
|
||||
EvidenceDecision.ANSWERABLE, "grounded_evidence_available",
|
||||
(ev,), "metformin", "resolved",
|
||||
)
|
||||
return GroundedAnswer(result, answer, (), False)
|
||||
|
||||
|
||||
class FakeAnswers:
|
||||
def __init__(self, answer_text, evidence_text):
|
||||
self._a = answer_text
|
||||
self._e = evidence_text
|
||||
self.calls = []
|
||||
|
||||
def answer(self, query, subject_scope, intent, drug_id=None):
|
||||
self.calls.append((query, drug_id))
|
||||
return _grounded(self._a, self._e)
|
||||
|
||||
|
||||
def _service(answers):
|
||||
return ConversationalLoopService(
|
||||
answers=answers,
|
||||
resolver=CatalogDrugResolver({"metformin": {"metformin"}}),
|
||||
section_resolver=SectionResolver(),
|
||||
store=InMemoryConversationStore(),
|
||||
summariser=DeterministicSummariser(),
|
||||
)
|
||||
|
||||
|
||||
def test_smalltalk_answers_socially_without_calling_engine():
|
||||
answers = FakeAnswers("x", "x")
|
||||
svc = _service(answers)
|
||||
out = svc.answer("c1", "chào bạn", SubjectScope.HUMAN, QueryIntent.FACT_LOOKUP)
|
||||
assert out.smalltalk is True
|
||||
assert out.answer == SMALLTALK_REPLY
|
||||
assert answers.calls == [] # a greeting is not a drug lookup
|
||||
|
||||
|
||||
def test_medical_turn_returns_grounded_answer():
|
||||
answers = FakeAnswers("Quá mẫn với metformin.", "Quá mẫn với metformin.")
|
||||
svc = _service(answers)
|
||||
out = svc.answer(
|
||||
"c2", "chống chỉ định metformin", SubjectScope.HUMAN, QueryIntent.FACT_LOOKUP
|
||||
)
|
||||
assert out.smalltalk is False
|
||||
assert out.answer == "Quá mẫn với metformin."
|
||||
assert out.grounded is not None
|
||||
|
||||
|
||||
def test_followup_inherits_drug_and_passes_it_resolved():
|
||||
answers = FakeAnswers(
|
||||
"Ở trẻ em điều chỉnh theo cân nặng.",
|
||||
"Ở trẻ em, liều metformin điều chỉnh theo cân nặng.",
|
||||
)
|
||||
svc = _service(answers)
|
||||
svc.answer("c3", "chống chỉ định metformin", SubjectScope.HUMAN, QueryIntent.FACT_LOOKUP)
|
||||
out = svc.answer("c3", "còn trẻ em thì sao?", SubjectScope.HUMAN, QueryIntent.FACT_LOOKUP)
|
||||
assert out.inherited_drug == "metformin"
|
||||
assert out.answer.startswith("Về Metformin:")
|
||||
# The inherited drug is passed already-resolved (not re-resolved from the
|
||||
# rewritten turn text), and the raw follow-up drives section routing.
|
||||
last_query, last_drug_id = answers.calls[-1]
|
||||
assert last_drug_id == "metformin"
|
||||
assert last_query == "còn trẻ em thì sao?"
|
||||
# State carried the drug forward.
|
||||
assert svc._store.load("c3").focus.drug_id == "metformin"
|
||||
|
||||
|
||||
def test_confirmation_is_not_fuzzy_matched_to_a_drug():
|
||||
"""'đúng' must not be fuzzy-matched to terbinafin/tretinoin (the did-you-mean
|
||||
loop the reviewer hit); it asks which drug instead."""
|
||||
answers = FakeAnswers("x", "x")
|
||||
svc = _service(answers)
|
||||
out = svc.answer("cc", "đúng", SubjectScope.HUMAN, QueryIntent.FACT_LOOKUP)
|
||||
assert out.clarification is not None
|
||||
assert out.clarification.reason == "confirm_without_context"
|
||||
assert answers.calls == []
|
||||
|
||||
|
||||
def test_a_long_sentence_that_names_no_drug_is_not_offered_did_you_mean():
|
||||
"""A full question ('EPO điều trị thiếu máu...') that resolves no drug is
|
||||
answered honestly, not with garbage suggestions from fuzzing the sentence."""
|
||||
answers = FakeAnswers("x", "x")
|
||||
svc = _service(answers) # catalog holds only metformin
|
||||
out = svc.answer(
|
||||
"cl", "EPO điều trị thiếu máu do hóa trị ung thư liều khởi đầu bao nhiêu",
|
||||
SubjectScope.HUMAN, QueryIntent.FACT_LOOKUP,
|
||||
)
|
||||
assert out.clarification is not None
|
||||
assert out.clarification.reason == "drug_not_supported"
|
||||
assert answers.calls == []
|
||||
|
||||
|
||||
def test_no_close_drug_reports_not_supported():
|
||||
answers = FakeAnswers("x", "x")
|
||||
svc = _service(answers) # catalog holds only metformin
|
||||
out = svc.answer("c4", "cái này thế nào?", SubjectScope.HUMAN, QueryIntent.FACT_LOOKUP)
|
||||
assert out.answer is None
|
||||
assert out.clarification is not None
|
||||
# Nothing close to a real drug: honest "not in the formulary", not a guess.
|
||||
assert out.clarification.reason == "drug_not_supported"
|
||||
assert answers.calls == []
|
||||
|
||||
|
||||
def test_typo_offers_did_you_mean_not_silent_resolution():
|
||||
answers = FakeAnswers("x", "x")
|
||||
svc = _service(answers) # catalog holds only metformin
|
||||
out = svc.answer("c5", "metformim", SubjectScope.HUMAN, QueryIntent.FACT_LOOKUP)
|
||||
# A near-miss is asked about, never auto-resolved on a similarity threshold.
|
||||
assert out.answer is None
|
||||
assert out.clarification is not None
|
||||
assert out.clarification.reason == "did_you_mean"
|
||||
assert "Metformin" in out.clarification.options
|
||||
assert answers.calls == []
|
||||
@@ -1,81 +0,0 @@
|
||||
from rag.conversation import DeterministicSummariser, InMemoryConversationStore
|
||||
from rag.conversational import ConversationalRagService, TurnResolution
|
||||
from rag.reasoning import ClarifyReason, MAX_LLM_CALLS, MAX_RETRIEVAL_ROUNDS, TurnBudget
|
||||
|
||||
|
||||
class FakeResolver:
|
||||
"""Maps a turn's text to what it resolves on its own (no context)."""
|
||||
|
||||
def __init__(self, table):
|
||||
self._table = table
|
||||
|
||||
def resolve_turn(self, text):
|
||||
for needle, resolution in self._table:
|
||||
if needle in text:
|
||||
return resolution
|
||||
return TurnResolution(drug_id=None, section_key=None, drug_status="not_found")
|
||||
|
||||
|
||||
def _service(resolver, retrieve, generate):
|
||||
return ConversationalRagService(
|
||||
store=InMemoryConversationStore(),
|
||||
summariser=DeterministicSummariser(),
|
||||
resolver=resolver,
|
||||
retrieve=retrieve,
|
||||
generate=generate,
|
||||
)
|
||||
|
||||
|
||||
def test_followup_inherits_drug_and_answer_names_it():
|
||||
resolver = FakeResolver([
|
||||
("metformin", TurnResolution("metformin", "chong_chi_dinh", "resolved")),
|
||||
# "còn trẻ em" names no drug on its own — must inherit.
|
||||
("trẻ em", TurnResolution(None, None, "not_found")),
|
||||
])
|
||||
# Evidence mentions "trẻ em" so the population assessor is satisfied.
|
||||
retrieve = lambda q: ("Ở trẻ em, liều metformin điều chỉnh theo cân nặng.",)
|
||||
generate = lambda q, ev, st: "liều theo cân nặng"
|
||||
svc = _service(resolver, retrieve, generate)
|
||||
|
||||
first = svc.answer("c1", "Chống chỉ định của metformin?")
|
||||
assert first.inherited_drug is None
|
||||
|
||||
second = svc.answer("c1", "còn trẻ em thì sao?")
|
||||
assert second.inherited_drug == "metformin"
|
||||
assert second.answer.startswith("Về metformin:")
|
||||
|
||||
|
||||
def test_no_drug_and_no_context_asks_without_spending_budget():
|
||||
resolver = FakeResolver([]) # nothing resolves
|
||||
calls = {"retrieve": 0, "generate": 0}
|
||||
|
||||
def retrieve(q):
|
||||
calls["retrieve"] += 1
|
||||
return ("x",)
|
||||
|
||||
def generate(q, ev, st):
|
||||
calls["generate"] += 1
|
||||
return "x"
|
||||
|
||||
svc = _service(resolver, retrieve, generate)
|
||||
budget = TurnBudget()
|
||||
out = svc.answer("c2", "cái này thế nào?", budget=budget)
|
||||
|
||||
assert out.answer is None
|
||||
assert out.clarification is not None
|
||||
assert out.clarification.reason == ClarifyReason.AMBIGUOUS_DRUG
|
||||
# Asking short-circuits before any spend.
|
||||
assert calls == {"retrieve": 0, "generate": 0}
|
||||
assert budget.retrieval_rounds == MAX_RETRIEVAL_ROUNDS
|
||||
assert budget.llm_calls == MAX_LLM_CALLS
|
||||
|
||||
|
||||
def test_state_persists_across_turns():
|
||||
resolver = FakeResolver([
|
||||
("metformin", TurnResolution("metformin", "chi_dinh", "resolved")),
|
||||
])
|
||||
svc = _service(resolver, lambda q: ("Chỉ định của metformin.",), lambda q, ev, st: "ok")
|
||||
svc.answer("c3", "chỉ định metformin?")
|
||||
state = svc._store.load("c3")
|
||||
assert state.turn_count == 2 # user + assistant
|
||||
assert state.focus.drug_id == "metformin"
|
||||
@@ -67,23 +67,28 @@ class _FixedRouting:
|
||||
class _Generator:
|
||||
"""Returns whatever payload the test wants the model to have produced.
|
||||
|
||||
`_generate` now makes up to four calls through this port: the main
|
||||
answer, a sufficiency check (skipped here — one evidence block), and up
|
||||
to two entailment calls (a reject retries once — live probing found the
|
||||
judge noisy on an identical claim/evidence pair). They're told apart by
|
||||
schema, so a test that only cares about the main answer doesn't also
|
||||
have to fake an entailment response by hand; `entailment_payload`
|
||||
overrides it when a test wants the entailment pass to reject. Pass a
|
||||
list of payloads to get a different answer on each successive
|
||||
entailment call (e.g. `[reject, accept]` for the retry-recovers case).
|
||||
`_generate` now makes up to five calls through this port: the main
|
||||
answer (a lone `evidence_sufficient: false` retries once — the same
|
||||
noisy-judge finding as entailment, live-confirmed 2026-08-07), a
|
||||
sufficiency check (skipped here — one evidence block), and up to three
|
||||
entailment calls (widened from two 2026-08-07: live probing found the
|
||||
judge noisy on an identical claim/evidence pair, and a real adversarial
|
||||
sample showed a single retry still discarding correct answers on the
|
||||
unlucky reject-reject draw). They're told apart by schema, so a test
|
||||
that only cares about one call doesn't have to fake the others; `payload`
|
||||
and `entailment_payload` each take either a fixed value or a list for a
|
||||
different answer on each successive call to that schema (e.g.
|
||||
`[reject, reject, accept]` for the third-attempt-recovers case).
|
||||
"""
|
||||
|
||||
def __init__(self, payload, entailment_payload=None) -> None:
|
||||
self._payload = payload
|
||||
payloads = payload
|
||||
self._payloads = list(payloads) if isinstance(payloads, list) else [payloads]
|
||||
self._call = 0
|
||||
default = {"entailed": True, "unsupported": []}
|
||||
payloads = entailment_payload if entailment_payload is not None else default
|
||||
e_payloads = entailment_payload if entailment_payload is not None else default
|
||||
self._entailment_payloads = (
|
||||
list(payloads) if isinstance(payloads, list) else [payloads]
|
||||
list(e_payloads) if isinstance(e_payloads, list) else [e_payloads]
|
||||
)
|
||||
self._entailment_call = 0
|
||||
|
||||
@@ -93,7 +98,9 @@ class _Generator:
|
||||
payload = self._entailment_payloads[index]
|
||||
self._entailment_call += 1
|
||||
else:
|
||||
payload = self._payload
|
||||
index = min(self._call, len(self._payloads) - 1)
|
||||
payload = self._payloads[index]
|
||||
self._call += 1
|
||||
if isinstance(payload, BaseException):
|
||||
raise payload
|
||||
if isinstance(payload, str):
|
||||
@@ -127,7 +134,12 @@ def test_invented_dose_is_refused_and_never_reaches_the_answer():
|
||||
# this is a real LLM chatbot, not the retired offline-extractive build).
|
||||
assert grounded.answer is None
|
||||
assert grounded.result.decision == EvidenceDecision.ABSTAIN
|
||||
assert grounded.result.reason == "generation_unavailable"
|
||||
# The specific check that rejected it, not a generic catch-all — found
|
||||
# live 2026-08-07: every rejection reason used to collapse into
|
||||
# "generation_unavailable" by the time it reached the API response,
|
||||
# making a real provider outage indistinguishable from ordinary
|
||||
# entailment noise without reading server metrics by hand.
|
||||
assert grounded.result.reason == "ungrounded_number"
|
||||
assert metrics.total(GENERATION_REJECTED, reason="ungrounded_number") == 1
|
||||
|
||||
|
||||
@@ -217,17 +229,42 @@ def test_entailment_retries_once_after_a_reject_and_a_later_accept_serves():
|
||||
assert metrics.total(GENERATION_SERVED) == 1
|
||||
|
||||
|
||||
def test_entailment_two_agreeing_rejects_still_discard():
|
||||
def test_entailment_recovers_on_third_attempt_after_two_rejects():
|
||||
"""The improvement 2026-08-07 widened the retry from 2 to 3 attempts
|
||||
after a live 50-question adversarial sample found the 2-attempt policy's
|
||||
own math (~11% false-discard rate on a genuinely valid claim, from the
|
||||
noise probed in the docstring above) matched the observed real
|
||||
abstention rate almost exactly. Two rejects followed by a real accept
|
||||
must now be served, not discarded."""
|
||||
grounded, metrics = _answer(
|
||||
{"answer": "Metformin dùng điều trị đái tháo đường [1].",
|
||||
"evidence_sufficient": True},
|
||||
entailment_payload=[
|
||||
{"entailed": False, "unsupported": [1]},
|
||||
{"entailed": False, "unsupported": [1]},
|
||||
{"entailed": True, "unsupported": []},
|
||||
],
|
||||
)
|
||||
|
||||
assert grounded.generated is True
|
||||
assert metrics.total(GENERATION_SERVED) == 1
|
||||
assert metrics.total(GENERATION_REJECTED) == 0
|
||||
|
||||
|
||||
def test_entailment_three_agreeing_rejects_still_discard():
|
||||
grounded, metrics = _answer(
|
||||
{"answer": "Metformin chữa ung thư [1].", "evidence_sufficient": True},
|
||||
entailment_payload=[
|
||||
{"entailed": False, "unsupported": [1]},
|
||||
{"entailed": False, "unsupported": [1]},
|
||||
{"entailed": False, "unsupported": [1]},
|
||||
],
|
||||
)
|
||||
|
||||
assert grounded.generated is False
|
||||
assert grounded.answer is None
|
||||
# All 3 attempts are noisy-judge calls against the SAME claim/evidence —
|
||||
# a real, reliable rejection must still discard exactly once, not 3x.
|
||||
assert metrics.total(GENERATION_REJECTED, reason="unsupported_claim") == 1
|
||||
|
||||
|
||||
@@ -287,10 +324,46 @@ def test_every_generation_failure_abstains_instead_of_a_raw_source_dump(payload,
|
||||
assert grounded.generated is False
|
||||
assert grounded.answer is None
|
||||
assert grounded.result.decision == EvidenceDecision.ABSTAIN
|
||||
assert grounded.result.reason == "generation_unavailable"
|
||||
# The API/trace-visible reason must match the specific check that
|
||||
# failed, not a generic "generation_unavailable" for every cause —
|
||||
# otherwise a real outage and ordinary model noise are indistinguishable
|
||||
# from the outside (the exact gap a live report 2026-08-07 named).
|
||||
assert grounded.result.reason == reason
|
||||
assert metrics.total(GENERATION_REJECTED, reason=reason) == 1
|
||||
|
||||
|
||||
def test_evidence_insufficient_retries_once_and_recovers():
|
||||
"""Found live 2026-08-07 via a 50-question adversarial sample: a real
|
||||
section that plainly contains the answer (confirmed by re-asking the
|
||||
identical question 3/3 times successfully right after) still drew an
|
||||
`evidence_sufficient: false` self-judgment once — the same noisy-judge
|
||||
pattern already known for entailment, just on a different field of the
|
||||
same call. A lone insufficient verdict must not be final."""
|
||||
grounded, metrics = _answer([
|
||||
{"answer": "...", "evidence_sufficient": False},
|
||||
{"answer": "Metformin dùng điều trị đái tháo đường [1].",
|
||||
"evidence_sufficient": True},
|
||||
])
|
||||
|
||||
assert grounded.generated is True
|
||||
assert grounded.answer == "Metformin dùng điều trị đái tháo đường [1]."
|
||||
assert metrics.total(GENERATION_SERVED) == 1
|
||||
assert metrics.total(GENERATION_REJECTED) == 0
|
||||
|
||||
|
||||
def test_evidence_insufficient_twice_still_abstains():
|
||||
grounded, metrics = _answer([
|
||||
{"answer": "...", "evidence_sufficient": False},
|
||||
{"answer": "...", "evidence_sufficient": False},
|
||||
])
|
||||
|
||||
assert grounded.generated is False
|
||||
assert grounded.answer is None
|
||||
# Both attempts are the same noisy self-judgment on the same evidence —
|
||||
# a real, reliable "insufficient" must still discard exactly once.
|
||||
assert metrics.total(GENERATION_REJECTED, reason="evidence_insufficient") == 1
|
||||
|
||||
|
||||
def test_no_generator_configured_still_answers():
|
||||
service = GroundedAnswerService(_FixedRouting(_result()))
|
||||
|
||||
|
||||
@@ -18,6 +18,9 @@ ROOT = Path(__file__).resolve().parents[3]
|
||||
CHUNKS = ROOT / "ingestion/data/processed/chunks.jsonl"
|
||||
PDF = ROOT / "ingestion/data/raw/duoc-thu-quoc-gia-viet-nam-2018.pdf"
|
||||
MIGRATION = Path(__file__).resolve().parents[1] / "migrations/001_rag_retrieval_trace.sql"
|
||||
CONVERSATION_MIGRATION = (
|
||||
Path(__file__).resolve().parents[1] / "migrations/002_rag_conversation_turn.sql"
|
||||
)
|
||||
|
||||
|
||||
class _PlumbingEmbedder:
|
||||
@@ -131,6 +134,38 @@ def test_real_postgres_migration_insert_and_read_back():
|
||||
assert stored.citations[0]["printed_page_start"] == 101
|
||||
|
||||
|
||||
def test_real_postgres_conversation_store_round_trip():
|
||||
"""F-08's durable conversation history against a real Postgres, not a
|
||||
fake — proves `append`/`recent` actually persist and window correctly,
|
||||
the concrete capability this whole feature exists to add over the
|
||||
in-process dict it replaces."""
|
||||
from adapters.postgres import PostgresConversationStore
|
||||
|
||||
store = PostgresConversationStore(
|
||||
"postgresql://duoc_thu:duoc_thu@localhost:5432/duoc_thu"
|
||||
)
|
||||
store.migrate(CONVERSATION_MIGRATION)
|
||||
conversation_id = f"test-{uuid.uuid4()}"
|
||||
|
||||
store.append(conversation_id, "Người dùng: liều paracetamol")
|
||||
store.append(conversation_id, "Trợ lý: cần biết đối tượng")
|
||||
store.append(conversation_id, "Người dùng: người lớn")
|
||||
|
||||
assert store.recent(conversation_id, limit=10) == [
|
||||
"Người dùng: liều paracetamol",
|
||||
"Trợ lý: cần biết đối tượng",
|
||||
"Người dùng: người lớn",
|
||||
]
|
||||
# Windowing at read time: the oldest line falls outside a limit=2 read.
|
||||
assert store.recent(conversation_id, limit=2) == [
|
||||
"Trợ lý: cần biết đối tượng",
|
||||
"Người dùng: người lớn",
|
||||
]
|
||||
# A conversation_id that was never written to reads back empty, not an
|
||||
# error — the same "no history yet" case a fresh conversation hits live.
|
||||
assert store.recent(f"never-seen-{uuid.uuid4()}", limit=10) == []
|
||||
|
||||
|
||||
class _FakeJsonLlm:
|
||||
"""Deterministic stand-in for the Bedrock Converse generator. Satisfies
|
||||
both `JsonLlm` (query understanding) and `AnswerGenerator` (answer +
|
||||
|
||||
@@ -1,4 +1,36 @@
|
||||
from adapters.qdrant import _source_refs
|
||||
from adapters.qdrant import QdrantRetriever, _source_refs
|
||||
|
||||
|
||||
class _FakePoint:
|
||||
def __init__(self, payload: dict) -> None:
|
||||
self.payload = payload
|
||||
self.score = 1.0
|
||||
|
||||
|
||||
class _FakeScrollClient:
|
||||
"""Mimics qdrant-client's `.scroll()` shape closely enough to exercise
|
||||
`find_by_indication`'s keyword-matching logic directly — a fake filter
|
||||
(not a real one), so it returns every payload handed to it regardless
|
||||
of `scroll_filter`; the payloads given in each test already represent
|
||||
what a real `section_key=chi_dinh, chunk_kind=prose` filter would have
|
||||
returned, which is the part `find_by_indication` cannot get wrong on
|
||||
its own (the filter construction itself is a one-line, inspectable
|
||||
`Filter(must=[...])` — not worth a second fake layer to prove)."""
|
||||
|
||||
def __init__(self, payloads: list[dict]) -> None:
|
||||
self._payloads = payloads
|
||||
|
||||
def scroll(self, collection_name, scroll_filter, limit, offset, with_payload): # noqa: ARG002
|
||||
return [_FakePoint(p) for p in self._payloads], None
|
||||
|
||||
|
||||
def _chi_dinh_payload(drug_id: str, text: str) -> dict:
|
||||
return {
|
||||
"chunk_id": f"{drug_id}__chi_dinh__0", "drug_id": drug_id,
|
||||
"drug_name": drug_id.upper(), "section_key": "chi_dinh",
|
||||
"chunk_kind": "prose", "text": text,
|
||||
"heading_physical_page": 100, "printed_page_range": [101, 101],
|
||||
}
|
||||
|
||||
|
||||
def test_descriptor_source_ref_comes_from_attachment_not_heading_page():
|
||||
@@ -45,3 +77,85 @@ def test_prose_ref_uses_exact_chunk_range_and_keeps_attachment_region():
|
||||
assert refs[1].block_id == "p105_t0"
|
||||
assert refs[1].physical_page == 105
|
||||
assert refs[1].printed_page == 106
|
||||
|
||||
|
||||
def test_find_by_indication_matches_a_drug_that_names_the_symptom():
|
||||
client = _FakeScrollClient([
|
||||
_chi_dinh_payload("paracetamol_acetaminophen", "Điều trị sốt, giảm đau nhẹ và vừa."),
|
||||
_chi_dinh_payload("amoxicilin", "Điều trị nhiễm khuẩn đường hô hấp."),
|
||||
])
|
||||
retriever = QdrantRetriever(client, "duocthu_v1", embedder=None)
|
||||
|
||||
hits = retriever.find_by_indication("sốt", limit=8)
|
||||
|
||||
assert [h.document.drug_id for h in hits] == ["paracetamol_acetaminophen"]
|
||||
|
||||
|
||||
def test_find_by_indication_requires_the_whole_phrase_not_a_scattered_match():
|
||||
""""sốt xuất huyết" (dengue) must not match a chunk that only says "sốt"
|
||||
— the phrase itself has to appear, not just each of its words somewhere."""
|
||||
client = _FakeScrollClient([
|
||||
_chi_dinh_payload("paracetamol_acetaminophen", "Điều trị sốt, giảm đau."),
|
||||
])
|
||||
retriever = QdrantRetriever(client, "duocthu_v1", embedder=None)
|
||||
|
||||
hits = retriever.find_by_indication("sốt xuất huyết", limit=8)
|
||||
|
||||
assert hits == []
|
||||
|
||||
|
||||
def test_find_by_indication_matches_a_multi_word_phrase_contiguously():
|
||||
client = _FakeScrollClient([
|
||||
_chi_dinh_payload("paracetamol_acetaminophen", "Điều trị sốt cao ở người lớn."),
|
||||
])
|
||||
retriever = QdrantRetriever(client, "duocthu_v1", embedder=None)
|
||||
|
||||
hits = retriever.find_by_indication("sốt cao", limit=8)
|
||||
|
||||
assert [h.document.drug_id for h in hits] == ["paracetamol_acetaminophen"]
|
||||
|
||||
|
||||
def test_find_by_indication_rejects_a_scattered_bag_of_common_words():
|
||||
"""Found live 2026-08-07: a token-SUBSET match (every word present
|
||||
*somewhere*, any order) let a long nonsense phrase built from common
|
||||
filler words false-positive against real chi_dinh text — the words are
|
||||
common enough to appear scattered through nearly anything. Phrase
|
||||
matching closes it: none of these words are contiguous in the target
|
||||
text the way they are in the query."""
|
||||
client = _FakeScrollClient([
|
||||
_chi_dinh_payload(
|
||||
"paracetamol_acetaminophen",
|
||||
"Điều trị sốt. Không dùng quá liều khuyến cáo trong sách hướng dẫn.",
|
||||
),
|
||||
])
|
||||
retriever = QdrantRetriever(client, "duocthu_v1", embedder=None)
|
||||
|
||||
hits = retriever.find_by_indication(
|
||||
"bệnh chưa từng ghi nhận trong sách abcxyz123", limit=8
|
||||
)
|
||||
|
||||
assert hits == []
|
||||
|
||||
|
||||
def test_find_by_indication_returns_at_most_one_hit_per_drug():
|
||||
client = _FakeScrollClient([
|
||||
_chi_dinh_payload("paracetamol_acetaminophen", "Điều trị sốt."),
|
||||
{**_chi_dinh_payload("paracetamol_acetaminophen", "Điều trị sốt cao ở trẻ em."),
|
||||
"chunk_id": "paracetamol_acetaminophen__chi_dinh__1"},
|
||||
])
|
||||
retriever = QdrantRetriever(client, "duocthu_v1", embedder=None)
|
||||
|
||||
hits = retriever.find_by_indication("sốt", limit=8)
|
||||
|
||||
assert len(hits) == 1
|
||||
|
||||
|
||||
def test_find_by_indication_respects_the_limit():
|
||||
client = _FakeScrollClient([
|
||||
_chi_dinh_payload(f"drug_{i}", "Điều trị đau.") for i in range(5)
|
||||
])
|
||||
retriever = QdrantRetriever(client, "duocthu_v1", embedder=None)
|
||||
|
||||
hits = retriever.find_by_indication("đau", limit=2)
|
||||
|
||||
assert len(hits) == 2
|
||||
|
||||
@@ -1,244 +0,0 @@
|
||||
"""The loop must improve answers, and must be unable to run away.
|
||||
|
||||
Bounded is the load-bearing property: an unbounded self-improvement loop on a
|
||||
paid provider is a bill and a latency incident, and on a clinical tool it is
|
||||
also an answer nobody is waiting for any more.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from rag.conversation import ConversationState, ResolvedQuestion
|
||||
from rag.metrics import CLARIFY_ASKED, LOOP_REFINED, InMemoryMetrics
|
||||
from rag.reasoning import (
|
||||
ClarifyReason,
|
||||
DeterministicAssessor,
|
||||
LoopTrace,
|
||||
Sufficiency,
|
||||
TurnBudget,
|
||||
run_turn,
|
||||
)
|
||||
|
||||
ADULT = "Người lớn: uống 0,5 - 1 g/lần, cách 4 - 6 giờ; tối đa 4 g/ngày."
|
||||
CHILD = "Trẻ em 6 - 12 tuổi: 240 - 250 mg mỗi lần."
|
||||
|
||||
|
||||
def _q(text: str = "liều dùng paracetamol", population: str | None = None) -> ResolvedQuestion:
|
||||
return ResolvedQuestion(
|
||||
text=text,
|
||||
drug_id="paracetamol",
|
||||
section_key="lieu_luong_va_cach_dung",
|
||||
population=population,
|
||||
verbosity=None,
|
||||
inherited_drug=False,
|
||||
inherited_section=False,
|
||||
)
|
||||
|
||||
|
||||
def _state() -> ConversationState:
|
||||
return ConversationState("c1", turn_count=1)
|
||||
|
||||
|
||||
class _Retriever:
|
||||
"""Returns a different evidence set on each round, recording calls."""
|
||||
|
||||
def __init__(self, *rounds: tuple[str, ...]) -> None:
|
||||
self._rounds = list(rounds)
|
||||
self.queries: list[str] = []
|
||||
|
||||
def __call__(self, resolved: ResolvedQuestion) -> tuple[str, ...]:
|
||||
self.queries.append(resolved.text)
|
||||
if self._rounds:
|
||||
return self._rounds.pop(0)
|
||||
return ()
|
||||
|
||||
|
||||
def _generator(answer: str | None):
|
||||
calls = {"n": 0}
|
||||
|
||||
def generate(resolved, evidence, state):
|
||||
calls["n"] += 1
|
||||
return answer
|
||||
|
||||
generate.calls = calls # type: ignore[attr-defined]
|
||||
return generate
|
||||
|
||||
|
||||
# --- the loop earns its rounds ------------------------------------------------
|
||||
|
||||
|
||||
def test_a_named_gap_buys_exactly_one_more_round():
|
||||
"""Asked for adults, first round returned only paediatric text."""
|
||||
retriever = _Retriever((CHILD,), (ADULT, CHILD))
|
||||
metrics = InMemoryMetrics()
|
||||
|
||||
outcome = run_turn(
|
||||
_state(),
|
||||
_q(population="nguoi_lon"),
|
||||
retriever,
|
||||
_generator("Người lớn: 0,5 - 1 g/lần [1]"),
|
||||
metrics=metrics,
|
||||
)
|
||||
|
||||
assert outcome.retrieval_rounds_used == 2
|
||||
assert outcome.generated is True
|
||||
assert metrics.total(LOOP_REFINED, missing="population:nguoi_lon") == 1
|
||||
assert retriever.queries[1] != retriever.queries[0]
|
||||
|
||||
|
||||
def test_a_satisfied_question_spends_one_round_only():
|
||||
retriever = _Retriever((ADULT,))
|
||||
|
||||
outcome = run_turn(
|
||||
_state(), _q(population="nguoi_lon"), retriever, _generator("ok [1]")
|
||||
)
|
||||
|
||||
assert outcome.retrieval_rounds_used == 1
|
||||
assert outcome.stopped_because == "sufficient"
|
||||
|
||||
|
||||
def test_a_simple_question_does_not_loop():
|
||||
"""No population asked for means nothing to be missing."""
|
||||
retriever = _Retriever((ADULT, CHILD))
|
||||
|
||||
outcome = run_turn(_state(), _q(), retriever, _generator("ok [1]"))
|
||||
|
||||
assert outcome.retrieval_rounds_used == 1
|
||||
|
||||
|
||||
# --- the loop cannot run away -------------------------------------------------
|
||||
|
||||
|
||||
def test_retrieval_rounds_are_hard_capped():
|
||||
"""Evidence never satisfies the assessor; the loop must still stop."""
|
||||
retriever = _Retriever((CHILD,), (CHILD,), (CHILD,), (CHILD,), (CHILD,))
|
||||
|
||||
outcome = run_turn(
|
||||
_state(),
|
||||
_q(population="nguoi_lon"),
|
||||
retriever,
|
||||
_generator("ok [1]"),
|
||||
budget=TurnBudget(retrieval_rounds=2),
|
||||
)
|
||||
|
||||
assert outcome.retrieval_rounds_used == 2
|
||||
assert outcome.stopped_because == "retrieval_budget"
|
||||
assert len(retriever.queries) == 2
|
||||
|
||||
|
||||
def test_repairs_are_hard_capped_and_degrade_to_no_answer():
|
||||
"""`generate` returning None means verification refused it every time."""
|
||||
generate = _generator(None)
|
||||
|
||||
outcome = run_turn(
|
||||
_state(),
|
||||
_q(),
|
||||
_Retriever((ADULT,)),
|
||||
generate,
|
||||
budget=TurnBudget(repairs=1, llm_calls=4),
|
||||
)
|
||||
|
||||
assert outcome.answer is None
|
||||
assert outcome.repairs_used == 1
|
||||
assert generate.calls["n"] == 2 # first attempt + one repair
|
||||
assert outcome.stopped_because == "repair_budget"
|
||||
|
||||
|
||||
def test_llm_call_budget_stops_generation_entirely():
|
||||
generate = _generator(None)
|
||||
|
||||
outcome = run_turn(
|
||||
_state(), _q(), _Retriever((ADULT,)), generate, budget=TurnBudget(llm_calls=0)
|
||||
)
|
||||
|
||||
assert generate.calls["n"] == 0
|
||||
assert outcome.stopped_because == "llm_budget"
|
||||
|
||||
|
||||
def test_a_refinement_that_changes_nothing_stops_the_loop():
|
||||
"""Guards against a loop that keeps re-issuing the same query."""
|
||||
|
||||
class _SameQuery:
|
||||
def assess(self, resolved, evidence):
|
||||
return Sufficiency(False, missing="x", refined_query=resolved.text)
|
||||
|
||||
retriever = _Retriever((CHILD,), (CHILD,))
|
||||
|
||||
outcome = run_turn(
|
||||
_state(), _q(), retriever, _generator("ok [1]"), assessor=_SameQuery()
|
||||
)
|
||||
|
||||
assert outcome.stopped_because == "query_unchanged"
|
||||
assert len(retriever.queries) == 1
|
||||
|
||||
|
||||
def test_an_unnamed_gap_does_not_buy_a_round():
|
||||
""""Feels incomplete" is not a reason to spend the budget."""
|
||||
|
||||
class _Vague:
|
||||
def assess(self, resolved, evidence):
|
||||
return Sufficiency(False)
|
||||
|
||||
retriever = _Retriever((CHILD,), (CHILD,))
|
||||
|
||||
outcome = run_turn(_state(), _q(), retriever, _generator("ok [1]"), assessor=_Vague())
|
||||
|
||||
assert outcome.stopped_because == "no_actionable_gap"
|
||||
assert len(retriever.queries) == 1
|
||||
|
||||
|
||||
# --- clarify beats guessing ---------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"signal",
|
||||
[ClarifyReason.NO_ATTRIBUTE, ClarifyReason.AMBIGUOUS_DRUG, ClarifyReason.MULTI_ATTRIBUTE],
|
||||
)
|
||||
def test_a_clarify_signal_short_circuits_before_any_spend(signal):
|
||||
retriever = _Retriever((ADULT,))
|
||||
generate = _generator("ok [1]")
|
||||
metrics = InMemoryMetrics()
|
||||
budget = TurnBudget()
|
||||
|
||||
outcome = run_turn(
|
||||
_state(), _q(), retriever, generate, clarify_signals=(signal,), budget=budget, metrics=metrics
|
||||
)
|
||||
|
||||
assert outcome.clarification is not None
|
||||
assert outcome.clarification.reason == signal
|
||||
assert outcome.answer is None
|
||||
assert retriever.queries == []
|
||||
assert generate.calls["n"] == 0
|
||||
assert budget.llm_calls == 4 and budget.retrieval_rounds == 2
|
||||
assert metrics.total(CLARIFY_ASKED, reason=signal) == 1
|
||||
|
||||
|
||||
def test_no_evidence_at_all_asks_rather_than_abstaining_silently():
|
||||
outcome = run_turn(_state(), _q(), _Retriever(()), _generator("ok [1]"))
|
||||
|
||||
assert outcome.clarification is not None
|
||||
assert outcome.clarification.reason == ClarifyReason.STILL_INSUFFICIENT
|
||||
assert outcome.stopped_because == "no_evidence"
|
||||
|
||||
|
||||
# --- the deterministic assessor ----------------------------------------------
|
||||
|
||||
|
||||
def test_assessor_only_reports_gaps_it_can_demonstrate():
|
||||
assessor = DeterministicAssessor()
|
||||
|
||||
assert assessor.assess(_q(population="nguoi_lon"), (ADULT,)).sufficient is True
|
||||
assert assessor.assess(_q(population="nguoi_lon"), (CHILD,)).sufficient is False
|
||||
# No population asked for: nothing can be shown missing.
|
||||
assert assessor.assess(_q(), (CHILD,)).sufficient is True
|
||||
|
||||
|
||||
def test_trace_records_the_stages_walked():
|
||||
trace = LoopTrace()
|
||||
|
||||
run_turn(_state(), _q(), _Retriever((ADULT,)), _generator("ok [1]"), trace=trace)
|
||||
|
||||
assert trace.stages[0] == "understand"
|
||||
assert "retrieve" in trace.stages
|
||||
assert "assess" in trace.stages
|
||||
assert trace.stages[-1] == "generate"
|
||||
@@ -339,3 +339,78 @@ def test_recommendation_intent_is_refused_at_policy_boundary():
|
||||
)
|
||||
assert result.decision == EvidenceDecision.ABSTAIN
|
||||
assert result.reason == "recommendation_out_of_scope"
|
||||
|
||||
|
||||
class _IndicationRetriever:
|
||||
"""A fake exposing only `find_by_indication`/`search_indication` (the
|
||||
Qdrant adapter's shape for the reverse-lookup path), so
|
||||
`retrieve_by_indication`'s own orchestration — keyword first, dense
|
||||
fallback only when keyword finds nothing — is what's under test here,
|
||||
not the matching algorithm itself (that's `test_qdrant_adapter.py`'s job)."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
keyword_hits: list[SearchHit] | None = None,
|
||||
dense_hits: list[SearchHit] | None = None,
|
||||
) -> None:
|
||||
self._keyword_hits = keyword_hits or []
|
||||
self._dense_hits = dense_hits or []
|
||||
self.dense_called = False
|
||||
|
||||
def find_by_indication(self, indication_text, limit): # noqa: ARG002
|
||||
return self._keyword_hits
|
||||
|
||||
def search_indication(self, query, limit): # noqa: ARG002
|
||||
self.dense_called = True
|
||||
return self._dense_hits
|
||||
|
||||
|
||||
def _indication_hit(drug_id: str) -> SearchHit:
|
||||
return SearchHit(
|
||||
document=RetrievalDocument(
|
||||
doc_id=f"{drug_id}__chi_dinh__0", drug_id=drug_id, kind="prose",
|
||||
section_key="chi_dinh", text="Điều trị sốt.", source_refs=(SOURCE,),
|
||||
),
|
||||
score=1.0,
|
||||
)
|
||||
|
||||
|
||||
def test_retrieve_by_indication_uses_keyword_hits_without_trying_dense():
|
||||
retriever = _IndicationRetriever(keyword_hits=[_indication_hit("paracetamol_acetaminophen")])
|
||||
service = RetrievalService(retriever, InMemoryParentStore([]))
|
||||
|
||||
result = service.retrieve_by_indication("sốt")
|
||||
|
||||
assert result.decision == EvidenceDecision.ANSWERABLE
|
||||
assert len(result.evidence) == 1
|
||||
assert retriever.dense_called is False
|
||||
|
||||
|
||||
def test_retrieve_by_indication_falls_back_to_dense_only_when_keyword_is_empty():
|
||||
retriever = _IndicationRetriever(dense_hits=[_indication_hit("ibuprofen")])
|
||||
service = RetrievalService(retriever, InMemoryParentStore([]))
|
||||
|
||||
result = service.retrieve_by_indication("thân nhiệt tăng")
|
||||
|
||||
assert result.decision == EvidenceDecision.ANSWERABLE
|
||||
assert retriever.dense_called is True
|
||||
|
||||
|
||||
def test_retrieve_by_indication_with_no_match_anywhere_abstains():
|
||||
retriever = _IndicationRetriever()
|
||||
service = RetrievalService(retriever, InMemoryParentStore([]))
|
||||
|
||||
result = service.retrieve_by_indication("bệnh chưa từng ghi nhận")
|
||||
|
||||
assert result.decision == EvidenceDecision.ABSTAIN
|
||||
assert result.reason == "no_indication_match"
|
||||
|
||||
|
||||
def test_retrieve_by_indication_with_blank_text_abstains_without_calling_retrieval():
|
||||
retriever = _IndicationRetriever(keyword_hits=[_indication_hit("paracetamol_acetaminophen")])
|
||||
service = RetrievalService(retriever, InMemoryParentStore([]))
|
||||
|
||||
result = service.retrieve_by_indication(" ")
|
||||
|
||||
assert result.decision == EvidenceDecision.ABSTAIN
|
||||
assert result.reason == "missing_indication"
|
||||
|
||||
@@ -77,7 +77,8 @@ CONTRA = [
|
||||
]
|
||||
INDICATION = [_doc("i1", "chi_dinh", "Giảm đau, hạ sốt, chống viêm.")]
|
||||
PHARMACOLOGY = [_doc("p1", "duoc_ly_va_co_che_tac_dung", "Ức chế cyclooxygenase.")]
|
||||
ALL_DOCS = CONTRA + INDICATION + PHARMACOLOGY
|
||||
PRECAUTION = [_doc("t1", "than_trong", "Thận trọng với người suy thận.")]
|
||||
ALL_DOCS = CONTRA + INDICATION + PHARMACOLOGY + PRECAUTION
|
||||
|
||||
|
||||
def _service(retriever, resolver: SectionResolver | None) -> RetrievalService:
|
||||
@@ -207,6 +208,22 @@ class TestSectionRouting:
|
||||
assert retriever.section_calls == []
|
||||
assert retriever.search_calls
|
||||
|
||||
def test_than_trong_also_pools_chong_chi_dinh(self) -> None:
|
||||
"""A precaution some drug's own "thận trọng" text never mentions can
|
||||
still be filed under "chống chỉ định" (found live: Aspirin + loét dạ
|
||||
dày). Pooling both keeps that answerable instead of a false "not in
|
||||
this source" clarify/abstain."""
|
||||
retriever = SectionAwareRetriever(ALL_DOCS)
|
||||
result = _service(retriever, SectionResolver()).retrieve(
|
||||
"Thận trọng khi dùng aspirin là gì?", "aspirin"
|
||||
)
|
||||
assert retriever.section_calls == [
|
||||
("aspirin", "than_trong"), ("aspirin", "chong_chi_dinh"),
|
||||
]
|
||||
assert {item.evidence_id for item in result.evidence} == {
|
||||
"t1", "c1", "c2", "c3", "c4", "c5",
|
||||
}
|
||||
|
||||
def test_named_but_empty_section_falls_back(self) -> None:
|
||||
"""A drug with no such section must not abstain — similarity still tries."""
|
||||
retriever = SectionAwareRetriever(INDICATION + PHARMACOLOGY)
|
||||
|
||||
@@ -10,7 +10,13 @@ from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
from rag.understanding import SECTION_KEY_HINTS, SECTION_KEYS, LlmQueryUnderstander
|
||||
from rag.ports import AnswerGenerationUnavailable
|
||||
from rag.understanding import (
|
||||
SECTION_KEY_HINTS,
|
||||
SECTION_KEYS,
|
||||
LlmQueryUnderstander,
|
||||
QueryFrame,
|
||||
)
|
||||
|
||||
CATALOG = {
|
||||
"paracetamol_acetaminophen": "paracetamol acetaminophen, PARACETAMOL",
|
||||
@@ -23,6 +29,8 @@ class _FixedLlm:
|
||||
self._payload = payload
|
||||
|
||||
def generate(self, system: str, user: str, schema: dict) -> str:
|
||||
if isinstance(self._payload, BaseException):
|
||||
raise self._payload
|
||||
if isinstance(self._payload, str):
|
||||
return self._payload
|
||||
return json.dumps(self._payload, ensure_ascii=False)
|
||||
@@ -139,11 +147,108 @@ def test_fuzzy_suggestion_bounds_a_typo_into_the_candidate_set():
|
||||
assert frame.drugs == ("metformin",)
|
||||
|
||||
|
||||
# --- F-10: a small battery of invented near-alias shapes, beyond the single
|
||||
# "aspirinol" case above — each simulates a different way a name could be
|
||||
# crafted to *look* like it should fuzzy-match a real drug ---------------
|
||||
|
||||
|
||||
def test_a_real_drug_name_with_a_brand_like_suffix_is_not_substituted():
|
||||
resolver = _FakeResolver({}) # nothing in this turn resolves or suggests
|
||||
understander = LlmQueryUnderstander(_FixedLlm({
|
||||
"turn_type": "drug_attribute", "drugs": [],
|
||||
"unknown_drugs": ["metforminex"], "attribute": None, "population": None,
|
||||
"weight_kg": None, "age_text": None, "indication": None,
|
||||
"needs_clarify": False, "clarify_reason": None,
|
||||
}), CATALOG, resolver)
|
||||
frame = understander.understand("liều metforminex")
|
||||
assert frame.drugs == ()
|
||||
assert frame.unknown_drugs == ("metforminex",)
|
||||
|
||||
|
||||
def test_a_name_blending_two_real_drugs_is_not_substituted_for_either():
|
||||
resolver = _FakeResolver({})
|
||||
understander = LlmQueryUnderstander(_FixedLlm({
|
||||
"turn_type": "drug_attribute", "drugs": ["metformin", "paracetamol_acetaminophen"],
|
||||
"unknown_drugs": [], "attribute": None, "population": None,
|
||||
"weight_kg": None, "age_text": None, "indication": None,
|
||||
"needs_clarify": False, "clarify_reason": None,
|
||||
}), CATALOG, resolver)
|
||||
frame = understander.understand("liều metformacetamol")
|
||||
# Neither real id has deterministic candidate support for this turn —
|
||||
# F-04's bound must reject both, not accept the ones that happen to be
|
||||
# real catalog members.
|
||||
assert frame.drugs == ()
|
||||
assert "metformin" in frame.unknown_drugs
|
||||
assert "paracetamol_acetaminophen" in frame.unknown_drugs
|
||||
|
||||
|
||||
def test_malformed_json_fails_closed_to_a_clarify():
|
||||
understander = LlmQueryUnderstander(_FixedLlm("not json"), CATALOG, RESOLVER)
|
||||
frame = understander.understand("gì đó")
|
||||
assert frame.turn_type == "out_of_scope"
|
||||
assert frame.needs_clarify is True
|
||||
assert frame.quick_replies == ()
|
||||
|
||||
|
||||
def test_quick_replies_are_parsed_when_the_model_offers_them():
|
||||
understander = LlmQueryUnderstander(_FixedLlm({
|
||||
"turn_type": "dosing_calc", "drugs": ["paracetamol_acetaminophen"],
|
||||
"unknown_drugs": [], "attribute": None, "population": None,
|
||||
"weight_kg": None, "age_text": None, "indication": None,
|
||||
"needs_clarify": True, "clarify_reason": "Người lớn hay trẻ em?",
|
||||
"quick_replies": ["Người lớn", "Trẻ em"],
|
||||
}), CATALOG, RESOLVER)
|
||||
frame = understander.understand("liều paracetamol")
|
||||
assert frame.quick_replies == ("Người lớn", "Trẻ em")
|
||||
|
||||
|
||||
def test_missing_quick_replies_key_defaults_to_empty_not_a_crash():
|
||||
"""The model is asked for `quick_replies` but structured-output providers
|
||||
aren't guaranteed to include every optional key — a clarify without it
|
||||
must still parse, just with no chips."""
|
||||
understander = LlmQueryUnderstander(_FixedLlm({
|
||||
"turn_type": "dosing_calc", "drugs": [],
|
||||
"unknown_drugs": [], "attribute": None, "population": None,
|
||||
"weight_kg": None, "age_text": None, "indication": None,
|
||||
"needs_clarify": True, "clarify_reason": "Cân nặng bao nhiêu kg?",
|
||||
}), CATALOG, RESOLVER)
|
||||
frame = understander.understand("liều cho bé")
|
||||
assert frame.quick_replies == ()
|
||||
|
||||
|
||||
def test_route_is_parsed_when_the_model_resolves_it():
|
||||
"""The 2026-08-07 bug: a bare reply like 'Uống' answering the model's own
|
||||
prior route question had nowhere to be recorded (QueryFrame had no route
|
||||
field), so the model could only repeat its clarify_reason verbatim."""
|
||||
understander = LlmQueryUnderstander(_FixedLlm({
|
||||
"turn_type": "dosing_calc", "drugs": ["paracetamol_acetaminophen"],
|
||||
"unknown_drugs": [], "attribute": "lieu_luong_va_cach_dung",
|
||||
"population": "nguoi_lon", "weight_kg": None, "age_text": None,
|
||||
"indication": None, "route": "uong",
|
||||
"needs_clarify": False, "clarify_reason": None,
|
||||
}), CATALOG, RESOLVER)
|
||||
frame = understander.understand(
|
||||
"Uống",
|
||||
history=(
|
||||
"Người dùng: Liều paracetamol hạ sốt là bao nhiêu?",
|
||||
"Trợ lý: Người lớn hay trẻ em? Uống hay đặt trực tràng?",
|
||||
"Người dùng: Người lớn",
|
||||
"Trợ lý: Uống hay đặt trực tràng?",
|
||||
),
|
||||
)
|
||||
assert frame.route == "uong"
|
||||
assert frame.needs_clarify is False
|
||||
|
||||
|
||||
def test_missing_route_key_defaults_to_none_not_a_crash():
|
||||
understander = LlmQueryUnderstander(_FixedLlm({
|
||||
"turn_type": "dosing_calc", "drugs": [],
|
||||
"unknown_drugs": [], "attribute": None, "population": None,
|
||||
"weight_kg": None, "age_text": None, "indication": None,
|
||||
"needs_clarify": False, "clarify_reason": None,
|
||||
}), CATALOG, RESOLVER)
|
||||
frame = understander.understand("liều metformin")
|
||||
assert frame.route is None
|
||||
|
||||
|
||||
def test_unrecognised_turn_type_falls_back_based_on_whether_a_drug_resolved():
|
||||
@@ -200,3 +305,139 @@ def test_invalid_attribute_is_dropped_not_passed_through():
|
||||
}), CATALOG, RESOLVER)
|
||||
frame = understander.understand("metformin")
|
||||
assert frame.attribute is None
|
||||
|
||||
|
||||
# --- F-10: provider outage during the ONE call site that had no error
|
||||
# handling at all ------------------------------------------------------------
|
||||
|
||||
|
||||
# --- F-11: prior-frame merge — the code-level backstop for the model
|
||||
# dropping an already-established slot mid clarify-chain. Found live
|
||||
# 2026-08-07 (50-question hand-typed browser audit): reproduced 3 times
|
||||
# independently as either a non-terminating re-ask of the same clarify
|
||||
# question, or a stale drug bleeding into an unrelated new topic. --------
|
||||
|
||||
|
||||
def test_prior_frame_known_fields_survive_a_short_reply_the_model_drops():
|
||||
"""The Insulin/Azithromycin shape: the new call's own JSON comes back
|
||||
with the just-answered field null (a real, observed failure — the model
|
||||
is asked to restate it and sometimes doesn't), but since this turn named
|
||||
no drug of its own (a short reply like "20kg" never does), the previously
|
||||
established fields must survive via the merge, not be silently lost."""
|
||||
understander = LlmQueryUnderstander(_FixedLlm({
|
||||
"turn_type": "dosing_calc", "drugs": [],
|
||||
"unknown_drugs": [], "attribute": None, "population": None,
|
||||
"weight_kg": 20, "age_text": None, "indication": None,
|
||||
"needs_clarify": False, "clarify_reason": None,
|
||||
}), CATALOG, RESOLVER)
|
||||
prior = QueryFrame(
|
||||
turn_type="dosing_calc", drugs=("paracetamol_acetaminophen",),
|
||||
attribute="lieu_luong_va_cach_dung", needs_clarify=True,
|
||||
clarify_reason="Bé nặng bao nhiêu kg?",
|
||||
)
|
||||
frame = understander.understand("bé nặng 20 cân", prior_frame=prior)
|
||||
assert frame.drugs == ("paracetamol_acetaminophen",)
|
||||
assert frame.attribute == "lieu_luong_va_cach_dung"
|
||||
assert frame.weight_kg == 20
|
||||
|
||||
|
||||
def test_prior_frame_is_not_merged_when_the_turn_resolves_a_different_drug():
|
||||
"""The headache/OMEPRAZOL bleed this guards against: a turn that itself
|
||||
names a real, different drug is a genuine topic change and must not
|
||||
inherit the old drug's population/weight/route — merging here would
|
||||
reproduce the exact bug being fixed."""
|
||||
understander = LlmQueryUnderstander(_FixedLlm({
|
||||
"turn_type": "drug_attribute", "drugs": ["metformin"],
|
||||
"unknown_drugs": [], "attribute": "chi_dinh", "population": None,
|
||||
"weight_kg": None, "age_text": None, "indication": None,
|
||||
"needs_clarify": False, "clarify_reason": None,
|
||||
}), CATALOG, RESOLVER)
|
||||
prior = QueryFrame(
|
||||
turn_type="drug_attribute", drugs=("paracetamol_acetaminophen",),
|
||||
population="tre_em", weight_kg=20, needs_clarify=True,
|
||||
clarify_reason="Bé nặng bao nhiêu kg?",
|
||||
)
|
||||
frame = understander.understand("chỉ định của metformin là gì", prior_frame=prior)
|
||||
assert frame.drugs == ("metformin",)
|
||||
assert frame.population is None
|
||||
assert frame.weight_kg is None
|
||||
|
||||
|
||||
def test_prior_frame_that_was_already_resolved_is_not_merged():
|
||||
"""A prior turn that already answered (needs_clarify=False) has nothing
|
||||
to continue — merging it into a brand-new turn would leak stale state
|
||||
into an unrelated question that happens to follow it."""
|
||||
understander = LlmQueryUnderstander(_FixedLlm({
|
||||
"turn_type": "smalltalk", "drugs": [],
|
||||
"unknown_drugs": [], "attribute": None, "population": None,
|
||||
"weight_kg": None, "age_text": None, "indication": None,
|
||||
"needs_clarify": False, "clarify_reason": None,
|
||||
}), CATALOG, RESOLVER)
|
||||
prior = QueryFrame(
|
||||
turn_type="drug_attribute", drugs=("metformin",),
|
||||
population="nguoi_lon", needs_clarify=False,
|
||||
)
|
||||
frame = understander.understand("cảm ơn bạn", prior_frame=prior)
|
||||
assert frame.drugs == ()
|
||||
assert frame.population is None
|
||||
|
||||
|
||||
def test_known_facts_block_is_sent_to_the_model_on_a_clarify_continuation():
|
||||
captured = {}
|
||||
|
||||
class _CapturingLlm:
|
||||
def generate(self, system, user, schema):
|
||||
captured["user"] = user
|
||||
return json.dumps({
|
||||
"turn_type": "dosing_calc", "drugs": ["paracetamol_acetaminophen"],
|
||||
"unknown_drugs": [], "attribute": None, "population": None,
|
||||
"weight_kg": 20, "age_text": None, "indication": None,
|
||||
"needs_clarify": False, "clarify_reason": None,
|
||||
})
|
||||
|
||||
understander = LlmQueryUnderstander(_CapturingLlm(), CATALOG, RESOLVER)
|
||||
prior = QueryFrame(
|
||||
turn_type="dosing_calc", drugs=("paracetamol_acetaminophen",),
|
||||
population="tre_em", needs_clarify=True,
|
||||
clarify_reason="Bé nặng bao nhiêu kg?",
|
||||
)
|
||||
understander.understand("20 cân", prior_frame=prior)
|
||||
|
||||
assert "THÔNG TIN ĐÃ XÁC ĐỊNH" in captured["user"]
|
||||
assert "paracetamol_acetaminophen" in captured["user"]
|
||||
|
||||
|
||||
def test_no_known_facts_block_when_there_is_no_prior_clarify():
|
||||
captured = {}
|
||||
|
||||
class _CapturingLlm:
|
||||
def generate(self, system, user, schema):
|
||||
captured["user"] = user
|
||||
return json.dumps({
|
||||
"turn_type": "smalltalk", "drugs": [],
|
||||
"unknown_drugs": [], "attribute": None, "population": None,
|
||||
"weight_kg": None, "age_text": None, "indication": None,
|
||||
"needs_clarify": False, "clarify_reason": None,
|
||||
})
|
||||
|
||||
understander = LlmQueryUnderstander(_CapturingLlm(), CATALOG, RESOLVER)
|
||||
understander.understand("chào bạn")
|
||||
assert "THÔNG TIN ĐÃ XÁC ĐỊNH" not in captured["user"]
|
||||
|
||||
|
||||
def test_provider_outage_fails_closed_to_a_clarify_not_an_unhandled_crash():
|
||||
"""Found live 2026-08-07: unlike every other LLM call site in this
|
||||
product, `understand()` had no try/except around its call at all — a
|
||||
Bedrock outage here propagated straight through `RagAgent.handle()`
|
||||
into an unhandled 500 (`routers/rag.py` only wraps the trace-save call,
|
||||
not `agent.handle()`), instead of the graceful abstain every other
|
||||
failure mode already gets."""
|
||||
understander = LlmQueryUnderstander(
|
||||
_FixedLlm(AnswerGenerationUnavailable("Bedrock unreachable")),
|
||||
CATALOG, RESOLVER,
|
||||
)
|
||||
frame = understander.understand("liều paracetamol cho người lớn")
|
||||
assert frame.needs_clarify is True
|
||||
assert frame.clarify_reason is not None
|
||||
assert frame.drugs == ()
|
||||
assert frame.quick_replies == ()
|
||||
|
||||
Reference in New Issue
Block a user