Wire the guarded conversational RAG answer layer end-to-end

This commit is contained in:
2026-08-05 14:33:13 +07:00
parent 834d9e51b0
commit ef08b4929e
127 changed files with 37921 additions and 169 deletions
+21 -3
View File
@@ -1,5 +1,23 @@
# ai-service
Python/FastAPI. RAG orchestration: embed query -> vector search in Qdrant ->
build grounded prompt -> call OpenAI chat completion -> return answer +
citations. Stateless — does not own chat history itself.
FastAPI service for drug resolution, guarded retrieval, printed-page citations,
and PostgreSQL retrieval traces.
Local infrastructure:
```powershell
docker compose -f ..\..\infra\docker\docker-compose.yml up -d postgres qdrant
python -m migrate
uvicorn main:app --reload
```
`GET /health` is always available. `POST /v1/rag/query` requires structured
`subject_scope` and `intent`; unknown/non-human/recommendation requests fail
closed. The default `EMBEDDING_PROVIDER=disabled` intentionally leaves the RAG
backend unavailable until the collection and matching query embedder are
configured.
`EMBEDDING_PROVIDER=local-smoke` is only for local plumbing checks. Its hashing
vectors are deterministic but not semantic and must not be used for retrieval
quality claims. Bedrock is not called by this service and no IAM change is
required.
+4
View File
@@ -0,0 +1,4 @@
from .postgres import PostgresTraceRepository
from .qdrant import QdrantParentStore, QdrantRetriever
__all__ = ["PostgresTraceRepository", "QdrantParentStore", "QdrantRetriever"]
+130
View File
@@ -0,0 +1,130 @@
"""Claude on Amazon Bedrock as the answer generator.
The only module that names the `anthropic` SDK, imported lazily — the same
arrangement that confines `boto3` to `embedding.py` and `qdrant_client` to
`qdrant.py`, so `rag/` imports and the whole suite runs with no SDK and no
cloud account.
Two provider facts, taken from the Anthropic API reference rather than from
memory: Bedrock model ids carry an `anthropic.` prefix (`anthropic.claude-opus-5`),
and the Messages-API path on Bedrock is the Mantle client — not the legacy
`bedrock-runtime` InvokeModel route the embedding adapter uses.
"""
from __future__ import annotations
import json
from typing import Any
from rag.ports import AnswerGenerationUnavailable
BEDROCK_CLAUDE_OPUS_5 = "anthropic.claude-opus-5"
# Generation must not outrun the evidence it is rewriting. The section route
# can return a long section, so this is sized for the rewrite, not the source.
MAX_OUTPUT_TOKENS = 4096
def _provider_error_types() -> tuple[type[BaseException], ...]:
"""SDK and transport error classes, or none when the SDK is absent."""
collected: list[type[BaseException]] = []
try:
import anthropic
collected.append(anthropic.APIError)
except ImportError:
pass
try:
from botocore.exceptions import BotoCoreError, ClientError
collected.extend((BotoCoreError, ClientError))
except ImportError:
pass
return tuple(collected)
class BedrockClaudeAnswerGenerator:
"""Rewrites evidence into prose under a schema the API enforces.
The output shape is constrained by `output_config.format` rather than by
asking for JSON in the prompt, so a malformed envelope is the provider's
error rather than this code's parsing problem. What the schema cannot
constrain is whether the *content* is faithful — that is
`rag.grounding.verify`'s job, and it runs on every response this returns.
"""
def __init__(
self,
region: str = "us-east-1",
client: Any | None = None,
model_id: str = BEDROCK_CLAUDE_OPUS_5,
max_tokens: int = MAX_OUTPUT_TOKENS,
) -> None:
self._region = region
self._client = client
self._model_id = model_id
self._max_tokens = max_tokens
@property
def model_id(self) -> str:
return self._model_id
def _runtime(self) -> Any:
if self._client is None:
from anthropic import AnthropicBedrockMantle
self._client = AnthropicBedrockMantle(aws_region=self._region)
return self._client
def generate(self, system: str, user: str, schema: dict) -> str:
try:
response = self._runtime().messages.create(
model=self._model_id,
max_tokens=self._max_tokens,
system=system,
messages=[{"role": "user", "content": user}],
output_config={"format": {"type": "json_schema", "schema": schema}},
)
except _provider_error_types() as error:
raise AnswerGenerationUnavailable(
f"{self._model_id} could not be invoked: {type(error).__name__}"
) from error
# A refusal is a successful HTTP response with no usable content, not
# an exception. Treating it as an outage routes it to the extractive
# fallback instead of letting `content[0]` raise.
if getattr(response, "stop_reason", None) == "refusal":
raise AnswerGenerationUnavailable(
f"{self._model_id} declined the request"
)
text = "".join(
block.text
for block in response.content
if getattr(block, "type", None) == "text"
)
if not text.strip():
raise AnswerGenerationUnavailable(
f"{self._model_id} returned no text content"
)
return text
class StubAnswerGenerator:
"""Returns a fixed payload; lets the whole answer path run with no cloud.
Not a fake for tests only — it is what `EMBEDDING_PROVIDER`-style local
demos use to exercise prompt building, schema parsing, grounding
verification and the fallback branch without spending anything.
"""
def __init__(self, answer: str, evidence_sufficient: bool = True) -> None:
self._payload = json.dumps(
{"answer": answer, "evidence_sufficient": evidence_sufficient},
ensure_ascii=False,
)
self.calls: list[tuple[str, str]] = []
def generate(self, system: str, user: str, schema: dict) -> str: # noqa: ARG002
# `schema` is unused here; the stub returns an already-valid payload.
self.calls.append((system, user))
return self._payload
+178
View File
@@ -0,0 +1,178 @@
from __future__ import annotations
import hashlib
import math
from typing import Any
from rag.ports import QueryEmbeddingUnavailable
BEDROCK_RUNTIME_SERVICE = "bedrock-runtime"
COHERE_EMBED_V4 = "cohere.embed-v4:0"
# Cohere embeds corpus records and queries into different subspaces. Sending
# the corpus value here raises no error — recall just drops silently — so this
# constant exists to make the asymmetry visible rather than incidental.
COHERE_QUERY_INPUT_TYPE = "search_query"
def _provider_error_types() -> tuple[type[BaseException], ...]:
"""botocore's error classes, or none when botocore is absent.
Resolved lazily and tolerantly so a test injecting a stub client — and a
deployment that never enables a cloud provider — neither imports botocore
nor depends on it being installed.
"""
try:
from botocore.exceptions import BotoCoreError, ClientError
except ImportError:
return ()
return (BotoCoreError, ClientError)
class BedrockCohereQueryEmbedder:
"""Embeds a query with the same model the collection was built from.
A collection loaded with Cohere vectors and queried with any other embedder
still returns results and still raises nothing — the hits are simply
meaningless. That failure is silent, which is why the corpus manifest
records `model_id` and why this adapter names the model explicitly.
The request shape is duplicated from `ingestion/embed/bedrock_cohere.py`
rather than imported: `ingestion` is a separate deployable and importing it
here would couple the API to the batch pipeline. The duplication is one
JSON body and is deliberate.
"""
def __init__(
self,
dimensions: int,
region: str = "us-east-1",
client: Any | None = None,
model_id: str = COHERE_EMBED_V4,
) -> None:
if dimensions <= 0:
raise ValueError("dimensions must be positive")
self._dimensions = dimensions
self._region = region
self._client = client
self._model_id = model_id
@property
def dimensions(self) -> int:
return self._dimensions
@property
def model_id(self) -> str:
return self._model_id
def _runtime(self) -> Any:
if self._client is None:
import boto3
from botocore.config import Config
self._client = boto3.client(
BEDROCK_RUNTIME_SERVICE,
region_name=self._region,
config=Config(
connect_timeout=10,
read_timeout=30,
retries={"max_attempts": 3, "mode": "standard"},
),
)
return self._client
def embed_query(self, text: str) -> list[float]:
import json
try:
response = self._runtime().invoke_model(
modelId=self._model_id,
body=json.dumps(
{
"texts": [text],
"input_type": COHERE_QUERY_INPUT_TYPE,
"embedding_types": ["float"],
"output_dimension": self._dimensions,
}
),
accept="*/*",
contentType="application/json",
)
except _provider_error_types() as error:
raise QueryEmbeddingUnavailable(
f"{self._model_id} could not be invoked: {type(error).__name__}"
) from error
body = json.loads(response["body"].read())
embeddings = body.get("embeddings")
if isinstance(embeddings, dict):
rows = embeddings.get("float")
else:
rows = embeddings
if not rows:
raise ValueError(
f"{self._model_id} returned no float embedding; "
f"response keys were {sorted(body)}"
)
values = list(rows[0])
if len(values) != self._dimensions:
raise ValueError(
f"{self._model_id} returned {len(values)} dimensions, "
f"expected {self._dimensions}"
)
return values
class SectionOnlyQueryEmbedder:
"""Refuses to embed, confining retrieval to the route that is verified.
The section route resolves the question's attribute to a `section_key` and
filters on it; no vector is involved, and it measured 16/16 on the
human-written golden questions on 2026-08-04. Similarity measured 0.544 and
its provider is currently revoked.
Declining locally and immediately is better than the two alternatives it
replaces: a `cohere-v4` round-trip spends the boto3 retry budget before
failing, and `LocalHashQueryEmbedder` searches a SHA-256 vector against a
Cohere collection, which returns confident and meaningless hits.
"""
def __init__(self, dimensions: int) -> None:
if dimensions <= 0:
raise ValueError("dimensions must be positive")
self._dimensions = dimensions
@property
def dimensions(self) -> int:
return self._dimensions
def embed_query(self, text: str) -> list[float]: # noqa: ARG002
# The text is irrelevant: this embedder exists to refuse, not to embed.
raise QueryEmbeddingUnavailable(
"no query embedding provider is enabled; set EMBEDDING_PROVIDER to "
"use the similarity fallback"
)
class LocalHashQueryEmbedder:
"""Deterministic local plumbing probe; not a semantic retrieval model."""
def __init__(self, dimensions: int) -> None:
if dimensions <= 0:
raise ValueError("dimensions must be positive")
self._dimensions = dimensions
@property
def dimensions(self) -> int:
return self._dimensions
def embed_query(self, text: str) -> list[float]:
vector = [0.0] * self._dimensions
for token in text.casefold().split():
digest = hashlib.sha256(token.encode("utf-8")).digest()
index = int.from_bytes(digest[:4], "big") % self._dimensions
sign = 1.0 if digest[4] & 1 else -1.0
vector[index] += sign
norm = math.sqrt(sum(value * value for value in vector))
if norm == 0:
return vector
return [value / norm for value in vector]
+82
View File
@@ -0,0 +1,82 @@
from __future__ import annotations
import json
import uuid
from dataclasses import dataclass
from datetime import datetime
from pathlib import Path
from typing import Any
@dataclass(frozen=True)
class RetrievalTrace:
trace_id: str
query: str
subject_scope: str
intent: str
decision: str
reason: str
resolved_drug_id: str | None
citations: tuple[dict[str, Any], ...]
created_at: datetime | None = None
class PostgresTraceRepository:
def __init__(self, dsn: str) -> None:
self._dsn = dsn
def migrate(self, migration_path: Path) -> None:
import psycopg
statement = migration_path.read_text(encoding="utf-8")
with psycopg.connect(self._dsn) as connection:
connection.execute(statement)
def save(
self,
*,
query: str,
subject_scope: str,
intent: str,
decision: str,
reason: str,
resolved_drug_id: str | None,
citations: tuple[dict[str, Any], ...],
) -> str:
import psycopg
trace_id = str(uuid.uuid4())
with psycopg.connect(self._dsn) as connection:
connection.execute(
"""
INSERT INTO rag_retrieval_trace (
trace_id, query_text, subject_scope, query_intent,
decision, reason, resolved_drug_id, citations
) VALUES (%s, %s, %s, %s, %s, %s, %s, %s::jsonb)
""",
(
trace_id, query, subject_scope, intent, decision, reason,
resolved_drug_id, json.dumps(citations, ensure_ascii=False),
),
)
return trace_id
def get(self, trace_id: str) -> RetrievalTrace | None:
import psycopg
with psycopg.connect(self._dsn) as connection:
row = connection.execute(
"""
SELECT trace_id::text, query_text, subject_scope, query_intent,
decision, reason, resolved_drug_id, citations, created_at
FROM rag_retrieval_trace WHERE trace_id = %s
""",
(trace_id,),
).fetchone()
if row is None:
return None
return RetrievalTrace(
trace_id=row[0], query=row[1], subject_scope=row[2], intent=row[3],
decision=row[4], reason=row[5], resolved_drug_id=row[6],
citations=tuple(row[7]), created_at=row[8],
)
+73
View File
@@ -0,0 +1,73 @@
"""Exports the domain's counters; the only module that names prometheus_client.
`rag/metrics.py` defines what is counted and why. This decides how it leaves
the process, and is imported lazily so the service runs — and the suite passes
— with no metrics stack installed.
Counter names carry a `duocthu_` prefix and a `_total` suffix because that is
what Prometheus expects of a counter; the dashboard queries them by name.
"""
from __future__ import annotations
from typing import Any
from rag.metrics import (
ABSTENTION,
ANSWER_EXTRACTIVE,
GENERATION_REJECTED,
GENERATION_SERVED,
RETRIEVAL_ROUTE,
)
_LABELS: dict[str, tuple[str, ...]] = {
ABSTENTION: ("reason",),
GENERATION_REJECTED: ("reason",),
RETRIEVAL_ROUTE: ("route",),
GENERATION_SERVED: (),
ANSWER_EXTRACTIVE: (),
}
_HELP = {
ABSTENTION: "Answers refused, by the reason retrieval gave.",
GENERATION_REJECTED: (
"Generations discarded before reaching the caller. `reason=\"ungrounded_number\"` "
"counts answers that stated a figure absent from the cited source."
),
GENERATION_SERVED: "Generations that passed grounding verification and were served.",
ANSWER_EXTRACTIVE: "Answers served as verbatim source text.",
RETRIEVAL_ROUTE: "Retrievals by route: section filter, or similarity fallback.",
}
class PrometheusMetrics:
"""Domain `Metrics` backed by a Prometheus registry."""
def __init__(self, registry: Any | None = None) -> None:
from prometheus_client import CollectorRegistry, Counter
self._registry = registry or CollectorRegistry()
self._counters = {
name: Counter(name, _HELP[name], labels, registry=self._registry)
for name, labels in _LABELS.items()
}
@property
def registry(self) -> Any:
return self._registry
def increment(self, name: str, **labels: str) -> None:
counter = self._counters.get(name)
if counter is None:
return
# An unexpected label would raise at scrape time, far from its cause.
# Metrics must not be able to break a clinical answer, so a mismatch
# drops the sample rather than the request.
expected = set(_LABELS[name])
if set(labels) != expected:
return
(counter.labels(**labels) if labels else counter).inc()
def render(self) -> tuple[bytes, str]:
from prometheus_client import CONTENT_TYPE_LATEST, generate_latest
return generate_latest(self._registry), CONTENT_TYPE_LATEST
+265
View File
@@ -0,0 +1,265 @@
from __future__ import annotations
from typing import Any, Protocol, Sequence
from rag.models import ParentDocument, RetrievalDocument, SearchHit, SourceRef
class QueryEmbedder(Protocol):
@property
def dimensions(self) -> int: ...
def embed_query(self, text: str) -> Sequence[float]: ...
def _source_refs(payload: dict[str, Any]) -> tuple[SourceRef, ...]:
explicit = payload.get("source_refs") or []
if explicit:
return tuple(
SourceRef(
physical_page=int(item["physical_page"]),
precision=item.get("precision", "region"),
block_id=item.get("block_id"),
bbox=tuple(item["bbox"]) if item.get("bbox") else None,
source_crop=item.get("source_crop"),
page_range=(
tuple(item["page_range"]) if item.get("page_range") else None
),
printed_page=item.get("printed_page"),
printed_page_range=(
tuple(item["printed_page_range"])
if item.get("printed_page_range") else None
),
)
for item in explicit
)
physical_range = payload.get("source_page_range")
printed_range = payload.get("printed_page_range")
attachments = payload.get("attachments") or []
attachment_refs = tuple(
SourceRef(
physical_page=int(item["physical_page"]),
precision="region",
block_id=item.get("block_id"),
bbox=tuple(item["bbox"]) if item.get("bbox") else None,
source_crop=item.get("source_crop"),
page_range=(int(item["physical_page"]), int(item["physical_page"])),
printed_page=(
int(item["printed_page"])
if item.get("printed_page") is not None else None
),
printed_page_range=(
(int(item["printed_page"]), int(item["printed_page"]))
if item.get("printed_page") is not None else None
),
)
for item in attachments
if item.get("physical_page") is not None
)
if payload.get("chunk_kind") == "block_descriptor" and attachment_refs:
return attachment_refs
physical_page = (
physical_range[0]
if physical_range else payload.get("heading_physical_page")
)
base_refs: tuple[SourceRef, ...] = ()
if physical_page is not None:
base_refs = (
SourceRef(
physical_page=int(physical_page),
precision="chunk_page_range",
page_range=tuple(physical_range) if physical_range else None,
printed_page=(int(printed_range[0]) if printed_range else None),
printed_page_range=tuple(printed_range) if printed_range else None,
),
)
return base_refs + attachment_refs
def _document(payload: dict[str, Any]) -> RetrievalDocument:
return RetrievalDocument(
doc_id=payload["chunk_id"],
drug_id=payload["drug_id"],
drug_name=payload.get("drug_name"),
kind=payload.get("chunk_kind", "prose"),
text=payload["text"],
section_key=payload["section_key"],
source_refs=_source_refs(payload),
parent_id=payload.get("parent_id"),
requires_visual_check=(
bool(payload.get("requires_visual_check"))
or bool(payload.get("has_quarantined_content"))
),
)
class QdrantRetriever:
def __init__(
self,
client: Any,
collection_name: str,
embedder: QueryEmbedder,
) -> None:
self._client = client
self._collection_name = collection_name
self._embedder = embedder
def search(self, query: str, drug_id: str, limit: int) -> list[SearchHit]:
from qdrant_client.models import FieldCondition, Filter, MatchValue
vector = list(self._embedder.embed_query(query))
if len(vector) != self._embedder.dimensions:
raise ValueError(
f"query vector has {len(vector)} dimensions; "
f"expected {self._embedder.dimensions}"
)
points = self._client.search(
collection_name=self._collection_name,
query_vector=vector,
query_filter=Filter(
must=[FieldCondition(key="drug_id", match=MatchValue(value=drug_id))]
),
limit=limit,
with_payload=True,
)
return [
SearchHit(_document(dict(point.payload or {})), float(point.score))
for point in points
]
def find_by_section(self, drug_id: str, section_key: str) -> list[SearchHit]:
"""Every chunk of one section, by payload filter — no vector involved.
A `scroll`, not a `search`: this must not be a top-k. Paging continues
until the offset is exhausted, because Qdrant's default page is 256 and
a long section silently truncated would read as a complete answer.
Score is 1.0 because the match is exact by construction. It is not a
similarity and must not be compared against one.
Results are re-sorted by `part_index` before returning. Qdrant scrolls
in point-id order, and point ids are `uuid5(chunk_id)`, so the natural
order is effectively random: PARACETAMOL's dosing section came back
3, 4, 1, 2, 0 — the answer opened mid-sentence on paediatric doses and
buried "Liều lượng: Người lớn:" last. A section served out of order is
a clinical hazard, not a formatting one: a reader who stops early
stops in the middle of a different population's dose.
"""
from qdrant_client.models import FieldCondition, Filter, MatchValue
scroll_filter = Filter(
must=[
FieldCondition(key="drug_id", match=MatchValue(value=drug_id)),
FieldCondition(key="section_key", match=MatchValue(value=section_key)),
]
)
hits: list[tuple[dict, SearchHit]] = []
offset = None
while True:
points, offset = self._client.scroll(
collection_name=self._collection_name,
scroll_filter=scroll_filter,
limit=256,
offset=offset,
with_payload=True,
)
hits.extend(
(dict(point.payload or {}), SearchHit(_document(dict(point.payload or {})), 1.0))
for point in points
)
if offset is None:
break
# `part_index` is the chunker's own position within the section. A
# payload missing it sorts last rather than raising: an unordered
# section is worse than a scrambled one only if it also disappears.
hits.sort(key=lambda item: item[0].get("part_index", 1 << 30))
return [hit for _, hit in hits]
def find_by_drug(self, drug_id: str) -> list[SearchHit]:
"""Every prose section of one drug, in book order — the monograph view.
For a query that names the drug but no attribute ("PARACETAMOL"), a drug
reference shows the whole monograph, not a "specify an attribute" prompt.
A `scroll` filtered on `drug_id`, prose only (block descriptors stay out
of a text answer), ordered by the book's section sequence then
`part_index`. Each section's first chunk gets a `【heading】` so the
result reads as a monograph, not a wall of text.
"""
from qdrant_client.models import FieldCondition, Filter, MatchValue
from rag.sections import SECTION_ORDER
scroll_filter = Filter(
must=[
FieldCondition(key="drug_id", match=MatchValue(value=drug_id)),
FieldCondition(key="chunk_kind", match=MatchValue(value="prose")),
]
)
payloads: list[dict] = []
offset = None
while True:
points, offset = self._client.scroll(
collection_name=self._collection_name,
scroll_filter=scroll_filter,
limit=256,
offset=offset,
with_payload=True,
)
payloads.extend(dict(point.payload or {}) for point in points)
if offset is None:
break
order = {key: index for index, key in enumerate(SECTION_ORDER)}
payloads.sort(
key=lambda p: (
order.get(p.get("section_key"), len(order)),
p.get("part_index", 1 << 30),
)
)
hits: list[SearchHit] = []
seen_sections: set[str] = set()
for payload in payloads:
section_key = payload.get("section_key")
if section_key not in seen_sections:
seen_sections.add(section_key)
name = payload.get("section_display_name") or section_key or ""
payload = {**payload, "text": f"{name}\n{payload.get('text', '')}"}
hits.append(SearchHit(_document(payload), 1.0))
return hits
class QdrantParentStore:
def __init__(self, client: Any, collection_name: str) -> None:
self._client = client
self._collection_name = collection_name
def get(self, parent_id: str) -> ParentDocument | None:
from qdrant_client.models import FieldCondition, Filter, MatchValue
points, _ = self._client.scroll(
collection_name=self._collection_name,
scroll_filter=Filter(
must=[FieldCondition(key="chunk_id", match=MatchValue(value=parent_id))]
),
limit=1,
with_payload=True,
)
if not points:
return None
payload = dict(points[0].payload or {})
return ParentDocument(
parent_id=parent_id,
kind=payload.get("chunk_kind", "parent"),
text=payload["text"],
source_refs=_source_refs(payload),
requires_visual_check=(
bool(payload.get("requires_visual_check"))
or bool(payload.get("has_quarantined_content"))
),
)
+111
View File
@@ -0,0 +1,111 @@
from __future__ import annotations
from qdrant_client import QdrantClient
from adapters.embedding import (
BedrockCohereQueryEmbedder,
LocalHashQueryEmbedder,
SectionOnlyQueryEmbedder,
)
from adapters.postgres import PostgresTraceRepository
from adapters.qdrant import QdrantParentStore, QdrantRetriever
from config import Settings
from rag.answer import GroundedAnswerService
from rag.artifacts import load_aliases
from rag.conversation import DeterministicSummariser, InMemoryConversationStore
from rag.conversational import ConversationalLoopService
from rag.metrics import NullMetrics
from rag.routing import CatalogDrugResolver, QueryRoutingService
from rag.sections import SectionResolver
from rag.service import EvidencePolicy, RetrievalService
def _build_metrics(settings: Settings):
"""A Prometheus exporter, or None when the package or the flag is absent.
Missing `prometheus_client` degrades to no metrics rather than to a service
that will not start: observability is not a precondition for answering.
"""
if not settings.metrics_enabled:
return None
try:
from adapters.prometheus import PrometheusMetrics
return PrometheusMetrics()
except ImportError:
return None
def _build_generator(settings: Settings):
if settings.answer_provider == "disabled":
return None
if settings.answer_provider == "stub":
from adapters.bedrock_claude import StubAnswerGenerator
return StubAnswerGenerator(
"Câu trả lời mẫu, không gọi nhà cung cấp nào. [1]"
)
if settings.answer_provider == "bedrock-claude":
from adapters.bedrock_claude import BedrockClaudeAnswerGenerator
return BedrockClaudeAnswerGenerator(
region=settings.aws_region, model_id=settings.answer_model_id
)
raise ValueError(
"Unknown ANSWER_PROVIDER. Supported values: disabled (default), "
"stub (local, no cloud), bedrock-claude"
)
def build_runtime(settings: Settings):
metrics = _build_metrics(settings)
if settings.embedding_provider == "disabled":
return None, None, PostgresTraceRepository(settings.postgres_dsn), metrics
if settings.embedding_provider not in ("section-only", "local-smoke", "cohere-v4"):
raise ValueError(
"No production query embedder is configured. Supported values: "
"EMBEDDING_PROVIDER=section-only (default; section route only), "
"local-smoke (plumbing only) or cohere-v4"
)
client = QdrantClient(
url=settings.qdrant_url,
api_key=settings.qdrant_api_key,
timeout=30,
)
# A collection built with one model and queried with another returns hits
# and raises nothing; the results are just meaningless. Keep this in step
# with `model_id` in the collection's manifest.
if settings.embedding_provider == "cohere-v4":
embedder = BedrockCohereQueryEmbedder(
settings.embedding_dimensions, region=settings.aws_region
)
elif settings.embedding_provider == "local-smoke":
embedder = LocalHashQueryEmbedder(settings.embedding_dimensions)
else:
embedder = SectionOnlyQueryEmbedder(settings.embedding_dimensions)
section_resolver = SectionResolver()
resolver = CatalogDrugResolver(load_aliases(settings.entities_path))
retrieval = RetrievalService(
QdrantRetriever(client, settings.qdrant_collection, embedder),
QdrantParentStore(client, settings.qdrant_collection),
EvidencePolicy(minimum_score=settings.evidence_minimum_score),
section_resolver=section_resolver,
)
routing = QueryRoutingService(retrieval, resolver)
answers = GroundedAnswerService(
routing, generator=_build_generator(settings), metrics=metrics or NullMetrics()
)
# The conversational layer reuses the same resolvers and the safe answer
# engine, adding only turn understanding, follow-up inheritance and the
# clarify/refine loop around it. InMemory store for now; a Postgres-backed
# store is the persistence follow-up.
conversational = ConversationalLoopService(
answers=answers,
resolver=resolver,
section_resolver=section_resolver,
store=InMemoryConversationStore(),
summariser=DeterministicSummariser(),
metrics=metrics or NullMetrics(),
)
return answers, conversational, PostgresTraceRepository(settings.postgres_dsn), metrics
+40
View File
@@ -0,0 +1,40 @@
from __future__ import annotations
from functools import lru_cache
from pathlib import Path
from pydantic import Field
from pydantic_settings import BaseSettings, SettingsConfigDict
class Settings(BaseSettings):
model_config = SettingsConfigDict(env_file=".env", extra="ignore")
app_name: str = "vsf-duoc-thu-ai-service"
qdrant_url: str = "http://localhost:6333"
qdrant_collection: str = "duocthu_v1"
qdrant_api_key: str | None = None
postgres_dsn: str = Field(
default="postgresql://duoc_thu:duoc_thu@localhost:5432/duoc_thu",
repr=False,
)
# Defaults to the route that is measured and needs no provider. Raising it
# to `cohere-v4` enables the similarity fallback and requires live Bedrock.
embedding_provider: str = "section-only"
embedding_dimensions: int = 1024
evidence_minimum_score: float = 0.12
aws_region: str = "us-east-1"
# Generation is off unless asked for. `stub` runs the whole answer path —
# prompt, schema parsing, grounding check, fallback — with no cloud call.
answer_provider: str = "disabled"
answer_model_id: str = "anthropic.claude-opus-5"
metrics_enabled: bool = True
entities_path: Path = (
Path(__file__).resolve().parents[2]
/ "ingestion/data/verified/drug_entities.json"
)
@lru_cache
def get_settings() -> Settings:
return Settings()
+6
View File
@@ -0,0 +1,6 @@
{
"thuoc_uong_bu_nuoc_va_ien_giai": [
"oresol",
"ORS"
]
}
@@ -0,0 +1,10 @@
{"case_id":"cross-page-contrast-dose","query":"Chụp đường tiêu hóa bằng acid ioxaglic ở trẻ em có thể tích tối đa bao nhiêu?","expected_drug_id":"acid_ioxaglic","expected_id":"p132_t0","origin":"manual_adversarial"}
{"case_id":"formula-no-printed-bar","query":"Tính tốc độ truyền adenosin theo cân nặng và nồng độ dung dịch như thế nào?","expected_drug_id":"adenosin","expected_id":"p147_f16","origin":"manual_adversarial"}
{"case_id":"renal-zoster-mid-band","query":"Famciclovir điều trị zona khi độ thanh thải creatinin khoảng 35 ml/phút tra ở đâu?","expected_drug_id":"famciclovir","expected_id":"p646_t0","origin":"manual_adversarial"}
{"case_id":"renal-herpes-typo","query":"famciclovia chỉnh liều Herpes simplex nếu ClCr 20 thì xem bảng nào","expected_drug_id":"famciclovir","expected_id":"p646_t0","origin":"manual_adversarial"}
{"case_id":"cross-page-angiography","query":"Liều iobitridol cho chụp động mạch chi dưới nằm trong bảng nào?","expected_drug_id":"iobitridol","expected_id":"p825_t1","origin":"manual_adversarial"}
{"case_id":"cross-page-ercp","query":"Chụp mật tụy ngược dòng dùng iobitridol có tổng thể tích giới hạn thế nào?","expected_drug_id":"iobitridol","expected_id":"p825_t1","origin":"manual_adversarial"}
{"case_id":"spatial-dose-formula","query":"Netilmicin 2 mg/kg phải hiệu chỉnh theo Clcr của bệnh nhân bằng công thức nào?","expected_drug_id":"netilmicin","expected_id":"p1043_f11","origin":"manual_adversarial"}
{"case_id":"ors-who-composition","query":"Công thức oresol WHO UNICEF pha một lít có bao nhiêu natri clorid?","expected_drug_id":"thuoc_uong_bu_nuoc_va_ien_giai","expected_id":"p1373_t0","origin":"manual_adversarial"}
{"case_id":"ors-infant-warning","query":"Thuốc uống bù nước và điện giải công thức nào ghi không dùng cho trẻ dưới ba tháng?","expected_drug_id":"thuoc_uong_bu_nuoc_va_ien_giai","expected_id":"p1373_t0","origin":"manual_adversarial"}
{"case_id":"unsupported-veterinary","query":"Liều famciclovir điều trị cho mèo là bao nhiêu?","expected_drug_id":"famciclovir","expected_id":null,"origin":"manual_adversarial","subject_scope":"non_human"}
+55
View File
@@ -0,0 +1,55 @@
from __future__ import annotations
from typing import Any
from fastapi import FastAPI, Response
from adapters.postgres import PostgresTraceRepository
from bootstrap import build_runtime
from config import Settings, get_settings
from rag.answer import GroundedAnswerService
from routers.rag import router as rag_router
def create_app(
*,
settings: Settings | None = None,
answer_service: GroundedAnswerService | None = None,
conversational: Any | None = None,
trace_writer: PostgresTraceRepository | None = None,
metrics: Any | None = None,
) -> FastAPI:
configured = settings or get_settings()
app = FastAPI(title=configured.app_name, version="0.1.0")
app.state.answer_service = answer_service
app.state.conversational = conversational
app.state.trace_writer = trace_writer
app.state.metrics = metrics
@app.get("/health")
def health() -> dict[str, str]:
return {"status": "ok"}
@app.get("/metrics")
def prometheus_metrics() -> Response:
exporter = getattr(app.state, "metrics", None)
if exporter is None or not hasattr(exporter, "render"):
# 404 rather than an empty 200: a scrape that silently succeeds
# with no samples looks identical to a service answering nothing.
return Response(status_code=404)
body, content_type = exporter.render()
return Response(content=body, media_type=content_type)
app.include_router(rag_router)
return app
_settings = get_settings()
_answer_service, _conversational, _trace_writer, _metrics = build_runtime(_settings)
app = create_app(
settings=_settings,
answer_service=_answer_service,
conversational=_conversational,
trace_writer=_trace_writer,
metrics=_metrics,
)
+14
View File
@@ -0,0 +1,14 @@
from pathlib import Path
from adapters.postgres import PostgresTraceRepository
from config import get_settings
def main() -> None:
migration = Path(__file__).parent / "migrations/001_rag_retrieval_trace.sql"
PostgresTraceRepository(get_settings().postgres_dsn).migrate(migration)
print(f"Applied {migration.name}")
if __name__ == "__main__":
main()
@@ -0,0 +1,14 @@
CREATE TABLE IF NOT EXISTS rag_retrieval_trace (
trace_id uuid PRIMARY KEY,
query_text text NOT NULL,
subject_scope text NOT NULL,
query_intent text NOT NULL,
decision text NOT NULL,
reason text NOT NULL,
resolved_drug_id text,
citations jsonb NOT NULL DEFAULT '[]'::jsonb,
created_at timestamptz NOT NULL DEFAULT now()
);
CREATE INDEX IF NOT EXISTS rag_retrieval_trace_created_at_idx
ON rag_retrieval_trace (created_at DESC);
+28 -2
View File
@@ -1,10 +1,36 @@
[project]
name = "ai-service"
version = "0.0.0"
description = "RAG orchestration + OpenAI calls for the Duoc Thu medical chatbot"
description = "RAG orchestration for the Duoc Thu medical chatbot"
requires-python = ">=3.11"
dependencies = []
dependencies = [
"fastapi>=0.115,<1",
"httpx>=0.27,<1",
"psycopg[binary]>=3.2,<4",
"pydantic-settings>=2.6,<3",
"qdrant-client>=1.7,<2",
"uvicorn[standard]>=0.30,<1",
]
[project.optional-dependencies]
test = ["pytest>=7.4,<9"]
# Both optional on purpose: the service answers without a metrics stack, and
# without a cloud generator. Neither is a precondition for a grounded answer.
metrics = ["prometheus-client>=0.20,<1"]
generation = ["anthropic>=0.112,<1"]
[build-system]
requires = ["setuptools>=68"]
build-backend = "setuptools.build_meta"
[tool.ruff]
line-length = 100
[tool.ruff.lint]
select = ["F", "E9", "B", "ARG"]
[tool.ruff.lint.per-file-ignores]
# Test doubles implement the domain's Protocols. Conformance requires the full
# signature even where a double ignores an argument, so ARG here would push
# tests toward fakes that no longer match the interface they stand in for.
"tests/*" = ["ARG001", "ARG002"]
+4
View File
@@ -0,0 +1,4 @@
from .models import EvidenceDecision, RetrievalResult
from .service import EvidencePolicy, RetrievalService
__all__ = ["EvidenceDecision", "EvidencePolicy", "RetrievalResult", "RetrievalService"]
+173
View File
@@ -0,0 +1,173 @@
from __future__ import annotations
import json
from dataclasses import dataclass, replace
from . import grounding, metrics as metric_names
from .metrics import Metrics, NullMetrics
from .models import EvidenceDecision, QueryIntent, RetrievalResult, SubjectScope
from .ports import AnswerGenerationUnavailable, AnswerGenerator
from .prompt import build_request
from .routing import QueryRoutingService
@dataclass(frozen=True)
class Citation:
chunk_id: str
printed_page_start: int
printed_page_end: int
physical_page: int
block_id: str | None = None
bbox: tuple[float, float, float, float] | None = None
source_crop: str | None = None
attachment: str | None = None
@dataclass(frozen=True)
class GroundedAnswer:
result: RetrievalResult
answer: str | None
citations: tuple[Citation, ...] = ()
generated: bool = False
class GroundedAnswerService:
"""Retrieval decides what is true; generation only decides how it reads.
When a generator is configured, its output replaces the extractive text
**only** if `grounding.verify` confirms every figure and citation in it
traces back to the retrieved evidence. Anything else — an unsupported
number, a citation to nothing, a provider outage, malformed output — falls
back to quoting the source verbatim, which is always available because it
was computed first.
"""
def __init__(
self,
routing: QueryRoutingService,
generator: AnswerGenerator | None = None,
metrics: Metrics | None = None,
) -> None:
self._routing = routing
self._generator = generator
self._metrics = metrics or NullMetrics()
def answer(
self,
query: str,
subject_scope: SubjectScope,
intent: QueryIntent,
) -> GroundedAnswer:
result = self._routing.retrieve(query, subject_scope, intent)
if result.decision == EvidenceDecision.ABSTAIN:
self._metrics.increment(metric_names.ABSTENTION, reason=result.reason)
return GroundedAnswer(result, None)
citations = self._citations(result)
if citations is None:
return GroundedAnswer(
replace(
result,
decision=EvidenceDecision.ABSTAIN,
reason="missing_printed_page_provenance",
evidence=(),
),
None,
)
if result.decision == EvidenceDecision.VERIFY_PDF:
# Never generated over. A quarantined table or formula is exactly
# the evidence whose numbers were not reliably reconstructed, so
# rephrasing it is the one case where fluency could invent a dose.
return GroundedAnswer(
result,
"Nguồn có bảng hoặc công thức cần đối chiếu trực tiếp với ảnh PDF; "
"không tự động trích số liệu.",
citations,
)
evidence_texts = tuple(item.text for item in result.evidence)
extractive = "\n\n".join(
f"{text} [{index}]" for index, text in enumerate(evidence_texts, start=1)
)
generated = self._generate(query, evidence_texts)
if generated is None:
self._metrics.increment(metric_names.ANSWER_EXTRACTIVE)
return GroundedAnswer(result, extractive, citations)
self._metrics.increment(metric_names.GENERATION_SERVED)
return GroundedAnswer(result, generated, citations, generated=True)
def _generate(self, query: str, evidence_texts: tuple[str, ...]) -> str | None:
"""A verified generation, or None to fall back to the source text."""
if self._generator is None or not evidence_texts:
return None
request = build_request(query, evidence_texts)
try:
raw = self._generator.generate(request.system, request.user, request.schema)
except AnswerGenerationUnavailable:
self._metrics.increment(
metric_names.GENERATION_REJECTED, reason="provider_unavailable"
)
return None
try:
payload = json.loads(raw)
answer = payload["answer"]
sufficient = payload["evidence_sufficient"]
except (ValueError, TypeError, KeyError):
self._metrics.increment(
metric_names.GENERATION_REJECTED, reason="malformed_output"
)
return None
if not isinstance(answer, str) or not isinstance(sufficient, bool):
self._metrics.increment(
metric_names.GENERATION_REJECTED, reason="malformed_output"
)
return None
if not sufficient:
# The model says the evidence does not answer the question. Showing
# the retrieved section verbatim lets the clinician judge that.
self._metrics.increment(
metric_names.GENERATION_REJECTED, reason="evidence_insufficient"
)
return None
report = grounding.verify(answer, evidence_texts)
if not report.grounded:
self._metrics.increment(
metric_names.GENERATION_REJECTED, reason=report.reason
)
return None
return answer
@staticmethod
def _citations(result: RetrievalResult) -> tuple[Citation, ...] | None:
citations = []
for evidence in result.evidence:
if not evidence.source_refs:
return None
for source in evidence.source_refs:
printed_range = source.printed_page_range
if printed_range is not None:
start, end = printed_range
elif source.printed_page is not None:
start = end = source.printed_page
else:
return None
citations.append(Citation(
chunk_id=evidence.matched_doc_id,
printed_page_start=int(start),
printed_page_end=int(end),
physical_page=source.physical_page,
block_id=source.block_id,
bbox=source.bbox,
source_crop=source.source_crop,
# Backward-compatible compact attachment identifier. A
# real crop path wins; otherwise the block id plus the
# structured page/bbox fields is enough to render later.
attachment=source.source_crop or source.block_id,
))
return tuple(citations)
+86
View File
@@ -0,0 +1,86 @@
from __future__ import annotations
import json
from pathlib import Path
from .models import ParentDocument, RetrievalDocument, SourceRef
def _read_jsonl(path: Path) -> list[dict]:
with path.open(encoding="utf-8") as handle:
return [json.loads(line) for line in handle if line.strip()]
def _source_ref(raw: dict) -> SourceRef:
bbox = raw.get("bbox")
page_range = raw.get("page_range")
printed_page_range = raw.get("printed_page_range")
return SourceRef(
physical_page=int(raw["physical_page"]),
precision=raw["precision"],
block_id=raw.get("block_id"),
bbox=tuple(bbox) if bbox else None,
source_crop=raw.get("source_crop"),
page_range=tuple(page_range) if page_range else None,
printed_page=raw.get("printed_page"),
printed_page_range=(
tuple(printed_page_range) if printed_page_range else None
),
)
def load_documents(path: Path) -> list[RetrievalDocument]:
documents = []
for raw in _read_jsonl(path):
documents.append(RetrievalDocument(
doc_id=raw["doc_id"],
drug_id=raw["drug_id"],
kind=raw["kind"],
text=raw["text"],
section_key=raw["section_key"],
source_refs=tuple(_source_ref(item) for item in raw.get("source_refs", [])),
parent_id=raw.get("parent_id"),
requires_visual_check=raw.get("requires_visual_check", False),
drug_name=raw.get("drug_name"),
))
return documents
def load_parents(path: Path) -> list[ParentDocument]:
parents = []
for raw in _read_jsonl(path):
parents.append(ParentDocument(
parent_id=raw["logical_table_id"],
kind=raw["kind"],
text=raw["markdown"],
source_refs=tuple(_source_ref(item) for item in raw.get("source_refs", [])),
requires_visual_check=raw.get("requires_visual_check", False),
))
return parents
def load_aliases(path: Path | None) -> dict[str, set[str]]:
if path is None:
return {}
raw = json.loads(path.read_text(encoding="utf-8"))
if isinstance(raw, dict) and "entities" in raw:
return {
entity["drug_id"]: set(entity["aliases"])
for entity in raw["entities"]
}
return {drug_id: set(aliases) for drug_id, aliases in raw.items()}
def build_drug_catalog(
documents: list[RetrievalDocument],
extra_aliases: dict[str, set[str]] | None = None,
) -> dict[str, set[str]]:
catalog: dict[str, set[str]] = {}
for document in documents:
aliases = catalog.setdefault(document.drug_id, set())
aliases.add(document.drug_id.replace("_", " "))
if document.drug_name:
aliases.add(document.drug_name)
for drug_id, aliases in (extra_aliases or {}).items():
catalog.setdefault(drug_id, set()).update(aliases)
return catalog
+24
View File
@@ -0,0 +1,24 @@
"""Deterministic clinical calculators.
Audit §7: a dose calculation or unit conversion must be a tested function, never
an LLM. Body surface area replaces Appendix 1's lookup table (Dược thư 2018,
printed page 1499) with the book's own DuBois formula, so a BSA-based dose is
*computed and traceable*, not read off a quarantined table crop.
"""
from __future__ import annotations
# Dược thư 2018, Phụ lục 1 (printed 1499), DuBois & DuBois (Arch Intern Med
# 1916;17:863-71): S(cm²) = W^0.425 × H^0.725 × 71.84, W in kg, H in cm.
_DUBOIS_COEFFICIENT = 71.84
def body_surface_area_m2(weight_kg: float, height_cm: float) -> float:
"""Body surface area in m² by the DuBois formula the formulary prints.
Raises ValueError on a non-positive input: a BSA from a zero or negative
weight/height is a data error, not a number to return silently.
"""
if weight_kg <= 0 or height_cm <= 0:
raise ValueError("weight_kg and height_cm must be positive")
area_cm2 = (weight_kg ** 0.425) * (height_cm ** 0.725) * _DUBOIS_COEFFICIENT
return area_cm2 / 10_000
+347
View File
@@ -0,0 +1,347 @@
"""Conversation state, and the rules for carrying context across turns.
Pure domain. Everything here works without an LLM, which is deliberate: the
part of "understanding a follow-up" that matters clinically — *which drug is
this still about* — must be deterministic and testable, not inferred.
Two structures with different jobs:
`Focus` is structured and drives routing. It is what makes "còn trẻ em thì
sao?" resolvable at all.
`summary` is prose for the generator. It records **what was discussed**, never
clinical content: a dose restated from a summary carries no citation and could
not be grounding-verified, because that check compares against retrieved
evidence and a summary is not evidence.
"""
from __future__ import annotations
from dataclasses import dataclass, field, replace
from typing import Literal, Protocol
# A drug named six turns ago is not context, it is a hazard: conversations
# drift, and inheriting a stale drug produces a confident answer about the
# wrong medicine.
FOCUS_TTL_TURNS = 6
# Three exchanges kept verbatim; older turns are folded into the summary.
RECENT_TURNS = 6
Role = Literal["user", "assistant"]
Verbosity = Literal["concise", "detailed"]
@dataclass(frozen=True)
class Turn:
role: Role
text: str
at: str
drug_id: str | None = None
section_key: str | None = None
# Storing what answered a turn is what lets the planner reuse evidence
# instead of retrieving the same section again.
evidence_ids: tuple[str, ...] = ()
@dataclass(frozen=True)
class Focus:
"""The entities a follow-up may inherit, each with the turn that set it."""
drug_id: str | None = None
drug_name: str | None = None
section_key: str | None = None
population: str | None = None
verbosity: Verbosity | None = None
set_at_turn: dict[str, int] = field(default_factory=dict)
def age_of(self, name: str, turn_count: int) -> int | None:
set_at = self.set_at_turn.get(name)
return None if set_at is None else turn_count - set_at
def is_fresh(self, name: str, turn_count: int, ttl: int = FOCUS_TTL_TURNS) -> bool:
age = self.age_of(name, turn_count)
return age is not None and age <= ttl
def with_field(self, name: str, value, turn: int) -> "Focus":
stamps = dict(self.set_at_turn)
stamps[name] = turn
return replace(self, **{name: value}, set_at_turn=stamps)
def expire(self, turn_count: int, ttl: int = FOCUS_TTL_TURNS) -> "Focus":
"""Drops every field older than the TTL, stamps included."""
kept = {
name: getattr(self, name)
for name in ("drug_id", "drug_name", "section_key", "population", "verbosity")
if self.is_fresh(name, turn_count, ttl)
}
stamps = {
name: at for name, at in self.set_at_turn.items() if name in kept
}
return Focus(**kept, set_at_turn=stamps)
@dataclass(frozen=True)
class ConversationState:
conversation_id: str
recent: tuple[Turn, ...] = ()
summary: str = ""
focus: Focus = field(default_factory=Focus)
turn_count: int = 0
def append(self, turn: Turn, window: int = RECENT_TURNS) -> "ConversationState":
"""Adds a turn and evicts the oldest beyond the window.
Eviction returns the dropped turns to the caller's summariser via
`overflow`, rather than discarding them here — this type does not
decide what a summary says.
"""
recent = (*self.recent, turn)[-window:]
return replace(
self,
recent=recent,
turn_count=self.turn_count + 1,
)
def overflow(self, window: int = RECENT_TURNS) -> tuple[Turn, ...]:
return self.recent[:-window] if len(self.recent) > window else ()
def inherited(self, name: str):
"""A focus value only if it is still fresh; otherwise None."""
return getattr(self.focus, name) if self.focus.is_fresh(name, self.turn_count) else None
# --- follow-up resolution -----------------------------------------------------
# Phrases that mean "same question, different population". Longest-first for the
# same reason `sections.py` sorts that way: "phụ nữ cho con bú" must be tested
# before "phụ nữ", or the more specific reading is never reached.
POPULATION_PHRASES: dict[str, str] = {
"phụ nữ cho con bú": "phu_nu_cho_con_bu",
"người cao tuổi": "nguoi_cao_tuoi",
"phụ nữ có thai": "phu_nu_co_thai",
"người suy thận": "suy_than",
"người suy gan": "suy_gan",
"trẻ sơ sinh": "tre_so_sinh",
"người lớn": "nguoi_lon",
"bà bầu": "phu_nu_co_thai",
"trẻ nhỏ": "tre_em",
"trẻ em": "tre_em",
"người già": "nguoi_cao_tuoi",
}
VERBOSITY_PHRASES: dict[str, Verbosity] = {
"giải thích kỹ hơn": "detailed",
"nói rõ hơn": "detailed",
"chi tiết hơn": "detailed",
"ngắn gọn": "concise",
"tóm tắt": "concise",
}
# A turn that is only a qualifier — no drug, no attribute — is a follow-up by
# construction. These are the openers that mark one.
FOLLOWUP_MARKERS = ("còn", "thế còn", "vậy còn", "so với", "thuốc vừa", "cái đó", "")
# Greetings, thanks, farewells and bare acknowledgements. A turn made up only of
# these is social, not a failed drug lookup: answering "Chưa xác định được
# thuốc" to "chào bạn" reads as broken. Longest-first so "cảm ơn nhiều" is
# stripped before "cảm ơn".
SMALLTALK_PHRASES = (
"xin chào", "chào bạn", "chào ad", "cảm ơn nhiều", "cảm ơn bạn", "cám ơn",
"cảm ơn", "tạm biệt", "hay quá", "tuyệt vời", "hiểu rồi", "được rồi",
"chào", "hello", "hi", "alo", "thanks", "thank", "ok", "oke", "okie",
"", "uh", "haha", "hihi", "bye",
)
def is_smalltalk(text: str) -> bool:
"""True when a turn carries nothing but social phrases.
Deliberately conservative: it strips every known social phrase and returns
True only if what remains is empty. "chào bạn, liều paracetamol?" keeps
"liều paracetamol" after stripping, so it is treated as a real question —
a greeting must never swallow the medical part of a turn.
"""
remainder = _normalise(text).strip(" .,!?;:")
for phrase in sorted(SMALLTALK_PHRASES, key=len, reverse=True):
# Space-pad both sides so a short phrase ("hi", "ok") matches a whole
# word only, never a substring of "chi" or "block".
remainder = f" {remainder} ".replace(f" {phrase} ", " ").strip(" .,!?;:")
return not remainder
def _normalise(text: str) -> str:
return " ".join(text.casefold().split())
def _longest_first(phrases: dict[str, str]) -> list[tuple[str, str]]:
return sorted(phrases.items(), key=lambda item: -len(item[0]))
def detect_population(text: str) -> str | None:
normalised = _normalise(text)
for phrase, tag in _longest_first(POPULATION_PHRASES):
if phrase in normalised:
return tag
return None
def detect_verbosity(text: str) -> Verbosity | None:
normalised = _normalise(text)
for phrase, level in _longest_first(VERBOSITY_PHRASES):
if phrase in normalised:
return level
return None
def looks_like_followup(text: str) -> bool:
normalised = _normalise(text)
return any(normalised.startswith(marker) for marker in FOLLOWUP_MARKERS)
@dataclass(frozen=True)
class ResolvedQuestion:
"""What this turn is asking, after the conversation is taken into account."""
text: str
drug_id: str | None
section_key: str | None
population: str | None
verbosity: Verbosity | None
inherited_drug: bool
inherited_section: bool
@property
def needs_carry_over_notice(self) -> bool:
"""Whether the answer must name what it inherited.
An inherited drug that is wrong is a wrong-drug answer, so the answer
has to say which drug it decided this was about.
"""
return self.inherited_drug
def resolve_against(
state: ConversationState,
text: str,
drug_id: str | None,
section_key: str | None,
) -> ResolvedQuestion:
"""Fills gaps in this turn from conversation focus, freshness permitting.
`drug_id` and `section_key` are what this turn resolved on its own — the
existing resolvers decide those, unchanged. Only what the turn left blank
is inherited, so an explicit mention always wins over context.
"""
inherited_drug = False
inherited_section = False
if drug_id is None:
carried = state.inherited("drug_id")
if carried is not None:
drug_id, inherited_drug = carried, True
if section_key is None:
carried = state.inherited("section_key")
if carried is not None:
section_key, inherited_section = carried, True
population = detect_population(text) or state.inherited("population")
verbosity = detect_verbosity(text) or state.inherited("verbosity")
return ResolvedQuestion(
text=text,
drug_id=drug_id,
section_key=section_key,
population=population,
verbosity=verbosity,
inherited_drug=inherited_drug,
inherited_section=inherited_section,
)
def update_focus(
state: ConversationState,
resolved: ResolvedQuestion,
) -> Focus:
"""Focus after this turn, stamped with the current turn index."""
focus = state.focus.expire(state.turn_count)
turn = state.turn_count
for name, value in (
("drug_id", resolved.drug_id),
("section_key", resolved.section_key),
("population", resolved.population),
("verbosity", resolved.verbosity),
):
if value is not None:
focus = focus.with_field(name, value, turn)
return focus
# --- persistence and summary --------------------------------------------------
#
# Protocol + no-LLM default co-located, matching how `reasoning.py` ships
# `SufficiencyAssessor`/`DeterministicAssessor` and `metrics.py` ships
# `Metrics`/`NullMetrics`. The Postgres-backed store lives in `adapters/`.
class ConversationStore(Protocol):
"""Loads and persists one conversation's state.
`load` returns a fresh empty state for an unknown id rather than raising: a
first turn has no prior state, and that is not an error.
"""
def load(self, conversation_id: str) -> "ConversationState": ...
def save(self, state: "ConversationState") -> None: ...
class InMemoryConversationStore:
"""Reference implementation and the offline/test default."""
def __init__(self) -> None:
self._states: dict[str, ConversationState] = {}
def load(self, conversation_id: str) -> ConversationState:
return self._states.get(conversation_id, ConversationState(conversation_id))
def save(self, state: ConversationState) -> None:
self._states[state.conversation_id] = state
class Summariser(Protocol):
"""Folds turns evicted from the recent window into rolling prose.
Contract, load-bearing for safety: the summary records *what was discussed*,
never a clinical value. A dose copied into a summary carries no citation and
cannot be grounding-verified — the check compares against retrieved
evidence, and a summary is not evidence.
"""
def fold(self, prev_summary: str, dropped: tuple["Turn", ...]) -> str: ...
class DeterministicSummariser:
"""No-LLM default: one topic line per evicted user turn, capped.
Records only the drug and section a turn was *about* — labels, never cell
values — so the no-clinical-content rule holds by construction rather than
by trusting a generator not to leak a dose.
"""
MAX_CHARS = 1600 # ~400 tokens, per ADR 0007 §2
def fold(self, prev_summary: str, dropped: tuple[Turn, ...]) -> str:
lines = [prev_summary] if prev_summary else []
for turn in dropped:
if turn.role != "user":
continue
drug = turn.drug_id or "thuốc chưa xác định"
section = turn.section_key or "thông tin chung"
lines.append(f"- đã hỏi {section} của {drug}")
text = "\n".join(lines)
# Keep the most recent topics when over budget: drop oldest lines, not
# mid-line characters, so the summary never ends on a fragment.
while len(text) > self.MAX_CHARS and len(lines) > 1:
lines.pop(0)
text = "\n".join(lines)
return text
+381
View File
@@ -0,0 +1,381 @@
"""Orchestration: turns a stateless single-turn engine into a conversation.
This is the glue ADR 0007 specified and nothing yet called. It owns no rules of
its own — inheritance lives in `conversation.py`, the bounded loop in
`reasoning.py`, grounding in `grounding.py`. Its whole job is the sequence:
load state
→ resolve this turn, then inherit gaps from focus
→ derive clarify signals from resolver state (never a model score)
→ run the bounded loop (retrieve / generate / verify)
→ update focus, append turns, summarise overflow, save
→ name any inherited drug in the answer
Everything here runs with no LLM and no live service: the collaborators are
protocols, so a turn can be exercised end-to-end with fakes.
"""
from __future__ import annotations
from dataclasses import dataclass, replace
from typing import Protocol
from . import metrics as metric_names
from .answer import GroundedAnswer, GroundedAnswerService
from .conversation import (
ConversationState,
ConversationStore,
Summariser,
Turn,
is_smalltalk,
resolve_against,
update_focus,
)
from .metrics import Metrics, NullMetrics
from .models import EvidenceDecision, QueryIntent, SubjectScope
from .reasoning import (
BudgetExhausted,
Clarification,
ClarifyReason,
DeterministicAssessor,
Generate,
LoopOutcome,
MAX_RETRIEVAL_ROUNDS,
Retrieve,
SufficiencyAssessor,
TurnBudget,
clarify_for,
run_turn,
)
from .routing import CatalogDrugResolver, DrugResolutionStatus
from .sections import SECTION_PHRASES, SectionResolver
SUMMARY_EVERY = 4 # regenerate the summary at most every S turns, per ADR 0007 §2
@dataclass(frozen=True)
class TurnResolution:
"""What one turn resolved on its own, before conversation is considered.
`drug_status` is the resolver's verdict — resolved / not_found / ambiguous —
kept distinct from `drug_id` so an ambiguous turn (asks which drug) reads
differently from a bare follow-up (inherits the drug).
"""
drug_id: str | None
section_key: str | None
drug_status: str
class TurnResolverPort(Protocol):
def resolve_turn(self, text: str) -> TurnResolution: ...
@dataclass(frozen=True)
class TurnResponse:
answer: str | None
clarification: Clarification | None
evidence_texts: tuple[str, ...]
stopped_because: str
inherited_drug: str | None
generated: bool
class ConversationalRagService:
def __init__(
self,
store: ConversationStore,
summariser: Summariser,
resolver: TurnResolverPort,
retrieve: Retrieve,
generate: Generate,
metrics: Metrics | None = None,
summary_every: int = SUMMARY_EVERY,
) -> None:
self._store = store
self._summariser = summariser
self._resolver = resolver
self._retrieve = retrieve
self._generate = generate
self._metrics = metrics or NullMetrics()
self._summary_every = summary_every
def answer(
self, conversation_id: str, text: str, budget: TurnBudget | None = None
) -> TurnResponse:
state = self._store.load(conversation_id)
turn = self._resolver.resolve_turn(text)
resolved = resolve_against(state, text, turn.drug_id, turn.section_key)
signals = self._clarify_signals(resolved, turn)
if resolved.inherited_drug:
self._metrics.increment(metric_names.FOLLOWUP_INHERITED)
outcome = run_turn(
state,
resolved,
self._retrieve,
self._generate,
clarify_signals=signals,
budget=budget or TurnBudget(),
metrics=self._metrics,
)
self._persist(state, resolved, outcome)
answer = outcome.answer
inherited = resolved.drug_id if resolved.needs_carry_over_notice else None
if answer is not None and inherited is not None:
# An inherited drug that is wrong is a wrong-drug answer, so the
# answer has to say which drug it decided this was about.
answer = f"Về {inherited}: {answer}"
return TurnResponse(
answer=answer,
clarification=outcome.clarification,
evidence_texts=outcome.evidence_texts,
stopped_because=outcome.stopped_because,
inherited_drug=inherited,
generated=outcome.generated,
)
@staticmethod
def _clarify_signals(resolved, turn: TurnResolution) -> tuple[str, ...]:
"""Resolver states that should ask instead of guess.
Only fires when the drug is *still* unknown after inheritance: a
follow-up like "còn trẻ em thì sao?" names no drug but inherits one, and
must not be turned into a clarify.
"""
if resolved.drug_id is None:
return (ClarifyReason.AMBIGUOUS_DRUG,)
return ()
def _persist(
self, state: ConversationState, resolved, outcome: LoopOutcome
) -> None:
focus = update_focus(state, resolved)
state = ConversationState(
conversation_id=state.conversation_id,
recent=state.recent,
summary=state.summary,
focus=focus,
turn_count=state.turn_count,
)
state = state.append(
Turn("user", resolved.text, _now(), resolved.drug_id, resolved.section_key)
)
if outcome.answer is not None:
state = state.append(
Turn(
"assistant",
outcome.answer,
_now(),
resolved.drug_id,
resolved.section_key,
evidence_ids=tuple(str(i) for i in range(len(outcome.evidence_texts))),
)
)
if state.turn_count % self._summary_every == 0 and state.overflow():
summary = self._summariser.fold(state.summary, state.overflow())
state = ConversationState(
conversation_id=state.conversation_id,
recent=state.recent,
summary=summary,
focus=state.focus,
turn_count=state.turn_count,
)
self._store.save(state)
def _now() -> str:
# Timestamps are provenance, not logic; the domain never branches on them,
# so a monotonic placeholder keeps this module free of wall-clock coupling.
return ""
# --- live chat core -----------------------------------------------------------
#
# The deployable multi-turn path. The loop is what *understands and clarifies*
# a turn; retrieval, citation, VERIFY_PDF and grounding stay inside
# GroundedAnswerService, untouched — so clarify + refine are added *around* the
# safe engine, never inside it.
SMALLTALK_REPLY = (
"Mình tra cứu Dược thư Quốc gia Việt Nam. Bạn muốn hỏi về thuốc nào, "
"hoặc thuộc tính nào (liều dùng, chống chỉ định, tương tác…)?"
)
@dataclass(frozen=True)
class ConversationTurnResult:
answer: str | None
clarification: Clarification | None
grounded: GroundedAnswer | None
smalltalk: bool
inherited_drug: str | None
reason: str
class ConversationalLoopService:
def __init__(
self,
answers: GroundedAnswerService,
resolver: CatalogDrugResolver,
section_resolver: SectionResolver,
store: ConversationStore,
assessor: SufficiencyAssessor | None = None,
summariser: Summariser | None = None,
metrics: Metrics | None = None,
) -> None:
self._answers = answers
self._resolver = resolver
self._section_resolver = section_resolver
self._store = store
self._assessor = assessor or DeterministicAssessor()
self._summariser = summariser
self._metrics = metrics or NullMetrics()
def answer(
self,
conversation_id: str,
query: str,
subject_scope: SubjectScope,
intent: QueryIntent,
budget: TurnBudget | None = None,
) -> ConversationTurnResult:
state = self._store.load(conversation_id)
resolution = self._resolver.resolve(query)
# Only an EXACT name is auto-accepted. A fuzzy match (score < 1.0) is a
# guess, and a formulary must not silently answer about a *different*
# drug than the one meant — a typo is asked about ("did you mean…?"),
# never resolved on a similarity threshold. Autocomplete at input is the
# first line; this is the backstop when a wrong name is still submitted.
is_exact = resolution.status == DrugResolutionStatus.RESOLVED and (
resolution.score is None or resolution.score >= 0.999
)
drug_self = resolution.drug_id if is_exact else None
# Social turn that names no drug: answer as a person, not a failed lookup.
if drug_self is None and is_smalltalk(query):
self._append_user(state, query, None, None)
return ConversationTurnResult(
SMALLTALK_REPLY, None, None, True, None, "smalltalk"
)
section = self._section_resolver.resolve(query)
section_self = section.section_key if section else None
resolved = resolve_against(state, query, drug_self, section_self)
# Clarify beats guessing: no drug even after inheritance. If the text is
# a near-miss for real drug names, offer them ("did you mean") rather
# than a bare "which drug?" — a typo should not dead-end.
if resolved.drug_id is None:
# Only genuinely-close names are offered. A far match (Arginin for
# "metfomin") is noise, not a suggestion — so the bar is high, and
# when nothing clears it the honest answer is "not in the formulary",
# never a padded list of unrelated drugs.
suggestions = self._resolver.suggest(query, k=3, min_score=0.72)
if suggestions:
names = [self._drug_name(drug_id) for drug_id, _ in suggestions]
reason = "did_you_mean"
clarification = Clarification(
reason=reason,
question=f"Ý bạn là: {', '.join(names)}?",
options=tuple(names),
)
else:
reason = "drug_not_supported"
clarification = Clarification(
reason=reason,
question=(
"Không có thuốc này trong Dược thư Quốc gia. Vui lòng kiểm "
"tra lại tên, hoặc gõ vài ký tự để chọn từ gợi ý."
),
options=(),
)
self._metrics.increment(metric_names.CLARIFY_ASKED, reason=reason)
self._persist(state, resolved, None)
return ConversationTurnResult(None, clarification, None, False, None, reason)
if resolved.inherited_drug:
self._metrics.increment(metric_names.FOLLOWUP_INHERITED)
# One call to the safe engine with the self-contained (rewritten) query.
# A multi-round retrieval-refine loop was tried and removed: refining an
# already-answerable whole-section result cannot fetch more (the section
# is complete) and, worse, the refined query drops the inherited drug and
# abstains — discarding a good answer. Refinement belongs to the
# similarity path, not here. Clarify + inheritance are the loop's value,
# and both happen above this line.
effective = self._rewrite(query, resolved)
grounded: GroundedAnswer | None = self._answers.answer(
effective, subject_scope, intent
)
answer = grounded.answer if grounded else None
inherited = resolved.drug_id if resolved.needs_carry_over_notice else None
if answer is not None and inherited is not None:
answer = f"Về {inherited}: {answer}"
if grounded is not None:
grounded = replace(grounded, answer=answer)
self._persist(state, resolved, grounded)
return ConversationTurnResult(
answer,
None,
grounded,
False,
inherited,
grounded.result.reason if grounded else "no_answer",
)
def complete(self, prefix: str, k: int = 8) -> list[str]:
"""Display names matching a typed prefix, for input autocomplete."""
return [self._drug_name(drug_id) for drug_id in self._resolver.complete(prefix, k)]
@staticmethod
def _drug_name(drug_id: str) -> str:
"""A readable display name from a drug id ('paracetamol_acetaminophen')."""
return drug_id.replace("_", " ").title()
@staticmethod
def _rewrite(query: str, resolved) -> str:
parts: list[str] = []
if resolved.inherited_drug and resolved.drug_id:
parts.append(resolved.drug_id)
if resolved.inherited_section and resolved.section_key:
phrases = SECTION_PHRASES.get(resolved.section_key)
if phrases:
parts.append(phrases[0])
parts.append(query)
return " ".join(parts)
def _append_user(self, state, text, drug_id, section_key) -> None:
state = state.append(Turn("user", text, _now(), drug_id, section_key))
self._store.save(state)
def _persist(self, state, resolved, grounded) -> None:
focus = update_focus(state, resolved)
state = replace(state, focus=focus)
state = state.append(
Turn("user", resolved.text, _now(), resolved.drug_id, resolved.section_key)
)
if grounded is not None and grounded.answer is not None:
state = state.append(
Turn(
"assistant",
grounded.answer,
_now(),
resolved.drug_id,
resolved.section_key,
)
)
if (
self._summariser is not None
and state.turn_count % SUMMARY_EVERY == 0
and state.overflow()
):
summary = self._summariser.fold(state.summary, state.overflow())
state = replace(state, summary=summary)
self._store.save(state)
+93
View File
@@ -0,0 +1,93 @@
from __future__ import annotations
from dataclasses import dataclass
from enum import StrEnum
from .models import SubjectScope
class CaseOrigin(StrEnum):
EXPERT = "expert"
MANUAL_ADVERSARIAL = "manual_adversarial"
SOURCE_DERIVED = "source_derived"
@dataclass(frozen=True)
class EvaluationCase:
case_id: str
query: str
expected_drug_id: str | None
expected_id: str | None
origin: CaseOrigin
subject_scope: SubjectScope
@dataclass(frozen=True)
class EvaluationOutcome:
case: EvaluationCase
retrieved_ids: tuple[str, ...]
resolved_drug_id: str | None = None
drug_resolution_status: str = "not_attempted"
@property
def passed(self) -> bool:
if self.case.expected_id is None:
return not self.retrieved_ids
return self.case.expected_id in self.retrieved_ids
def summarize(outcomes: list[EvaluationOutcome]) -> dict:
def metrics(rows: list[EvaluationOutcome]) -> dict:
positive = [row for row in rows if row.case.expected_id is not None]
negative = [row for row in rows if row.case.expected_id is None]
resolution_rows = [
row for row in rows
if row.case.expected_drug_id is not None
and row.case.subject_scope == SubjectScope.HUMAN
]
return {
"cases": len(rows),
"positive_cases": len(positive),
"negative_cases": len(negative),
"recall_at_1": _recall_at(positive, 1),
"recall_at_3": _recall_at(positive, 3),
"drug_resolution_accuracy": (
round(sum(
row.resolved_drug_id == row.case.expected_drug_id
for row in resolution_rows
) / len(resolution_rows), 4)
if resolution_rows else None
),
"drug_resolution_status_counts": {
status: sum(
row.drug_resolution_status == status for row in resolution_rows
)
for status in ("resolved", "ambiguous", "not_found", "invalid_state")
},
"negative_abstain_rate": (
round(sum(row.passed for row in negative) / len(negative), 4)
if negative else None
),
}
return {
"expert_release_gate": metrics([
row for row in outcomes if row.case.origin == CaseOrigin.EXPERT
]),
"manual_routing_diagnostic": metrics([
row for row in outcomes if row.case.origin == CaseOrigin.MANUAL_ADVERSARIAL
]),
"source_derived_diagnostic": metrics([
row for row in outcomes if row.case.origin == CaseOrigin.SOURCE_DERIVED
]),
}
def _recall_at(rows: list[EvaluationOutcome], limit: int) -> float | None:
if not rows:
return None
matched = sum(
row.case.expected_id in row.retrieved_ids[:limit]
for row in rows
)
return round(matched / len(rows), 4)
+83
View File
@@ -0,0 +1,83 @@
"""Checks a generated answer against the evidence it was built from.
The answer layer may only rephrase retrieved text. This module is what makes
that a checkable property rather than a promise in a prompt: it recomputes,
from the evidence alone, whether every number and every citation in a
generated answer can be traced back to the source. A generation that fails is
discarded, never shown.
Numbers are compared **character for character**, deliberately. "7,5" and
"7.5" are not treated as equal, and no attempt is made to parse either into a
quantity. Parsing invites the one error that matters most here: `1.500` is
1500 under one reading and 1.5 under another, and a normaliser that strips
separators maps "7,5" and "75" to the same key — a tenfold dose error scored
as a match. The model is told to copy figures verbatim, so an exact match is
achievable, and every deviation from it is refused rather than interpreted.
"""
from __future__ import annotations
import re
from dataclasses import dataclass
# A digit run with internal separators kept: "500", "7,5", "1.000".
# Ranges ("4 - 6 giờ") yield two tokens, and each is checked on its own.
_NUMBER = re.compile(r"\d+(?:[.,]\d+)*")
# Citation markers are stripped before number extraction so that "[2]" is
# never mistaken for the quantity 2.
_CITATION = re.compile(r"\[(\d+)\]")
@dataclass(frozen=True)
class GroundingReport:
grounded: bool
unsupported_numbers: tuple[str, ...]
invalid_citations: tuple[int, ...]
cited_indices: tuple[int, ...]
@property
def reason(self) -> str:
if self.unsupported_numbers:
return "ungrounded_number"
if self.invalid_citations:
return "invalid_citation"
return "grounded"
def numbers_in(text: str) -> tuple[str, ...]:
"""Numeric tokens, with citation markers removed first."""
return tuple(_NUMBER.findall(_CITATION.sub(" ", text)))
def citations_in(text: str) -> tuple[int, ...]:
return tuple(int(marker) for marker in _CITATION.findall(text))
def verify(answer: str, evidence_texts: tuple[str, ...]) -> GroundingReport:
"""Whether `answer` states only figures and sources present in evidence.
`evidence_texts` is positional: citation `[n]` refers to
`evidence_texts[n - 1]`, so an out-of-range marker is a defect even when
the prose around it is faithful — a citation nobody can follow is not a
citation.
"""
source_numbers = set()
for text in evidence_texts:
source_numbers.update(numbers_in(text))
unsupported = tuple(
token for token in numbers_in(answer) if token not in source_numbers
)
invalid = tuple(
index
for index in citations_in(answer)
if not 1 <= index <= len(evidence_texts)
)
cited = tuple(sorted({index for index in citations_in(answer)} - set(invalid)))
return GroundingReport(
grounded=not unsupported and not invalid,
unsupported_numbers=unsupported,
invalid_citations=invalid,
cited_indices=cited,
)
+97
View File
@@ -0,0 +1,97 @@
from __future__ import annotations
import re
import unicodedata
from collections import Counter
from math import log
from .models import ParentDocument, RetrievalDocument, SearchHit
WORD_RE = re.compile(r"\w+", re.UNICODE)
def _normalized(text: str) -> str:
return " ".join(WORD_RE.findall(unicodedata.normalize("NFKC", text).casefold()))
def _terms(text: str) -> set[str]:
return set(_normalized(text).split())
def _char_ngrams(text: str, size: int = 3) -> set[str]:
normalized = _normalized(text)
if len(normalized) <= size:
return {normalized} if normalized else set()
return {
normalized[index:index + size]
for index in range(len(normalized) - size + 1)
}
class InMemoryLexicalRetriever:
"""Deterministic test/fallback retriever, not the production neural backend."""
def __init__(self, documents: list[RetrievalDocument]) -> None:
self._documents = tuple(documents)
self._term_counts = {
document.doc_id: Counter(_normalized(document.text).split())
for document in self._documents
}
self._average_length = (
sum(sum(counts.values()) for counts in self._term_counts.values())
/ max(1, len(self._term_counts))
)
document_frequency: Counter[str] = Counter()
for counts in self._term_counts.values():
document_frequency.update(counts.keys())
self._document_frequency = document_frequency
def search(self, query: str, drug_id: str, limit: int) -> list[SearchHit]:
query_terms = _terms(query)
query_ngrams = _char_ngrams(query)
candidates = []
for document in self._documents:
if document.drug_id != drug_id:
continue
counts = self._term_counts[document.doc_id]
bm25 = self._bm25(query_terms, counts)
ngrams = _char_ngrams(document.text)
char_score = len(query_ngrams & ngrams) / max(1, len(query_ngrams))
if bm25 > 0 or char_score > 0:
candidates.append((document, bm25, char_score))
max_bm25 = max((row[1] for row in candidates), default=0.0)
hits = [
SearchHit(
document=document,
score=0.8 * (bm25 / max_bm25 if max_bm25 else 0.0) + 0.2 * char_score,
)
for document, bm25, char_score in candidates
]
return sorted(hits, key=lambda hit: (-hit.score, hit.document.doc_id))[:limit]
def _bm25(self, query_terms: set[str], counts: Counter[str]) -> float:
total_documents = len(self._documents)
document_length = sum(counts.values())
score = 0.0
for term in query_terms:
frequency = counts.get(term, 0)
if not frequency:
continue
document_frequency = self._document_frequency[term]
inverse_frequency = log(
1 + (total_documents - document_frequency + 0.5)
/ (document_frequency + 0.5)
)
denominator = frequency + 1.5 * (
1 - 0.75 + 0.75 * document_length / max(1.0, self._average_length)
)
score += inverse_frequency * frequency * 2.5 / denominator
return score
class InMemoryParentStore:
def __init__(self, parents: list[ParentDocument]) -> None:
self._parents = {parent.parent_id: parent for parent in parents}
def get(self, parent_id: str) -> ParentDocument | None:
return self._parents.get(parent_id)
+56
View File
@@ -0,0 +1,56 @@
"""Domain counters, defined here so the numbers on a dashboard are the
numbers the domain actually decided.
Kept behind a tiny protocol rather than importing `prometheus_client` into
`rag/`: the domain records that a generation was refused for an ungrounded
number, and the process that happens to expose Prometheus does the exporting.
`NullMetrics` is the default, so tests and any deployment without a metrics
stack run unchanged.
The counter that matters is `generation_rejected` — it is the measured form of
the claim that the answer layer cannot state a figure the book does not.
"""
from __future__ import annotations
from typing import Protocol
class Metrics(Protocol):
def increment(self, name: str, **labels: str) -> None: ...
class NullMetrics:
def increment(self, name: str, **labels: str) -> None: # noqa: ARG002
# Deliberately inert: the default when no metrics stack is configured.
return None
class InMemoryMetrics:
"""Reference implementation of the contract; also what tests assert on."""
def __init__(self) -> None:
self.counts: dict[tuple[str, tuple[tuple[str, str], ...]], int] = {}
def increment(self, name: str, **labels: str) -> None:
key = (name, tuple(sorted(labels.items())))
self.counts[key] = self.counts.get(key, 0) + 1
def total(self, name: str, **labels: str) -> int:
if labels:
return self.counts.get((name, tuple(sorted(labels.items()))), 0)
return sum(count for (n, _), count in self.counts.items() if n == name)
RETRIEVAL_ROUTE = "duocthu_retrieval_route_total"
ABSTENTION = "duocthu_abstention_total"
GENERATION_REJECTED = "duocthu_generation_rejected_total"
GENERATION_SERVED = "duocthu_generation_served_total"
ANSWER_EXTRACTIVE = "duocthu_answer_extractive_total"
# Conversational loop. `CLARIFY_ASKED` is the counter that shows the system
# asking instead of guessing — the behaviour a reviewer will probe first.
CLARIFY_ASKED = "duocthu_clarify_asked_total"
LOOP_ROUNDS = "duocthu_loop_retrieval_rounds_total"
LOOP_REFINED = "duocthu_loop_refined_total"
LOOP_REPAIRED = "duocthu_loop_repaired_total"
FOLLOWUP_INHERITED = "duocthu_followup_inherited_total"
+83
View File
@@ -0,0 +1,83 @@
from __future__ import annotations
from dataclasses import dataclass, field
from enum import StrEnum
class EvidenceDecision(StrEnum):
ANSWERABLE = "answerable"
VERIFY_PDF = "verify_pdf"
ABSTAIN = "abstain"
class SubjectScope(StrEnum):
HUMAN = "human"
NON_HUMAN = "non_human"
UNKNOWN = "unknown"
class QueryIntent(StrEnum):
FACT_LOOKUP = "fact_lookup"
RECOMMENDATION = "recommendation"
UNKNOWN = "unknown"
@dataclass(frozen=True)
class SourceRef:
physical_page: int
precision: str
block_id: str | None = None
bbox: tuple[float, float, float, float] | None = None
source_crop: str | None = None
page_range: tuple[int, int] | None = None
printed_page: int | None = None
printed_page_range: tuple[int, int] | None = None
@dataclass(frozen=True)
class RetrievalDocument:
doc_id: str
drug_id: str
kind: str
text: str
section_key: str
source_refs: tuple[SourceRef, ...]
parent_id: str | None = None
requires_visual_check: bool = False
drug_name: str | None = None
@dataclass(frozen=True)
class ParentDocument:
parent_id: str
kind: str
text: str
source_refs: tuple[SourceRef, ...]
requires_visual_check: bool = False
@dataclass(frozen=True)
class SearchHit:
document: RetrievalDocument
score: float
@dataclass(frozen=True)
class Evidence:
evidence_id: str
matched_doc_id: str
kind: str
text: str
score: float
source_refs: tuple[SourceRef, ...]
hydrated_from_parent: bool
requires_visual_check: bool
@dataclass(frozen=True)
class RetrievalResult:
decision: EvidenceDecision
reason: str
evidence: tuple[Evidence, ...] = field(default_factory=tuple)
resolved_drug_id: str | None = None
drug_resolution_status: str = "not_attempted"
+57
View File
@@ -0,0 +1,57 @@
from __future__ import annotations
from typing import Protocol
from .models import ParentDocument, SearchHit
class QueryEmbeddingUnavailable(RuntimeError):
"""The similarity route's embedding provider could not be reached.
Raised by an adapter and caught by the domain, which abstains. It exists so
a provider outage refuses to answer instead of returning a 500: an
unreachable embedder means the question was never actually searched, and an
error page hides that from the caller just as effectively as a wrong answer
would. The domain catches this without importing any SDK.
"""
class AnswerGenerationUnavailable(RuntimeError):
"""The answer generator could not be reached.
Same contract as `QueryEmbeddingUnavailable`: the adapter translates its
SDK's failure into this, and the domain degrades to the extractive answer
rather than returning an error. Generation is a presentation improvement
over quoting the source; losing it must never lose the answer.
"""
class AnswerGenerator(Protocol):
"""Rewrites retrieved evidence into prose. Never a source of facts.
Whatever it returns is checked by `rag.grounding.verify` before a caller
sees it, so this port carries no trust: an implementation that fabricates
a dose produces a discarded generation, not a wrong answer.
"""
def generate(self, system: str, user: str, schema: dict) -> str: ...
class Retriever(Protocol):
def search(self, query: str, drug_id: str, limit: int) -> list[SearchHit]: ...
class SectionRetriever(Protocol):
"""Exact retrieval of one whole section, with no similarity involved.
Separate from `Retriever` so a store that cannot filter by payload is still
a valid `Retriever` (interface segregation). `find_by_section` must return
**every** part of the section: a partial contraindication list reads as a
complete one, which is worse than returning nothing.
"""
def find_by_section(self, drug_id: str, section_key: str) -> list[SearchHit]: ...
class ParentStore(Protocol):
def get(self, parent_id: str) -> ParentDocument | None: ...
+77
View File
@@ -0,0 +1,77 @@
"""The answer contract given to the generator, and the schema it must fill.
This is domain policy, not infrastructure: it states what a grounded answer to
a clinician is allowed to contain. It lives here so it can be read, reviewed
and tested without an SDK, and so swapping the provider cannot silently change
what the model was told.
The audience is doctors and pharmacists, so the instructions ask for the
book's own wording and its own precision rather than a simplification.
"""
from __future__ import annotations
from dataclasses import dataclass
SYSTEM_PROMPT = """\
Bạn trình bày lại nội dung Dược thư Quốc gia Việt Nam cho bác sĩ và dược sĩ.
Bạn KHÔNG phải nguồn tri thức. Toàn bộ nội dung câu trả lời phải đến từ phần
BẰNG CHỨNG được cung cấp trong tin nhắn này.
Quy tắc bắt buộc:
1. Chỉ dùng thông tin có trong BẰNG CHỨNG. Không thêm kiến thức y khoa từ
bên ngoài, kể cả khi bạn chắc chắn nó đúng.
2. Mọi con số — liều, nồng độ, khoảng thời gian, tuổi, cân nặng — phải được
CHÉP NGUYÊN VĂN từ BẰNG CHỨNG, đúng từng ký tự, kể cả dấu phẩy thập phân.
Không làm tròn, không đổi đơn vị, không quy đổi.
3. Mỗi ý phải gắn số nguồn dạng [n], với n là số thứ tự đoạn bằng chứng.
4. Nếu BẰNG CHỨNG không đủ để trả lời, nói rõ là không đủ. Đó là câu trả lời
hợp lệ, không phải thất bại.
5. Giữ nguyên thuật ngữ chuyên môn của sách. Không diễn giải cho người
không chuyên.
Viết gọn. Trả lời đúng điều được hỏi, không mở rộng phạm vi."""
ANSWER_SCHEMA = {
"type": "object",
"properties": {
"answer": {
"type": "string",
"description": (
"Câu trả lời cho bác sĩ/dược sĩ, mỗi ý gắn [n] chỉ nguồn. "
"Mọi con số chép nguyên văn từ bằng chứng."
),
},
"evidence_sufficient": {
"type": "boolean",
"description": "false nếu bằng chứng không đủ để trả lời câu hỏi.",
},
},
"required": ["answer", "evidence_sufficient"],
"additionalProperties": False,
}
@dataclass(frozen=True)
class GenerationRequest:
system: str
user: str
schema: dict
def build_request(question: str, evidence_texts: tuple[str, ...]) -> GenerationRequest:
"""The prompt for one question over one ordered evidence list.
Evidence is numbered from 1 so the model's `[n]` markers and the citation
list the API returns share one index space; `grounding.verify` rejects any
marker outside it.
"""
if not evidence_texts:
raise ValueError("cannot build a grounded prompt with no evidence")
blocks = "\n\n".join(
f"[{index}] {text}" for index, text in enumerate(evidence_texts, start=1)
)
user = f"BẰNG CHỨNG:\n\n{blocks}\n\nCÂU HỎI: {question}"
return GenerationRequest(system=SYSTEM_PROMPT, user=user, schema=ANSWER_SCHEMA)
+305
View File
@@ -0,0 +1,305 @@
"""The bounded reasoning loop.
Understand → plan → retrieve → assess → refine → generate → verify → repair.
Every edge is bounded, and every budget is decremented **before** the call it
pays for, so exhaustion degrades to the best answer so far rather than to an
error.
Two rules hold across every path and are the reason this can be added to a
formulary at all:
- `grounding.verify` still gates every generated answer. Reasoning chooses what
to look up and how to phrase it; it is never a source of facts.
- A clarify signal bypasses the loop entirely. Asking beats guessing, and the
signals are resolver states — ambiguous drug, unresolved attribute — not a
model's confidence score.
"""
from __future__ import annotations
from dataclasses import dataclass, field, replace
from typing import Protocol
from . import metrics as metric_names
from .conversation import ConversationState, ResolvedQuestion
from .metrics import Metrics, NullMetrics
MAX_RETRIEVAL_ROUNDS = 2
MAX_REPAIRS = 1
MAX_LLM_CALLS = 4
MAX_WALL_CLOCK_MS = 20_000
class BudgetExhausted(RuntimeError):
"""Raised only inside the loop, never surfaced; the loop catches it."""
@dataclass
class TurnBudget:
"""Mutable on purpose: one budget is threaded through one turn."""
llm_calls: int = MAX_LLM_CALLS
retrieval_rounds: int = MAX_RETRIEVAL_ROUNDS
repairs: int = MAX_REPAIRS
wall_clock_ms: int = MAX_WALL_CLOCK_MS
elapsed_ms: int = 0
def spend_llm(self) -> None:
if self.llm_calls <= 0:
raise BudgetExhausted("llm_calls")
self.llm_calls -= 1
def spend_retrieval(self) -> None:
if self.retrieval_rounds <= 0:
raise BudgetExhausted("retrieval_rounds")
self.retrieval_rounds -= 1
def spend_repair(self) -> None:
if self.repairs <= 0:
raise BudgetExhausted("repairs")
self.repairs -= 1
def out_of_time(self) -> bool:
return self.elapsed_ms >= self.wall_clock_ms
class ClarifyReason:
AMBIGUOUS_DRUG = "ambiguous_drug"
NO_ATTRIBUTE = "no_attribute"
MULTI_ATTRIBUTE = "multi_attribute"
STILL_INSUFFICIENT = "still_insufficient"
@dataclass(frozen=True)
class Clarification:
reason: str
question: str
options: tuple[str, ...] = ()
@dataclass(frozen=True)
class Sufficiency:
"""The assessor's verdict on retrieved evidence.
`missing` must name something specific — a section, a population, a second
drug. "Feels incomplete" does not buy a retrieval round; a round is only
spent when there is a concrete thing to go and fetch.
"""
sufficient: bool
missing: str | None = None
refined_query: str | None = None
class SufficiencyAssessor(Protocol):
def assess(
self, resolved: ResolvedQuestion, evidence_texts: tuple[str, ...]
) -> Sufficiency: ...
class DeterministicAssessor:
"""The no-LLM default, and the reference for what the port must do.
Runs offline and is what the loop uses until a provider is enabled. It only
reports insufficiency it can *demonstrate* — a population was asked for and
no retrieved text mentions it — so it can never spin the loop on a feeling.
"""
POPULATION_TERMS = {
"nguoi_lon": ("người lớn",),
"tre_em": ("trẻ em", "trẻ nhỏ", "trẻ "),
"tre_so_sinh": ("sơ sinh",),
"phu_nu_co_thai": ("thai", "mang thai"),
"phu_nu_cho_con_bu": ("cho con bú", "sữa mẹ"),
"nguoi_cao_tuoi": ("người cao tuổi", "người già"),
"suy_than": ("suy thận", "clcr"),
"suy_gan": ("suy gan",),
}
def assess(
self, resolved: ResolvedQuestion, evidence_texts: tuple[str, ...]
) -> Sufficiency:
if not evidence_texts:
return Sufficiency(False, missing="no_evidence")
if resolved.population is None:
return Sufficiency(True)
terms = self.POPULATION_TERMS.get(resolved.population, ())
haystack = " ".join(evidence_texts).casefold()
if any(term in haystack for term in terms):
return Sufficiency(True)
return Sufficiency(
False,
missing=f"population:{resolved.population}",
refined_query=f"{resolved.text} {terms[0] if terms else ''}".strip(),
)
@dataclass(frozen=True)
class LoopOutcome:
"""What one turn produced, plus what it cost."""
answer: str | None
clarification: Clarification | None
evidence_texts: tuple[str, ...]
retrieval_rounds_used: int
repairs_used: int
stopped_because: str
generated: bool = False
@dataclass
class LoopTrace:
"""Ordered record of stages, for the dashboard and for debugging."""
stages: list[str] = field(default_factory=list)
def enter(self, stage: str) -> None:
self.stages.append(stage)
def clarify_for(
reason: str, options: tuple[str, ...] = ()
) -> Clarification:
questions = {
ClarifyReason.NO_ATTRIBUTE: (
"Anh/chị muốn tra thuộc tính nào của thuốc này?"
),
ClarifyReason.AMBIGUOUS_DRUG: (
"Câu hỏi có thể ứng với nhiều thuốc. Anh/chị muốn tra thuốc nào?"
),
ClarifyReason.MULTI_ATTRIBUTE: (
"Câu hỏi nhắc tới nhiều mục. Anh/chị muốn xem mục nào trước?"
),
ClarifyReason.STILL_INSUFFICIENT: (
"Chưa tìm đủ căn cứ trong Dược thư cho ý này. "
"Anh/chị có thể nêu rõ hơn điều cần tra không?"
),
}
return Clarification(reason, questions[reason], options)
class Retrieve(Protocol):
def __call__(self, resolved: ResolvedQuestion) -> tuple[str, ...]: ...
class Generate(Protocol):
def __call__(
self, resolved: ResolvedQuestion, evidence: tuple[str, ...], state: ConversationState
) -> str | None: ...
def run_turn(
state: ConversationState,
resolved: ResolvedQuestion,
retrieve: Retrieve,
generate: Generate,
clarify_signals: tuple[str, ...] = (),
assessor: SufficiencyAssessor | None = None,
budget: TurnBudget | None = None,
metrics: Metrics | None = None,
trace: LoopTrace | None = None,
) -> LoopOutcome:
"""One conversational turn through the bounded loop.
`clarify_signals` comes from the existing resolvers — ambiguous drug,
unresolved section, multi-attribute. They short-circuit before any spend,
because a question worth asking is cheaper and safer than a guess.
"""
budget = budget or TurnBudget()
assessor = assessor or DeterministicAssessor()
metrics = metrics or NullMetrics()
trace = trace or LoopTrace()
trace.enter("understand")
if clarify_signals:
reason = clarify_signals[0]
metrics.increment(metric_names.CLARIFY_ASKED, reason=reason)
trace.enter("clarify")
return LoopOutcome(
answer=None,
clarification=clarify_for(reason),
evidence_texts=(),
retrieval_rounds_used=0,
repairs_used=0,
stopped_because="clarify_signal",
)
evidence: tuple[str, ...] = ()
rounds_used = 0
stopped = "sufficient"
while True:
try:
budget.spend_retrieval()
except BudgetExhausted:
stopped = "retrieval_budget"
break
trace.enter("retrieve")
evidence = retrieve(resolved)
rounds_used += 1
trace.enter("assess")
verdict = assessor.assess(resolved, evidence)
if verdict.sufficient:
break
if budget.retrieval_rounds <= 0 or budget.out_of_time():
stopped = "retrieval_budget"
break
# A round is spent only on a named gap with a genuinely new query.
if not verdict.missing or not verdict.refined_query:
stopped = "no_actionable_gap"
break
if verdict.refined_query == resolved.text:
stopped = "query_unchanged"
break
trace.enter("refine")
metrics.increment(metric_names.LOOP_REFINED, missing=verdict.missing)
resolved = replace(resolved, text=verdict.refined_query)
metrics.increment(metric_names.LOOP_ROUNDS, rounds=str(rounds_used))
if not evidence:
trace.enter("clarify")
metrics.increment(
metric_names.CLARIFY_ASKED, reason=ClarifyReason.STILL_INSUFFICIENT
)
return LoopOutcome(
answer=None,
clarification=clarify_for(ClarifyReason.STILL_INSUFFICIENT),
evidence_texts=(),
retrieval_rounds_used=rounds_used,
repairs_used=0,
stopped_because="no_evidence",
)
repairs_used = 0
answer: str | None = None
while True:
trace.enter("generate")
try:
budget.spend_llm()
except BudgetExhausted:
stopped = "llm_budget"
break
answer = generate(resolved, evidence, state)
if answer is not None:
break
# `generate` returning None means verification already refused it.
try:
budget.spend_repair()
except BudgetExhausted:
stopped = "repair_budget"
break
repairs_used += 1
trace.enter("repair")
metrics.increment(metric_names.LOOP_REPAIRED)
return LoopOutcome(
answer=answer,
clarification=None,
evidence_texts=evidence,
retrieval_rounds_used=rounds_used,
repairs_used=repairs_used,
stopped_because=stopped,
generated=answer is not None,
)
+260
View File
@@ -0,0 +1,260 @@
from __future__ import annotations
import re
from dataclasses import dataclass, replace
from difflib import SequenceMatcher
from enum import StrEnum
from .models import EvidenceDecision, QueryIntent, RetrievalResult, SubjectScope
from .service import RetrievalService
from .text import WORD_RE, normalize_name
__all__ = [
"WORD_RE",
"CatalogDrugResolver",
"DrugResolution",
"DrugResolutionStatus",
"QueryRoutingService",
"normalize_name",
]
class DrugResolutionStatus(StrEnum):
RESOLVED = "resolved"
NOT_FOUND = "not_found"
AMBIGUOUS = "ambiguous"
@dataclass(frozen=True)
class DrugResolution:
status: DrugResolutionStatus
drug_id: str | None = None
score: float | None = None
candidate_drug_ids: tuple[str, ...] = ()
class CatalogDrugResolver:
def __init__(
self,
catalog: dict[str, set[str]],
fuzzy_threshold: float = 0.84,
ambiguity_margin: float = 0.04,
) -> None:
self._catalog = {
drug_id: {
normalized for alias in aliases if (normalized := normalize_name(alias))
}
for drug_id, aliases in catalog.items()
}
self._aliases = [
(drug_id, normalized)
for drug_id, aliases in catalog.items()
for alias in aliases
if (normalized := normalize_name(alias))
]
self._fuzzy_threshold = fuzzy_threshold
self._ambiguity_margin = ambiguity_margin
def resolve(self, query: str) -> DrugResolution:
normalized_query = normalize_name(query)
query_tokens = normalized_query.split()
exact = [
(drug_id, alias, match.start(1), match.end(1))
for drug_id, alias in self._aliases
for match in [
re.search(rf"(?:^| )({re.escape(alias)})(?:$| )", normalized_query)
]
if match
]
if exact:
maximal = [
row for row in exact
if not any(
other[2] <= row[2] and row[3] <= other[3]
and (other[2], other[3]) != (row[2], row[3])
for other in exact
)
]
drug_ids = {drug_id for drug_id, _, _, _ in maximal}
if len(drug_ids) == 1:
return DrugResolution(
DrugResolutionStatus.RESOLVED, next(iter(drug_ids)), 1.0,
)
return DrugResolution(
DrugResolutionStatus.AMBIGUOUS,
candidate_drug_ids=tuple(sorted(drug_ids)),
)
scores: dict[str, float] = {}
for drug_id, alias in self._aliases:
width = len(alias.split())
if width > len(query_tokens):
continue
spans = (
" ".join(query_tokens[start:start + width])
for start in range(len(query_tokens) - width + 1)
)
score = max(
(SequenceMatcher(None, alias, span).ratio() for span in spans),
default=0.0,
)
scores[drug_id] = max(scores.get(drug_id, 0.0), score)
ranked = sorted(scores.items(), key=lambda item: (-item[1], item[0]))
if not ranked or ranked[0][1] < self._fuzzy_threshold:
return DrugResolution(DrugResolutionStatus.NOT_FOUND)
if len(ranked) > 1 and ranked[0][1] - ranked[1][1] < self._ambiguity_margin:
return DrugResolution(
DrugResolutionStatus.AMBIGUOUS,
candidate_drug_ids=(ranked[0][0], ranked[1][0]),
)
return DrugResolution(DrugResolutionStatus.RESOLVED, *ranked[0])
def aliases_for(self, drug_id: str) -> set[str]:
return self._catalog.get(drug_id, set())
def complete(self, prefix: str, k: int = 8) -> list[str]:
"""Drug ids whose alias contains `prefix`, for as-you-type autocomplete.
Substring match on the normalized alias, ranked prefix-first then by
alias length, so "para" surfaces "paracetamol" ahead of a drug that only
contains "para" mid-word. Distinct drug ids, best first.
"""
needle = normalize_name(prefix)
if not needle:
return []
matches: list[tuple[tuple[int, int], str]] = []
for drug_id, alias in self._aliases:
position = alias.find(needle)
if position < 0:
continue
matches.append(((0 if position == 0 else 1, len(alias)), drug_id))
matches.sort()
ordered: list[str] = []
seen: set[str] = set()
for _, drug_id in matches:
if drug_id in seen:
continue
seen.add(drug_id)
ordered.append(drug_id)
if len(ordered) >= k:
break
return ordered
def suggest(
self, query: str, k: int = 3, min_score: float = 0.5
) -> list[tuple[str, float]]:
"""Closest drug ids by fuzzy score, for a 'did you mean' on a miss.
Uses the same windowed SequenceMatcher scoring as `resolve`, but returns
the top-k *below* the resolution threshold too, so a typo that does not
confidently resolve ("metfomin") can still be offered as a suggestion.
`min_score` keeps a genuinely non-drug query ("cái này thế nào") from
surfacing spurious suggestions.
"""
query_tokens = normalize_name(query).split()
if not query_tokens:
return []
scores: dict[str, float] = {}
for drug_id, alias in self._aliases:
width = len(alias.split())
if width > len(query_tokens):
continue
spans = (
" ".join(query_tokens[start:start + width])
for start in range(len(query_tokens) - width + 1)
)
score = max(
(SequenceMatcher(None, alias, span).ratio() for span in spans),
default=0.0,
)
scores[drug_id] = max(scores.get(drug_id, 0.0), score)
ranked = sorted(scores.items(), key=lambda item: (-item[1], item[0]))
return [(drug_id, score) for drug_id, score in ranked[:k] if score >= min_score]
class QueryRoutingService:
def __init__(
self,
retrieval: RetrievalService,
resolver: CatalogDrugResolver,
) -> None:
self._retrieval = retrieval
self._resolver = resolver
def retrieve(
self,
query: str,
subject_scope: SubjectScope = SubjectScope.UNKNOWN,
intent: QueryIntent = QueryIntent.UNKNOWN,
) -> RetrievalResult:
# Scope comes from the API/policy layer. Unknown is deliberately
# fail-closed; retrieval must not infer clinical scope from keywords.
if subject_scope == SubjectScope.NON_HUMAN:
return RetrievalResult(EvidenceDecision.ABSTAIN, "out_of_scope_non_human")
if subject_scope == SubjectScope.UNKNOWN:
return RetrievalResult(EvidenceDecision.ABSTAIN, "subject_scope_unknown")
if intent == QueryIntent.RECOMMENDATION:
return RetrievalResult(EvidenceDecision.ABSTAIN, "recommendation_out_of_scope")
if intent == QueryIntent.UNKNOWN:
return RetrievalResult(EvidenceDecision.ABSTAIN, "query_intent_unknown")
resolution = self._resolver.resolve(query)
if resolution.status == DrugResolutionStatus.NOT_FOUND:
return RetrievalResult(
EvidenceDecision.ABSTAIN,
"drug_not_resolved",
drug_resolution_status=resolution.status,
)
if resolution.status == DrugResolutionStatus.AMBIGUOUS:
disambiguated = self._disambiguate_with_evidence(
query, resolution.candidate_drug_ids,
)
if disambiguated is not None:
drug_id, result = disambiguated
return replace(
result,
resolved_drug_id=drug_id,
drug_resolution_status=DrugResolutionStatus.RESOLVED,
)
return RetrievalResult(
EvidenceDecision.ABSTAIN, "drug_resolution_ambiguous",
drug_resolution_status=resolution.status,
)
if resolution.drug_id is None:
return RetrievalResult(
EvidenceDecision.ABSTAIN,
"drug_resolution_invalid_state",
drug_resolution_status="invalid_state",
)
result = self._retrieval.retrieve(query, resolution.drug_id)
return replace(
result,
resolved_drug_id=resolution.drug_id,
drug_resolution_status=resolution.status,
)
def _disambiguate_with_evidence(
self,
query: str,
candidate_ids: tuple[str, ...],
) -> tuple[str, RetrievalResult] | None:
"""Resolve subject-vs-component ambiguity through asymmetric evidence.
A candidate wins only when its retrieved evidence explicitly contains
every other mentioned entity, while the reverse direction does not.
This keeps genuine multi-drug questions ambiguous.
"""
if len(candidate_ids) < 2:
return None
winners = []
for candidate_id in candidate_ids:
result = self._retrieval.retrieve(query, candidate_id)
if result.decision == EvidenceDecision.ABSTAIN:
continue
evidence_text = normalize_name(" ".join(item.text for item in result.evidence))
others = [item for item in candidate_ids if item != candidate_id]
if all(any(
re.search(rf"(?:^| ){re.escape(alias)}(?:$| )", evidence_text)
for alias in self._resolver.aliases_for(other)
) for other in others):
winners.append((candidate_id, result))
return winners[0] if len(winners) == 1 else None
+97
View File
@@ -0,0 +1,97 @@
from __future__ import annotations
import argparse
import json
from pathlib import Path
from .artifacts import build_drug_catalog, load_aliases, load_documents, load_parents
from .evaluation import CaseOrigin, EvaluationCase, EvaluationOutcome, summarize
from .in_memory import InMemoryLexicalRetriever, InMemoryParentStore
from .models import EvidenceDecision, QueryIntent, SubjectScope
from .routing import CatalogDrugResolver, QueryRoutingService
from .service import EvidencePolicy, RetrievalService
def read_cases(path: Path) -> list[EvaluationCase]:
with path.open(encoding="utf-8") as handle:
return [
EvaluationCase(
case_id=raw["case_id"],
query=raw["query"],
expected_drug_id=raw.get("expected_drug_id"),
expected_id=raw.get("expected_id"),
origin=CaseOrigin(raw["origin"]),
subject_scope=SubjectScope(raw.get("subject_scope", "human")),
)
for line in handle
if line.strip()
for raw in [json.loads(line)]
]
def run(
cases_path: Path,
documents_path: Path,
parents_path: Path,
aliases_path: Path | None = None,
) -> dict:
documents = load_documents(documents_path)
retrieval = RetrievalService(
InMemoryLexicalRetriever(documents),
InMemoryParentStore(load_parents(parents_path)),
EvidencePolicy(),
)
service = QueryRoutingService(
retrieval,
CatalogDrugResolver(build_drug_catalog(documents, load_aliases(aliases_path))),
)
outcomes = []
details = []
for case in read_cases(cases_path):
result = service.retrieve(
case.query,
case.subject_scope,
QueryIntent.FACT_LOOKUP,
)
retrieved = (
tuple(item.evidence_id for item in result.evidence)
if result.decision != EvidenceDecision.ABSTAIN
else ()
)
outcome = EvaluationOutcome(
case=case,
retrieved_ids=retrieved,
resolved_drug_id=result.resolved_drug_id,
drug_resolution_status=result.drug_resolution_status,
)
outcomes.append(outcome)
details.append({
"case_id": case.case_id,
"passed": outcome.passed,
"decision": result.decision,
"reason": result.reason,
"expected_id": case.expected_id,
"retrieved_ids": retrieved,
"expected_drug_id": case.expected_drug_id,
"resolved_drug_id": result.resolved_drug_id,
"drug_resolution_status": result.drug_resolution_status,
})
return {**summarize(outcomes), "details": details}
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--cases", type=Path, required=True)
parser.add_argument("--documents", type=Path, required=True)
parser.add_argument("--parents", type=Path, required=True)
parser.add_argument("--aliases", type=Path)
args = parser.parse_args()
print(json.dumps(
run(args.cases, args.documents, args.parents, args.aliases),
ensure_ascii=False,
indent=2,
))
if __name__ == "__main__":
main()
+210
View File
@@ -0,0 +1,210 @@
"""Resolve an attribute question to the monograph section that answers it.
Measured 2026-08-04: letting vector similarity choose the section gives
hit@1 0.544 overall and **0.05 on `chong_chi_dinh`**, because
`duoc_ly_va_co_che_tac_dung` is the largest section and describes the drug in
general terms, so it sits close to almost any question about that drug. A
question that names its own attribute does not need similarity to guess.
Two rules make this safe:
**Longest phrase wins.** "chống chỉ định" and "chỉ định" differ by one prefix
word and mean opposite things clinically. Ordering by phrase length means the
contraindication phrase is tested first and the indication phrase can never
capture it. The same rule keeps "quá liều" from being read as "liều" and
"hướng dẫn xử trí ADR" from being read as "tác dụng phụ".
**No match is not a guess.** An unrecognised question returns `None` and the
caller falls back to similarity search. This layer never picks a section it is
not sure of.
Adding a section or a phrasing means adding an entry to `SECTION_PHRASES` —
never editing the matching code (CLAUDE.md, open/closed).
"""
from __future__ import annotations
from dataclasses import dataclass
from .text import normalize_name
# Phrases a clinician would actually type. Order within a list does not matter;
# the resolver sorts every phrase by length across all sections.
SECTION_PHRASES: dict[str, tuple[str, ...]] = {
"chong_chi_dinh": (
"chống chỉ định",
"không được dùng cho",
"không được dùng khi",
"cấm dùng",
),
"chi_dinh": (
"chỉ định",
"dùng để điều trị",
"dùng trong trường hợp nào",
"điều trị bệnh gì",
"dùng khi nào",
),
"lieu_luong_va_cach_dung": (
"liều lượng và cách dùng",
"liều lượng",
"liều dùng",
"cách dùng",
"dùng liều",
"uống bao nhiêu",
"tiêm bao nhiêu",
# Bare "liều" is safe only because longer phrases are tested first:
# "quá liều" and "xử trí quá liều" both contain it and both win.
# Measured need: 4 of 16 human-written golden questions say just
# "Liều Metformin cho người lớn?".
"liều",
),
"than_trong": (
"thận trọng",
"cần lưu ý gì",
"lưu ý khi dùng",
),
"tac_dung_khong_mong_muon": (
"tác dụng không mong muốn",
"tác dụng phụ",
"phản ứng có hại",
"tác dụng ngoại ý",
),
"huong_dan_xu_tri_adr": (
"hướng dẫn xử trí adr",
"xử trí tác dụng phụ",
"xử trí phản ứng có hại",
"xử trí adr",
),
"qua_lieu_va_xu_tri": (
"quá liều và xử trí",
"xử trí quá liều",
"quá liều",
"ngộ độc",
),
"tuong_tac_thuoc": (
"tương tác thuốc",
"tương tác với",
"tương tác",
),
"tuong_ky": (
"tương kỵ",
),
"thoi_ky_mang_thai": (
"thời kỳ mang thai",
"phụ nữ có thai",
"phụ nữ mang thai",
"mang thai",
"có thai",
"thai kỳ",
# Colloquial, and clinicians type it: one golden question asks
# "Bà bầu dùng Ibuprofen được không?".
"bà bầu",
"phụ nữ mang bầu",
),
"thoi_ky_cho_con_bu": (
"thời kỳ cho con bú",
"phụ nữ cho con bú",
"cho con bú",
"đang cho bú",
"thời kỳ bú mẹ",
),
"duoc_ly_va_co_che_tac_dung": (
"dược lý và cơ chế tác dụng",
"cơ chế tác dụng",
"dược lý",
"cơ chế",
),
"dang_thuoc_va_ham_luong": (
"dạng thuốc và hàm lượng",
"dạng bào chế",
"dạng thuốc",
"hàm lượng",
),
"do_on_dinh_va_bao_quan": (
"độ ổn định và bảo quản",
"độ ổn định",
"bảo quản",
),
"ten_chung_quoc_te": (
"tên chung quốc tế",
"tên quốc tế",
),
"ten_thuong_mai": (
"tên thương mại",
"biệt dược",
),
"loai_thuoc": (
"loại thuốc",
"nhóm thuốc",
"thuộc nhóm",
),
"ma_atc": (
"mã atc",
),
"thong_tin_quy_che": (
"thông tin quy chế",
"quy chế",
),
}
# Book order of monograph sections (Hướng dẫn sử dụng, printed page 39). Used to
# present a whole-drug overview when the query names the drug but no attribute —
# typing "PARACETAMOL" should return the monograph, never a "specify an
# attribute" dead-end.
SECTION_ORDER: tuple[str, ...] = (
"ten_chung_quoc_te",
"ma_atc",
"loai_thuoc",
"dang_thuoc_va_ham_luong",
"duoc_ly_va_co_che_tac_dung",
"chi_dinh",
"chong_chi_dinh",
"than_trong",
"thoi_ky_mang_thai",
"thoi_ky_cho_con_bu",
"tac_dung_khong_mong_muon",
"huong_dan_xu_tri_adr",
"lieu_luong_va_cach_dung",
"tuong_tac_thuoc",
"do_on_dinh_va_bao_quan",
"tuong_ky",
"qua_lieu_va_xu_tri",
"thong_tin_quy_che",
)
@dataclass(frozen=True)
class SectionMatch:
section_key: str
phrase: str
def _index(phrases: dict[str, tuple[str, ...]]) -> tuple[tuple[str, str, str], ...]:
"""(normalized_phrase, section_key, original_phrase), longest first."""
rows = [
(normalized, section_key, phrase)
for section_key, section_phrases in phrases.items()
for phrase in section_phrases
if (normalized := normalize_name(phrase))
]
# Length first so a superstring is always tested before its substring;
# the phrase text breaks ties so the order is deterministic.
rows.sort(key=lambda row: (-len(row[0]), row[0]))
return tuple(rows)
class SectionResolver:
"""Maps a question to a `section_key`, or to nothing at all."""
def __init__(self, phrases: dict[str, tuple[str, ...]] | None = None) -> None:
self._index = _index(phrases if phrases is not None else SECTION_PHRASES)
def resolve(self, query: str) -> SectionMatch | None:
normalized_query = normalize_name(query)
if not normalized_query:
return None
padded = f" {normalized_query} "
for normalized_phrase, section_key, phrase in self._index:
if f" {normalized_phrase} " in padded:
return SectionMatch(section_key, phrase)
return None
+149
View File
@@ -0,0 +1,149 @@
from __future__ import annotations
from dataclasses import dataclass
from .models import Evidence, EvidenceDecision, RetrievalResult, SearchHit
from .ports import ParentStore, QueryEmbeddingUnavailable, Retriever
from .sections import SectionResolver
@dataclass(frozen=True)
class EvidencePolicy:
minimum_score: float = 0.12
candidate_limit: int = 5
evidence_limit: int = 3
class RetrievalService:
"""Section-filtered retrieval when the question names its attribute.
Similarity is the fallback, not the default. Measured 2026-08-04, letting
similarity choose the section answers "chống chỉ định" correctly 1 time in
20, because the largest section (`duoc_ly_va_co_che_tac_dung`) sits close
to any question about the drug. When the question says which section it
wants, filtering answers it exactly.
"""
def __init__(
self,
retriever: Retriever,
parent_store: ParentStore,
policy: EvidencePolicy | None = None,
section_resolver: SectionResolver | None = None,
) -> None:
self._retriever = retriever
self._parent_store = parent_store
self._policy = policy or EvidencePolicy()
self._section_resolver = section_resolver
def retrieve(self, query: str, drug_id: str) -> RetrievalResult:
if not query.strip() or not drug_id.strip():
return RetrievalResult(EvidenceDecision.ABSTAIN, "missing_query_or_drug")
section_hits = self._section_hits(query, drug_id)
if section_hits is not None:
# No `evidence_limit` here: the whole section is the answer, and a
# truncated list of contraindications reads as a complete one.
return self._decide(self._hydrate(section_hits, limit=None))
# Drug resolved but no attribute named ("PARACETAMOL"): show the whole
# monograph, in book order, rather than dead-ending on "specify an
# attribute". A drug reference answers a drug name with the drug.
overview_hits = self._drug_overview(drug_id)
if overview_hits is not None:
return self._decide(self._hydrate(overview_hits, limit=None))
try:
hits = self._retriever.search(
query=query,
drug_id=drug_id,
limit=self._policy.candidate_limit,
)
except QueryEmbeddingUnavailable:
# Fail closed. The section route needs no embedder, so this only
# ever narrows the fallback: the caller is told nothing was found
# rather than being shown an error page or, worse, an answer built
# from a search that never ran.
return RetrievalResult(
EvidenceDecision.ABSTAIN, "query_embedding_unavailable"
)
if not hits or hits[0].score < self._policy.minimum_score:
return RetrievalResult(EvidenceDecision.ABSTAIN, "insufficient_retrieval_score")
return self._decide(self._hydrate(hits))
def _drug_overview(self, drug_id: str) -> list[SearchHit] | None:
"""Every prose section of the drug, or None if the store cannot scroll."""
find_by_drug = getattr(self._retriever, "find_by_drug", None)
if find_by_drug is None:
return None
hits = find_by_drug(drug_id)
return hits or None
def _section_hits(self, query: str, drug_id: str) -> list[SearchHit] | None:
"""Hits for an explicitly named section, or None to fall back.
Returns None — not an empty list — when this route does not apply, so
"no section named" stays distinguishable from "section named but empty".
"""
if self._section_resolver is None:
return None
find_by_section = getattr(self._retriever, "find_by_section", None)
if find_by_section is None:
return None
match = self._section_resolver.resolve(query)
if match is None:
return None
hits = find_by_section(drug_id, match.section_key)
return hits or None
def _decide(self, evidence: tuple[Evidence, ...]) -> RetrievalResult:
if not evidence:
return RetrievalResult(EvidenceDecision.ABSTAIN, "parent_hydration_failed")
if any(not item.source_refs for item in evidence):
return RetrievalResult(EvidenceDecision.ABSTAIN, "missing_provenance")
if any(item.requires_visual_check for item in evidence):
return RetrievalResult(EvidenceDecision.VERIFY_PDF, "visual_verification_required", evidence)
return RetrievalResult(EvidenceDecision.ANSWERABLE, "grounded_evidence_available", evidence)
def _hydrate(
self, hits: list[SearchHit], limit: int | None = -1
) -> tuple[Evidence, ...]:
output: list[Evidence] = []
seen: set[str] = set()
for hit in hits:
document = hit.document
evidence_id = document.parent_id or document.doc_id
if evidence_id in seen:
continue
seen.add(evidence_id)
if document.parent_id:
parent = self._parent_store.get(document.parent_id)
if parent is None:
continue
output.append(Evidence(
evidence_id=parent.parent_id,
matched_doc_id=document.doc_id,
kind=parent.kind,
text=parent.text,
score=hit.score,
source_refs=parent.source_refs,
hydrated_from_parent=True,
requires_visual_check=(
document.requires_visual_check or parent.requires_visual_check
),
))
else:
output.append(Evidence(
evidence_id=document.doc_id,
matched_doc_id=document.doc_id,
kind=document.kind,
text=document.text,
score=hit.score,
source_refs=document.source_refs,
hydrated_from_parent=False,
requires_visual_check=document.requires_visual_check,
))
cap = self._policy.evidence_limit if limit == -1 else limit
if cap is not None and len(output) >= cap:
break
return tuple(output)
+23
View File
@@ -0,0 +1,23 @@
"""Vietnamese text normalisation shared by drug and section resolution.
Lives here rather than in `routing.py` because `sections.py` needs it too, and
importing it from `routing` made `service -> sections -> routing -> service` a
cycle. It is a text utility with no knowledge of drugs or sections.
"""
from __future__ import annotations
import re
import unicodedata
WORD_RE = re.compile(r"\w+", re.UNICODE)
def normalize_name(text: str) -> str:
"""Casefold, strip diacritics, collapse to space-separated word tokens.
`đ` is replaced before decomposition because it is a distinct letter rather
than a base letter plus a combining mark, so NFKD leaves it intact.
"""
decomposed = unicodedata.normalize("NFKD", text.casefold()).replace("đ", "d")
without_marks = "".join(char for char in decomposed if not unicodedata.combining(char))
return " ".join(WORD_RE.findall(without_marks))
+146
View File
@@ -0,0 +1,146 @@
from __future__ import annotations
from typing import Annotated, Any, Protocol
from fastapi import APIRouter, Depends, HTTPException, Request
from pydantic import BaseModel, Field
from rag.answer import GroundedAnswerService
from rag.models import QueryIntent, SubjectScope
class TraceWriter(Protocol):
def save(self, **fields: Any) -> str: ...
class RagQueryRequest(BaseModel):
query: str = Field(min_length=1, max_length=4000)
subject_scope: SubjectScope
intent: QueryIntent
# Optional: when present, the turn is answered in conversation context
# (follow-up inheritance, clarify, smalltalk). Absent → single-turn, exactly
# as before, so existing callers are unchanged.
conversation_id: str | None = Field(default=None, max_length=128)
class CitationResponse(BaseModel):
chunk_id: str
printed_page_start: int
printed_page_end: int
physical_page: int
block_id: str | None = None
bbox: tuple[float, float, float, float] | None = None
source_crop: str | None = None
attachment: str | None = None
class RagQueryResponse(BaseModel):
trace_id: str
decision: str
reason: str
answer: str | None
resolved_drug_id: str | None
citations: list[CitationResponse]
def _answer_service(request: Request) -> GroundedAnswerService:
service = getattr(request.app.state, "answer_service", None)
if service is None:
raise HTTPException(status_code=503, detail="RAG backend is not configured")
return service
def _trace_writer(request: Request) -> TraceWriter:
writer = getattr(request.app.state, "trace_writer", None)
if writer is None:
raise HTTPException(status_code=503, detail="Trace database is not configured")
return writer
router = APIRouter(prefix="/v1/rag", tags=["rag"])
class SuggestResponse(BaseModel):
suggestions: list[str]
@router.get("/suggest", response_model=SuggestResponse)
def suggest_drugs(q: str, request: Request) -> SuggestResponse:
"""As-you-type drug-name autocomplete, so a name is picked, not mistyped."""
conversational = getattr(request.app.state, "conversational", None)
if conversational is None or not q.strip():
return SuggestResponse(suggestions=[])
return SuggestResponse(suggestions=conversational.complete(q.strip()))
def _map_citations(items) -> list[CitationResponse]:
return [
CitationResponse(
chunk_id=item.chunk_id,
printed_page_start=item.printed_page_start,
printed_page_end=item.printed_page_end,
physical_page=item.physical_page,
block_id=item.block_id,
bbox=item.bbox,
source_crop=item.source_crop,
attachment=item.attachment,
)
for item in items
]
@router.post("/query", response_model=RagQueryResponse)
def query_rag(
payload: RagQueryRequest,
request: Request,
answers: Annotated[GroundedAnswerService, Depends(_answer_service)],
traces: Annotated[TraceWriter, Depends(_trace_writer)],
) -> RagQueryResponse:
conversational = getattr(request.app.state, "conversational", None)
# Single-turn path (no conversation id, or conversational layer disabled):
# unchanged behaviour so existing callers keep working.
if payload.conversation_id is None or conversational is None:
grounded = answers.answer(payload.query, payload.subject_scope, payload.intent)
decision = grounded.result.decision.value
reason = grounded.result.reason
answer = grounded.answer
resolved_drug_id = grounded.result.resolved_drug_id
citations = _map_citations(grounded.citations)
else:
turn = conversational.answer(
payload.conversation_id,
payload.query,
payload.subject_scope,
payload.intent,
)
if turn.clarification is not None:
decision, reason = "clarify", turn.clarification.reason
answer, resolved_drug_id, citations = turn.clarification.question, None, []
elif turn.grounded is not None:
decision = turn.grounded.result.decision.value
reason = turn.grounded.result.reason
answer = turn.answer
resolved_drug_id = turn.grounded.result.resolved_drug_id
citations = _map_citations(turn.grounded.citations)
else: # smalltalk
decision, reason = "answerable", turn.reason
answer, resolved_drug_id, citations = turn.answer, None, []
trace_id = traces.save(
query=payload.query,
subject_scope=payload.subject_scope.value,
intent=payload.intent.value,
decision=decision,
reason=reason,
resolved_drug_id=resolved_drug_id,
citations=tuple(item.model_dump() for item in citations),
)
return RagQueryResponse(
trace_id=trace_id,
decision=decision,
reason=reason,
answer=answer,
resolved_drug_id=resolved_drug_id,
citations=citations,
)
@@ -0,0 +1,90 @@
from rag.answer import GroundedAnswerService
from rag.models import (
Evidence,
EvidenceDecision,
QueryIntent,
RetrievalResult,
SourceRef,
SubjectScope,
)
class FixedRouting:
def __init__(self, result: RetrievalResult) -> None:
self._result = result
def retrieve(self, query, subject_scope, intent):
del query, subject_scope, intent
return self._result
def evidence(source: SourceRef, *, visual: bool = False) -> Evidence:
return Evidence(
evidence_id="chunk-1", matched_doc_id="chunk-1", kind="prose",
text="Liều được ghi trong nguồn.", score=0.9, source_refs=(source,),
hydrated_from_parent=False, requires_visual_check=visual,
)
def test_answer_uses_only_printed_page_citations():
source = SourceRef(
physical_page=100, precision="chunk_page_range",
page_range=(100, 102), printed_page_range=(101, 103),
)
service = GroundedAnswerService(FixedRouting(RetrievalResult(
EvidenceDecision.ANSWERABLE, "grounded_evidence_available",
(evidence(source),), "abacavir", "resolved",
)))
answer = service.answer("q", SubjectScope.HUMAN, QueryIntent.FACT_LOOKUP)
assert answer.answer == "Liều được ghi trong nguồn. [1]"
assert answer.citations[0].printed_page_start == 101
assert answer.citations[0].printed_page_end == 103
def test_answer_abstains_when_only_physical_page_is_available():
source = SourceRef(physical_page=100, precision="chunk_page_range")
service = GroundedAnswerService(FixedRouting(RetrievalResult(
EvidenceDecision.ANSWERABLE, "grounded_evidence_available",
(evidence(source),), "abacavir", "resolved",
)))
answer = service.answer("q", SubjectScope.HUMAN, QueryIntent.FACT_LOOKUP)
assert answer.result.decision == EvidenceDecision.ABSTAIN
assert answer.result.reason == "missing_printed_page_provenance"
assert answer.answer is None
assert answer.citations == ()
def test_visual_evidence_never_auto_extracts_numbers():
source = SourceRef(
physical_page=100, precision="region", printed_page=101,
)
service = GroundedAnswerService(FixedRouting(RetrievalResult(
EvidenceDecision.VERIFY_PDF, "visual_verification_required",
(evidence(source, visual=True),), "abacavir", "resolved",
)))
answer = service.answer("q", SubjectScope.HUMAN, QueryIntent.FACT_LOOKUP)
assert "không tự động trích số liệu" in answer.answer
assert "Liều được ghi" not in answer.answer
def test_visual_citation_preserves_block_page_and_bbox_without_a_crop_file():
source = SourceRef(
physical_page=209,
precision="region",
block_id="p209_t0",
bbox=(49.5, 68.1, 289.4, 789.4),
printed_page=210,
)
service = GroundedAnswerService(FixedRouting(RetrievalResult(
EvidenceDecision.VERIFY_PDF, "visual_verification_required",
(evidence(source, visual=True),), "arsenic_trioxyd", "resolved",
)))
answer = service.answer("q", SubjectScope.HUMAN, QueryIntent.FACT_LOOKUP)
citation = answer.citations[0]
assert citation.physical_page == 209
assert citation.block_id == "p209_t0"
assert citation.bbox == (49.5, 68.1, 289.4, 789.4)
assert citation.source_crop is None
assert citation.attachment == "p209_t0"
+53
View File
@@ -0,0 +1,53 @@
from fastapi.testclient import TestClient
from config import Settings
from main import create_app
from rag.answer import GroundedAnswerService
from rag.models import EvidenceDecision, QueryIntent, RetrievalResult, SubjectScope
class FixedRouting:
def retrieve(self, query, subject_scope, intent):
assert query == "Liều thuốc?"
assert subject_scope == SubjectScope.HUMAN
assert intent == QueryIntent.FACT_LOOKUP
return RetrievalResult(EvidenceDecision.ABSTAIN, "drug_not_resolved")
class MemoryTraceWriter:
def __init__(self):
self.rows = []
def save(self, **fields):
self.rows.append(fields)
return "trace-1"
def test_health_and_fail_closed_rag_response_are_traced():
traces = MemoryTraceWriter()
app = create_app(
settings=Settings(),
answer_service=GroundedAnswerService(FixedRouting()),
trace_writer=traces,
)
client = TestClient(app)
assert client.get("/health").json() == {"status": "ok"}
response = client.post("/v1/rag/query", json={
"query": "Liều thuốc?",
"subject_scope": "human",
"intent": "fact_lookup",
})
assert response.status_code == 200
assert response.json()["decision"] == "abstain"
assert response.json()["trace_id"] == "trace-1"
assert traces.rows[0]["reason"] == "drug_not_resolved"
def test_query_requires_structured_scope_and_intent():
app = create_app(
settings=Settings(),
answer_service=GroundedAnswerService(FixedRouting()),
trace_writer=MemoryTraceWriter(),
)
response = TestClient(app).post("/v1/rag/query", json={"query": "Liều?"})
assert response.status_code == 422
+21
View File
@@ -0,0 +1,21 @@
import pytest
from rag.calculators import body_surface_area_m2
def test_bsa_matches_book_worked_example():
# Dược thư Phụ lục 1: "165 cm và 60 kg sẽ có diện tích 1,66 m²".
assert round(body_surface_area_m2(60, 165), 2) == 1.66
def test_bsa_matches_book_table_cells():
# Independent cells read from the BSA table (printed 1499): ground truth.
assert round(body_surface_area_m2(10, 90), 2) == 0.50
assert round(body_surface_area_m2(70, 170), 2) == 1.81
def test_bsa_rejects_nonpositive_inputs():
with pytest.raises(ValueError):
body_surface_area_m2(0, 165)
with pytest.raises(ValueError):
body_surface_area_m2(60, -1)
+146
View File
@@ -0,0 +1,146 @@
"""Follow-ups must inherit context, and must never inherit it silently.
The cases here are the ones the owner named on 2026-08-05: "còn trẻ em thì
sao?", "giải thích kỹ hơn", and not making the user repeat themselves. The
adversarial cases are the ones that make inheritance dangerous in a formulary
— a stale drug, and an explicit mention being overridden by context.
"""
from __future__ import annotations
from rag.conversation import (
FOCUS_TTL_TURNS,
ConversationState,
Focus,
Turn,
detect_population,
detect_verbosity,
looks_like_followup,
resolve_against,
update_focus,
)
def _state(turn_count: int = 1, **focus_fields) -> ConversationState:
focus = Focus()
for name, value in focus_fields.items():
focus = focus.with_field(name, value, turn_count - 1)
return ConversationState("c1", focus=focus, turn_count=turn_count)
# --- the follow-ups the owner asked for --------------------------------------
def test_con_tre_em_thi_sao_inherits_drug_and_section():
state = _state(drug_id="metformin", section_key="lieu_luong_va_cach_dung")
resolved = resolve_against(state, "còn trẻ em thì sao?", None, None)
assert resolved.drug_id == "metformin"
assert resolved.section_key == "lieu_luong_va_cach_dung"
assert resolved.population == "tre_em"
assert resolved.inherited_drug is True
def test_giai_thich_ky_hon_sets_verbosity_and_keeps_the_topic():
state = _state(drug_id="warfarin", section_key="tuong_tac_thuoc")
resolved = resolve_against(state, "giải thích kỹ hơn", None, None)
assert resolved.drug_id == "warfarin"
assert resolved.verbosity == "detailed"
def test_the_user_is_not_made_to_repeat_the_drug():
state = _state(drug_id="metformin")
resolved = resolve_against(state, "chống chỉ định", None, "chong_chi_dinh")
assert resolved.drug_id == "metformin"
assert resolved.section_key == "chong_chi_dinh"
# --- what makes inheritance safe ---------------------------------------------
def test_an_explicit_drug_always_beats_context():
"""Naming a drug must override whatever the conversation was about, or a
deliberate topic change silently answers about the previous medicine."""
state = _state(drug_id="metformin", section_key="lieu_luong_va_cach_dung")
resolved = resolve_against(state, "liều dùng warfarin", "warfarin", None)
assert resolved.drug_id == "warfarin"
assert resolved.inherited_drug is False
def test_a_stale_drug_is_dropped_rather_than_inherited():
"""Beyond the TTL the drug is not context, it is a hazard."""
state = _state(turn_count=FOCUS_TTL_TURNS + 3, drug_id="metformin")
# `_state` stamps at turn_count - 1, so age is 1; age it past the TTL.
aged = ConversationState(
"c1",
focus=Focus(drug_id="metformin", set_at_turn={"drug_id": 0}),
turn_count=FOCUS_TTL_TURNS + 2,
)
assert state.inherited("drug_id") == "metformin"
assert aged.inherited("drug_id") is None
resolved = resolve_against(aged, "còn trẻ em thì sao?", None, None)
assert resolved.drug_id is None
def test_an_inherited_drug_must_be_named_in_the_answer():
state = _state(drug_id="metformin")
inherited = resolve_against(state, "còn trẻ em thì sao?", None, None)
explicit = resolve_against(state, "liều warfarin", "warfarin", None)
assert inherited.needs_carry_over_notice is True
assert explicit.needs_carry_over_notice is False
# --- phrase detection ---------------------------------------------------------
def test_longest_population_phrase_wins():
"""`phụ nữ cho con bú` must not be read as `phụ nữ`, and `trẻ sơ sinh`
must not be read as `trẻ em` — the same rule `sections.py` relies on."""
assert detect_population("phụ nữ cho con bú") == "phu_nu_cho_con_bu"
assert detect_population("trẻ sơ sinh dùng sao") == "tre_so_sinh"
assert detect_population("bà bầu uống được không") == "phu_nu_co_thai"
assert detect_population("liều cho người lớn") == "nguoi_lon"
def test_no_population_named_is_none_not_a_guess():
assert detect_population("liều dùng paracetamol") is None
assert detect_verbosity("liều dùng paracetamol") is None
def test_followup_markers():
assert looks_like_followup("còn trẻ em thì sao?") is True
assert looks_like_followup("so với metformin thì sao") is True
assert looks_like_followup("liều dùng paracetamol") is False
# --- window and focus update --------------------------------------------------
def test_recent_window_evicts_oldest():
state = ConversationState("c1")
for index in range(8):
state = state.append(Turn("user", f"q{index}", "2026-08-05"), window=6)
assert len(state.recent) == 6
assert state.recent[0].text == "q2"
assert state.turn_count == 8
def test_focus_update_stamps_the_current_turn():
state = _state(turn_count=3)
resolved = resolve_against(state, "liều dùng metformin", "metformin", "lieu_luong_va_cach_dung")
focus = update_focus(state, resolved)
assert focus.drug_id == "metformin"
assert focus.set_at_turn["drug_id"] == 3
@@ -0,0 +1,50 @@
from rag.conversation import (
ConversationState,
DeterministicSummariser,
InMemoryConversationStore,
Turn,
)
def test_store_returns_fresh_state_for_unknown_id():
store = InMemoryConversationStore()
state = store.load("conv-new")
assert state.conversation_id == "conv-new"
assert state.turn_count == 0
assert state.recent == ()
def test_store_round_trips_saved_state():
store = InMemoryConversationStore()
state = ConversationState("conv-1", summary="s", turn_count=3)
store.save(state)
assert store.load("conv-1") is state
def test_summariser_records_topic_labels_only():
s = DeterministicSummariser()
dropped = (
Turn("user", "Chống chỉ định của metformin?", "t0",
drug_id="metformin", section_key="chong_chi_dinh"),
Turn("assistant", "Quá mẫn với metformin, suy thận Clcr < 60...", "t1",
drug_id="metformin", section_key="chong_chi_dinh"),
)
out = s.fold("", dropped)
# The label line is present...
assert "chong_chi_dinh của metformin" in out
# ...and no clinical value leaked from the assistant turn.
assert "Clcr" not in out
assert "60" not in out
def test_summariser_stays_within_budget_dropping_oldest():
s = DeterministicSummariser()
dropped = tuple(
Turn("user", f"q{i}", f"t{i}", drug_id=f"drug{i}", section_key="lieu_luong")
for i in range(400)
)
out = s.fold("", dropped)
assert len(out) <= DeterministicSummariser.MAX_CHARS
# Most-recent topic survives, oldest is dropped.
assert "drug399" in out
assert "drug0 " not in out
@@ -0,0 +1,105 @@
from rag.answer import GroundedAnswer
from rag.conversation import DeterministicSummariser, InMemoryConversationStore
from rag.conversational import (
SMALLTALK_REPLY,
ConversationalLoopService,
)
from rag.models import (
Evidence,
EvidenceDecision,
QueryIntent,
RetrievalResult,
SubjectScope,
)
from rag.reasoning import ClarifyReason
from rag.routing import CatalogDrugResolver
from rag.sections import SectionResolver
def _grounded(answer, evidence_text):
ev = Evidence("e1", "e1", "prose", evidence_text, 1.0, (), False, False)
result = RetrievalResult(
EvidenceDecision.ANSWERABLE, "grounded_evidence_available",
(ev,), "metformin", "resolved",
)
return GroundedAnswer(result, answer, (), False)
class FakeAnswers:
def __init__(self, answer_text, evidence_text):
self._a = answer_text
self._e = evidence_text
self.calls = []
def answer(self, query, subject_scope, intent):
self.calls.append(query)
return _grounded(self._a, self._e)
def _service(answers):
return ConversationalLoopService(
answers=answers,
resolver=CatalogDrugResolver({"metformin": {"metformin"}}),
section_resolver=SectionResolver(),
store=InMemoryConversationStore(),
summariser=DeterministicSummariser(),
)
def test_smalltalk_answers_socially_without_calling_engine():
answers = FakeAnswers("x", "x")
svc = _service(answers)
out = svc.answer("c1", "chào bạn", SubjectScope.HUMAN, QueryIntent.FACT_LOOKUP)
assert out.smalltalk is True
assert out.answer == SMALLTALK_REPLY
assert answers.calls == [] # a greeting is not a drug lookup
def test_medical_turn_returns_grounded_answer():
answers = FakeAnswers("Quá mẫn với metformin.", "Quá mẫn với metformin.")
svc = _service(answers)
out = svc.answer(
"c2", "chống chỉ định metformin", SubjectScope.HUMAN, QueryIntent.FACT_LOOKUP
)
assert out.smalltalk is False
assert out.answer == "Quá mẫn với metformin."
assert out.grounded is not None
def test_followup_inherits_drug_and_names_it_and_rewrites_query():
answers = FakeAnswers(
"Ở trẻ em điều chỉnh theo cân nặng.",
"Ở trẻ em, liều metformin điều chỉnh theo cân nặng.",
)
svc = _service(answers)
svc.answer("c3", "chống chỉ định metformin", SubjectScope.HUMAN, QueryIntent.FACT_LOOKUP)
out = svc.answer("c3", "còn trẻ em thì sao?", SubjectScope.HUMAN, QueryIntent.FACT_LOOKUP)
assert out.inherited_drug == "metformin"
assert out.answer.startswith("Về metformin:")
# The follow-up was rewritten self-contained before hitting the engine.
assert "metformin" in answers.calls[-1]
# State carried the drug forward.
assert svc._store.load("c3").focus.drug_id == "metformin"
def test_no_close_drug_reports_not_supported():
answers = FakeAnswers("x", "x")
svc = _service(answers) # catalog holds only metformin
out = svc.answer("c4", "cái này thế nào?", SubjectScope.HUMAN, QueryIntent.FACT_LOOKUP)
assert out.answer is None
assert out.clarification is not None
# Nothing close to a real drug: honest "not in the formulary", not a guess.
assert out.clarification.reason == "drug_not_supported"
assert answers.calls == []
def test_typo_offers_did_you_mean_not_silent_resolution():
answers = FakeAnswers("x", "x")
svc = _service(answers) # catalog holds only metformin
out = svc.answer("c5", "metformim", SubjectScope.HUMAN, QueryIntent.FACT_LOOKUP)
# A near-miss is asked about, never auto-resolved on a similarity threshold.
assert out.answer is None
assert out.clarification is not None
assert out.clarification.reason == "did_you_mean"
assert "Metformin" in out.clarification.options
assert answers.calls == []
@@ -0,0 +1,81 @@
from rag.conversation import DeterministicSummariser, InMemoryConversationStore
from rag.conversational import ConversationalRagService, TurnResolution
from rag.reasoning import ClarifyReason, MAX_LLM_CALLS, MAX_RETRIEVAL_ROUNDS, TurnBudget
class FakeResolver:
"""Maps a turn's text to what it resolves on its own (no context)."""
def __init__(self, table):
self._table = table
def resolve_turn(self, text):
for needle, resolution in self._table:
if needle in text:
return resolution
return TurnResolution(drug_id=None, section_key=None, drug_status="not_found")
def _service(resolver, retrieve, generate):
return ConversationalRagService(
store=InMemoryConversationStore(),
summariser=DeterministicSummariser(),
resolver=resolver,
retrieve=retrieve,
generate=generate,
)
def test_followup_inherits_drug_and_answer_names_it():
resolver = FakeResolver([
("metformin", TurnResolution("metformin", "chong_chi_dinh", "resolved")),
# "còn trẻ em" names no drug on its own — must inherit.
("trẻ em", TurnResolution(None, None, "not_found")),
])
# Evidence mentions "trẻ em" so the population assessor is satisfied.
retrieve = lambda q: ("Ở trẻ em, liều metformin điều chỉnh theo cân nặng.",)
generate = lambda q, ev, st: "liều theo cân nặng"
svc = _service(resolver, retrieve, generate)
first = svc.answer("c1", "Chống chỉ định của metformin?")
assert first.inherited_drug is None
second = svc.answer("c1", "còn trẻ em thì sao?")
assert second.inherited_drug == "metformin"
assert second.answer.startswith("Về metformin:")
def test_no_drug_and_no_context_asks_without_spending_budget():
resolver = FakeResolver([]) # nothing resolves
calls = {"retrieve": 0, "generate": 0}
def retrieve(q):
calls["retrieve"] += 1
return ("x",)
def generate(q, ev, st):
calls["generate"] += 1
return "x"
svc = _service(resolver, retrieve, generate)
budget = TurnBudget()
out = svc.answer("c2", "cái này thế nào?", budget=budget)
assert out.answer is None
assert out.clarification is not None
assert out.clarification.reason == ClarifyReason.AMBIGUOUS_DRUG
# Asking short-circuits before any spend.
assert calls == {"retrieve": 0, "generate": 0}
assert budget.retrieval_rounds == MAX_RETRIEVAL_ROUNDS
assert budget.llm_calls == MAX_LLM_CALLS
def test_state_persists_across_turns():
resolver = FakeResolver([
("metformin", TurnResolution("metformin", "chi_dinh", "resolved")),
])
svc = _service(resolver, lambda q: ("Chỉ định của metformin.",), lambda q, ev, st: "ok")
svc.answer("c3", "chỉ định metformin?")
state = svc._store.load("c3")
assert state.turn_count == 2 # user + assistant
assert state.focus.drug_id == "metformin"
@@ -0,0 +1,103 @@
"""A dead embedding provider must abstain, never crash the request.
Measured 2026-08-05 against the live `duocthu_v1` collection: with Bedrock
access revoked, `Tôi sốt cao, uống Paracetamol được không?` returned
**HTTP 500** from `botocore AccessDeniedException`. The section route needs no
embedder, so only the similarity fallback is affected — but that fallback is
reached by any question whose attribute is not in the phrase table, and an
error page is not an acceptable answer for a clinician.
"""
from __future__ import annotations
import pytest
from adapters.embedding import BedrockCohereQueryEmbedder
from rag.in_memory import InMemoryParentStore
from rag.models import EvidenceDecision
from rag.ports import QueryEmbeddingUnavailable
from rag.service import EvidencePolicy, RetrievalService
class _AccessDenied(Exception):
"""Stands in for botocore's ClientError without importing botocore."""
class _DeadEmbedder:
dimensions = 1024
def embed_query(self, text: str) -> list[float]:
raise QueryEmbeddingUnavailable("provider is revoked")
class _DeadRetriever:
def __init__(self) -> None:
self.calls = 0
def search(self, query: str, drug_id: str, limit: int) -> list:
self.calls += 1
return list(_DeadEmbedder().embed_query(query))
def _service(retriever: _DeadRetriever) -> RetrievalService:
return RetrievalService(
retriever, InMemoryParentStore([]), EvidencePolicy(minimum_score=0.01)
)
def test_similarity_fallback_abstains_when_the_provider_is_unreachable():
retriever = _DeadRetriever()
result = _service(retriever).retrieve("uống được không", "paracetamol")
assert retriever.calls == 1
assert result.decision == EvidenceDecision.ABSTAIN
assert result.reason == "query_embedding_unavailable"
assert result.evidence == ()
def test_abstention_reason_is_distinct_from_a_genuine_no_match():
"""An outage and an empty corpus must not report the same reason.
Reading `insufficient_retrieval_score` when the search never ran would send
anyone debugging this at the corpus instead of at the provider.
"""
result = _service(_DeadRetriever()).retrieve("uống được không", "paracetamol")
assert result.reason != "insufficient_retrieval_score"
def test_bedrock_adapter_translates_provider_errors_into_the_domain_error():
"""The domain must never see a botocore type; the adapter translates."""
class _RefusingClient:
def invoke_model(self, **kwargs):
raise _AccessDenied("not authorized to perform: bedrock:InvokeModel")
embedder = BedrockCohereQueryEmbedder(1024, client=_RefusingClient())
# botocore is installed here, so `_AccessDenied` is deliberately NOT one of
# the translated types: an unrecognised error must still surface loudly
# rather than be silently downgraded to an abstention.
with pytest.raises(_AccessDenied):
embedder.embed_query("liều paracetamol")
def test_real_botocore_client_error_becomes_an_abstainable_domain_error():
botocore_exceptions = pytest.importorskip("botocore.exceptions")
class _RefusingClient:
def invoke_model(self, **kwargs):
raise botocore_exceptions.ClientError(
{
"Error": {
"Code": "AccessDeniedException",
"Message": "not authorized to perform: bedrock:InvokeModel",
}
},
"InvokeModel",
)
embedder = BedrockCohereQueryEmbedder(1024, client=_RefusingClient())
with pytest.raises(QueryEmbeddingUnavailable):
embedder.embed_query("liều paracetamol")
@@ -0,0 +1,210 @@
"""The answer layer may rephrase evidence; it may not add to it.
Every test here is a fabrication the generator could plausibly produce, and
the assertion is that the clinician never sees it. The dose figures are taken
from the real METFORMIN and PARACETAMOL sections in `duocthu_v1`.
"""
from __future__ import annotations
import json
import pytest
from rag import grounding
from rag.answer import GroundedAnswerService
from rag.metrics import GENERATION_REJECTED, GENERATION_SERVED, InMemoryMetrics
from rag.models import (
Evidence,
EvidenceDecision,
QueryIntent,
RetrievalResult,
SourceRef,
SubjectScope,
)
from rag.ports import AnswerGenerationUnavailable
from rag.prompt import build_request
SOURCE = SourceRef(
physical_page=812,
precision="region",
printed_page_range=(714, 714),
)
EVIDENCE_TEXT = (
"Người lớn: uống 500 mg metformin hydroclorid, 2 lần mỗi ngày. "
"Liều tối đa 2 g mỗi ngày, chia làm nhiều lần."
)
def _result(text: str = EVIDENCE_TEXT) -> RetrievalResult:
return RetrievalResult(
EvidenceDecision.ANSWERABLE,
"grounded_evidence_available",
(
Evidence(
evidence_id="metformin::lieu::0",
matched_doc_id="metformin::lieu::0",
kind="prose",
text=text,
score=1.0,
source_refs=(SOURCE,),
hydrated_from_parent=False,
requires_visual_check=False,
),
),
resolved_drug_id="metformin",
)
class _FixedRouting:
def __init__(self, result: RetrievalResult) -> None:
self._result = result
def retrieve(self, query, subject_scope, intent):
return self._result
class _Generator:
"""Returns whatever payload the test wants the model to have produced."""
def __init__(self, payload) -> None:
self._payload = payload
def generate(self, system: str, user: str, schema: dict) -> str:
if isinstance(self._payload, BaseException):
raise self._payload
if isinstance(self._payload, str):
return self._payload
return json.dumps(self._payload, ensure_ascii=False)
def _answer(payload, result: RetrievalResult | None = None):
metrics = InMemoryMetrics()
service = GroundedAnswerService(
_FixedRouting(result or _result()), _Generator(payload), metrics
)
grounded = service.answer("Liều Metformin?", SubjectScope.HUMAN, QueryIntent.FACT_LOOKUP)
return grounded, metrics
# --- the guardrail's whole reason to exist ------------------------------------
def test_invented_dose_is_refused_and_never_reaches_the_answer():
grounded, metrics = _answer(
{"answer": "Người lớn uống 850 mg, 2 lần mỗi ngày [1].",
"evidence_sufficient": True}
)
assert grounded.generated is False
assert "850" not in grounded.answer
assert grounded.answer.startswith(EVIDENCE_TEXT)
assert metrics.total(GENERATION_REJECTED, reason="ungrounded_number") == 1
def test_a_rounded_figure_counts_as_invented():
"""`2 g` is in the source; `2000 mg` is a conversion, and conversions are
where unit errors live. The prompt forbids it and the check enforces it."""
grounded, metrics = _answer(
{"answer": "Liều tối đa 2000 mg mỗi ngày [1].", "evidence_sufficient": True}
)
assert grounded.generated is False
assert metrics.total(GENERATION_REJECTED, reason="ungrounded_number") == 1
def test_citation_pointing_at_nothing_is_refused():
grounded, metrics = _answer(
{"answer": "Người lớn uống 500 mg [3].", "evidence_sufficient": True}
)
assert grounded.generated is False
assert metrics.total(GENERATION_REJECTED, reason="invalid_citation") == 1
def test_faithful_rewrite_is_served():
grounded, metrics = _answer(
{"answer": "Người lớn: 500 mg, 2 lần/ngày; tối đa 2 g/ngày [1].",
"evidence_sufficient": True}
)
assert grounded.generated is True
assert grounded.answer == "Người lớn: 500 mg, 2 lần/ngày; tối đa 2 g/ngày [1]."
assert metrics.total(GENERATION_SERVED) == 1
assert metrics.total(GENERATION_REJECTED) == 0
def test_citations_survive_generation():
"""Provenance is the point; a prettier answer must not cost the folio."""
grounded, _ = _answer(
{"answer": "Người lớn: 500 mg [1].", "evidence_sufficient": True}
)
assert grounded.generated is True
assert len(grounded.citations) == 1
assert grounded.citations[0].printed_page_start == 714
# --- degradation is always to the source, never to an error -------------------
@pytest.mark.parametrize(
"payload, reason",
[
(AnswerGenerationUnavailable("revoked"), "provider_unavailable"),
("not json at all", "malformed_output"),
({"answer": "500 mg [1]"}, "malformed_output"),
({"answer": 500, "evidence_sufficient": True}, "malformed_output"),
({"answer": "...", "evidence_sufficient": False}, "evidence_insufficient"),
],
)
def test_every_generation_failure_falls_back_to_the_source_text(payload, reason):
grounded, metrics = _answer(payload)
assert grounded.generated is False
assert grounded.answer.startswith(EVIDENCE_TEXT)
assert metrics.total(GENERATION_REJECTED, reason=reason) == 1
def test_no_generator_configured_still_answers():
service = GroundedAnswerService(_FixedRouting(_result()))
grounded = service.answer(
"Liều Metformin?", SubjectScope.HUMAN, QueryIntent.FACT_LOOKUP
)
assert grounded.generated is False
assert grounded.answer.startswith(EVIDENCE_TEXT)
# --- the comparison rule itself ----------------------------------------------
def test_decimal_separators_are_not_interchangeable():
"""`7,5` and `7.5` differ, and so do `7,5` and `75`. Normalising them
together is how a tenfold dose error scores as a match."""
source = ("Sơ sinh: 7,5 mg/kg cách 8 giờ/lần.",)
assert grounding.verify("7,5 mg/kg [1]", source).grounded is True
assert grounding.verify("7.5 mg/kg [1]", source).grounded is False
assert grounding.verify("75 mg/kg [1]", source).grounded is False
def test_citation_markers_are_not_read_as_quantities():
report = grounding.verify("Không dùng cho người suy thận [1].", ("Suy thận.",))
assert report.grounded is True
assert report.cited_indices == (1,)
def test_prompt_numbers_evidence_from_one():
request = build_request("Liều?", ("đoạn A", "đoạn B"))
assert "[1] đoạn A" in request.user
assert "[2] đoạn B" in request.user
assert "CHÉP NGUYÊN VĂN" in request.system
def test_prompt_refuses_to_build_without_evidence():
with pytest.raises(ValueError):
build_request("Liều?", ())
@@ -0,0 +1,172 @@
from __future__ import annotations
import json
import os
import uuid
from contextlib import suppress
from functools import lru_cache
from pathlib import Path
import pytest
pytestmark = pytest.mark.skipif(
os.getenv("RUN_INTEGRATION") != "1",
reason="set RUN_INTEGRATION=1 with local PostgreSQL and Qdrant running",
)
ROOT = Path(__file__).resolve().parents[3]
CHUNKS = ROOT / "ingestion/data/processed/chunks.jsonl"
PDF = ROOT / "ingestion/data/raw/duoc-thu-quoc-gia-viet-nam-2018.pdf"
MIGRATION = Path(__file__).resolve().parents[1] / "migrations/001_rag_retrieval_trace.sql"
@lru_cache
def _first_real_chunk() -> dict:
with CHUNKS.open(encoding="utf-8") as handle:
record = json.loads(next(handle))
import fitz
from ingestion.extract.page_map import build_page_map
with fitz.open(PDF) as document:
page_map = build_page_map(document)
physical_start, physical_end = record["source_page_range"]
printed_start = page_map[physical_start]
printed_end = page_map[physical_end]
assert printed_start is not None and printed_end is not None
record["printed_page_range"] = [printed_start, printed_end]
return record
def test_real_qdrant_round_trip_uses_real_chunk_and_printed_folio():
from qdrant_client import QdrantClient
from qdrant_client.models import Distance, PointStruct, VectorParams
from adapters.embedding import LocalHashQueryEmbedder
from adapters.qdrant import QdrantRetriever
client = QdrantClient(url="http://localhost:6333")
collection = f"integration_{uuid.uuid4().hex}"
embedder = LocalHashQueryEmbedder(32)
record = _first_real_chunk()
try:
client.create_collection(
collection_name=collection,
vectors_config=VectorParams(size=32, distance=Distance.COSINE),
)
client.upsert(
collection_name=collection,
points=[PointStruct(
id=str(uuid.uuid4()),
vector=embedder.embed_query(record["text"]),
payload=record,
)],
wait=True,
)
hits = QdrantRetriever(client, collection, embedder).search(
record["text"], record["drug_id"], 3,
)
assert [hit.document.doc_id for hit in hits] == [record["chunk_id"]]
assert hits[0].document.text == record["text"]
assert hits[0].document.source_refs[0].printed_page_range == tuple(
record["printed_page_range"]
)
finally:
with suppress(Exception):
client.delete_collection(collection)
def test_real_postgres_migration_insert_and_read_back():
from adapters.postgres import PostgresTraceRepository
repository = PostgresTraceRepository(
"postgresql://duoc_thu:duoc_thu@localhost:5432/duoc_thu"
)
repository.migrate(MIGRATION)
trace_id = repository.save(
query="Liều abacavir?",
subject_scope="human",
intent="fact_lookup",
decision="answerable",
reason="grounded_evidence_available",
resolved_drug_id="abacavir",
citations=({
"chunk_id": "abacavir__ten_chung_quoc_te__0",
"printed_page_start": 101,
"printed_page_end": 103,
},),
)
stored = repository.get(trace_id)
assert stored is not None
assert stored.query == "Liều abacavir?"
assert stored.resolved_drug_id == "abacavir"
assert stored.citations[0]["printed_page_start"] == 101
def test_api_round_trip_uses_qdrant_and_persists_postgres_trace():
from fastapi.testclient import TestClient
from qdrant_client import QdrantClient
from qdrant_client.models import Distance, PointStruct, VectorParams
from adapters.embedding import LocalHashQueryEmbedder
from adapters.postgres import PostgresTraceRepository
from adapters.qdrant import QdrantParentStore, QdrantRetriever
from config import Settings
from main import create_app
from rag.answer import GroundedAnswerService
from rag.routing import CatalogDrugResolver, QueryRoutingService
from rag.service import EvidencePolicy, RetrievalService
qdrant = QdrantClient(url="http://localhost:6333")
collection = f"integration_{uuid.uuid4().hex}"
embedder = LocalHashQueryEmbedder(32)
record = dict(_first_real_chunk())
traces = PostgresTraceRepository(
"postgresql://duoc_thu:duoc_thu@localhost:5432/duoc_thu"
)
traces.migrate(MIGRATION)
try:
qdrant.create_collection(
collection_name=collection,
vectors_config=VectorParams(size=32, distance=Distance.COSINE),
)
qdrant.upsert(
collection_name=collection,
points=[PointStruct(
id=str(uuid.uuid4()),
vector=embedder.embed_query(record["text"]),
payload=record,
)],
wait=True,
)
retrieval = RetrievalService(
QdrantRetriever(qdrant, collection, embedder),
QdrantParentStore(qdrant, collection),
EvidencePolicy(minimum_score=0.01),
)
answers = GroundedAnswerService(QueryRoutingService(
retrieval,
CatalogDrugResolver({record["drug_id"]: {record["drug_name"]}}),
))
app = create_app(
settings=Settings(), answer_service=answers, trace_writer=traces,
)
response = TestClient(app).post("/v1/rag/query", json={
"query": record["text"],
"subject_scope": "human",
"intent": "fact_lookup",
})
assert response.status_code == 200
body = response.json()
assert body["decision"] == "answerable"
assert body["citations"][0]["chunk_id"] == record["chunk_id"]
assert body["citations"][0]["printed_page_start"] == (
record["printed_page_range"][0]
)
stored = traces.get(body["trace_id"])
assert stored is not None
assert stored.decision == "answerable"
assert stored.citations[0]["chunk_id"] == record["chunk_id"]
finally:
with suppress(Exception):
qdrant.delete_collection(collection)
@@ -0,0 +1,47 @@
from adapters.qdrant import _source_refs
def test_descriptor_source_ref_comes_from_attachment_not_heading_page():
refs = _source_refs({
"chunk_kind": "block_descriptor",
"heading_physical_page": 208,
"source_page_range": [209, 209],
"printed_page_range": [210, 210],
"attachments": [{
"block_id": "p209_t0",
"physical_page": 209,
"printed_page": 210,
"bbox": [49.5, 68.1, 289.4, 789.4],
"source_crop": "crops/p209_t0.png",
}],
})
assert len(refs) == 1
assert refs[0].physical_page == 209
assert refs[0].printed_page == 210
assert refs[0].block_id == "p209_t0"
assert refs[0].bbox == (49.5, 68.1, 289.4, 789.4)
assert refs[0].source_crop == "crops/p209_t0.png"
assert refs[0].precision == "region"
def test_prose_ref_uses_exact_chunk_range_and_keeps_attachment_region():
refs = _source_refs({
"chunk_kind": "prose",
"heading_physical_page": 100,
"source_page_range": [104, 105],
"printed_page_range": [105, 106],
"attachments": [{
"block_id": "p105_t0",
"physical_page": 105,
"printed_page": 106,
"bbox": [1.0, 2.0, 3.0, 4.0],
}],
})
assert refs[0].physical_page == 104
assert refs[0].page_range == (104, 105)
assert refs[0].printed_page_range == (105, 106)
assert refs[1].block_id == "p105_t0"
assert refs[1].physical_page == 105
assert refs[1].printed_page == 106
@@ -0,0 +1,244 @@
"""The loop must improve answers, and must be unable to run away.
Bounded is the load-bearing property: an unbounded self-improvement loop on a
paid provider is a bill and a latency incident, and on a clinical tool it is
also an answer nobody is waiting for any more.
"""
from __future__ import annotations
import pytest
from rag.conversation import ConversationState, ResolvedQuestion
from rag.metrics import CLARIFY_ASKED, LOOP_REFINED, InMemoryMetrics
from rag.reasoning import (
ClarifyReason,
DeterministicAssessor,
LoopTrace,
Sufficiency,
TurnBudget,
run_turn,
)
ADULT = "Người lớn: uống 0,5 - 1 g/lần, cách 4 - 6 giờ; tối đa 4 g/ngày."
CHILD = "Trẻ em 6 - 12 tuổi: 240 - 250 mg mỗi lần."
def _q(text: str = "liều dùng paracetamol", population: str | None = None) -> ResolvedQuestion:
return ResolvedQuestion(
text=text,
drug_id="paracetamol",
section_key="lieu_luong_va_cach_dung",
population=population,
verbosity=None,
inherited_drug=False,
inherited_section=False,
)
def _state() -> ConversationState:
return ConversationState("c1", turn_count=1)
class _Retriever:
"""Returns a different evidence set on each round, recording calls."""
def __init__(self, *rounds: tuple[str, ...]) -> None:
self._rounds = list(rounds)
self.queries: list[str] = []
def __call__(self, resolved: ResolvedQuestion) -> tuple[str, ...]:
self.queries.append(resolved.text)
if self._rounds:
return self._rounds.pop(0)
return ()
def _generator(answer: str | None):
calls = {"n": 0}
def generate(resolved, evidence, state):
calls["n"] += 1
return answer
generate.calls = calls # type: ignore[attr-defined]
return generate
# --- the loop earns its rounds ------------------------------------------------
def test_a_named_gap_buys_exactly_one_more_round():
"""Asked for adults, first round returned only paediatric text."""
retriever = _Retriever((CHILD,), (ADULT, CHILD))
metrics = InMemoryMetrics()
outcome = run_turn(
_state(),
_q(population="nguoi_lon"),
retriever,
_generator("Người lớn: 0,5 - 1 g/lần [1]"),
metrics=metrics,
)
assert outcome.retrieval_rounds_used == 2
assert outcome.generated is True
assert metrics.total(LOOP_REFINED, missing="population:nguoi_lon") == 1
assert retriever.queries[1] != retriever.queries[0]
def test_a_satisfied_question_spends_one_round_only():
retriever = _Retriever((ADULT,))
outcome = run_turn(
_state(), _q(population="nguoi_lon"), retriever, _generator("ok [1]")
)
assert outcome.retrieval_rounds_used == 1
assert outcome.stopped_because == "sufficient"
def test_a_simple_question_does_not_loop():
"""No population asked for means nothing to be missing."""
retriever = _Retriever((ADULT, CHILD))
outcome = run_turn(_state(), _q(), retriever, _generator("ok [1]"))
assert outcome.retrieval_rounds_used == 1
# --- the loop cannot run away -------------------------------------------------
def test_retrieval_rounds_are_hard_capped():
"""Evidence never satisfies the assessor; the loop must still stop."""
retriever = _Retriever((CHILD,), (CHILD,), (CHILD,), (CHILD,), (CHILD,))
outcome = run_turn(
_state(),
_q(population="nguoi_lon"),
retriever,
_generator("ok [1]"),
budget=TurnBudget(retrieval_rounds=2),
)
assert outcome.retrieval_rounds_used == 2
assert outcome.stopped_because == "retrieval_budget"
assert len(retriever.queries) == 2
def test_repairs_are_hard_capped_and_degrade_to_no_answer():
"""`generate` returning None means verification refused it every time."""
generate = _generator(None)
outcome = run_turn(
_state(),
_q(),
_Retriever((ADULT,)),
generate,
budget=TurnBudget(repairs=1, llm_calls=4),
)
assert outcome.answer is None
assert outcome.repairs_used == 1
assert generate.calls["n"] == 2 # first attempt + one repair
assert outcome.stopped_because == "repair_budget"
def test_llm_call_budget_stops_generation_entirely():
generate = _generator(None)
outcome = run_turn(
_state(), _q(), _Retriever((ADULT,)), generate, budget=TurnBudget(llm_calls=0)
)
assert generate.calls["n"] == 0
assert outcome.stopped_because == "llm_budget"
def test_a_refinement_that_changes_nothing_stops_the_loop():
"""Guards against a loop that keeps re-issuing the same query."""
class _SameQuery:
def assess(self, resolved, evidence):
return Sufficiency(False, missing="x", refined_query=resolved.text)
retriever = _Retriever((CHILD,), (CHILD,))
outcome = run_turn(
_state(), _q(), retriever, _generator("ok [1]"), assessor=_SameQuery()
)
assert outcome.stopped_because == "query_unchanged"
assert len(retriever.queries) == 1
def test_an_unnamed_gap_does_not_buy_a_round():
""""Feels incomplete" is not a reason to spend the budget."""
class _Vague:
def assess(self, resolved, evidence):
return Sufficiency(False)
retriever = _Retriever((CHILD,), (CHILD,))
outcome = run_turn(_state(), _q(), retriever, _generator("ok [1]"), assessor=_Vague())
assert outcome.stopped_because == "no_actionable_gap"
assert len(retriever.queries) == 1
# --- clarify beats guessing ---------------------------------------------------
@pytest.mark.parametrize(
"signal",
[ClarifyReason.NO_ATTRIBUTE, ClarifyReason.AMBIGUOUS_DRUG, ClarifyReason.MULTI_ATTRIBUTE],
)
def test_a_clarify_signal_short_circuits_before_any_spend(signal):
retriever = _Retriever((ADULT,))
generate = _generator("ok [1]")
metrics = InMemoryMetrics()
budget = TurnBudget()
outcome = run_turn(
_state(), _q(), retriever, generate, clarify_signals=(signal,), budget=budget, metrics=metrics
)
assert outcome.clarification is not None
assert outcome.clarification.reason == signal
assert outcome.answer is None
assert retriever.queries == []
assert generate.calls["n"] == 0
assert budget.llm_calls == 4 and budget.retrieval_rounds == 2
assert metrics.total(CLARIFY_ASKED, reason=signal) == 1
def test_no_evidence_at_all_asks_rather_than_abstaining_silently():
outcome = run_turn(_state(), _q(), _Retriever(()), _generator("ok [1]"))
assert outcome.clarification is not None
assert outcome.clarification.reason == ClarifyReason.STILL_INSUFFICIENT
assert outcome.stopped_because == "no_evidence"
# --- the deterministic assessor ----------------------------------------------
def test_assessor_only_reports_gaps_it_can_demonstrate():
assessor = DeterministicAssessor()
assert assessor.assess(_q(population="nguoi_lon"), (ADULT,)).sufficient is True
assert assessor.assess(_q(population="nguoi_lon"), (CHILD,)).sufficient is False
# No population asked for: nothing can be shown missing.
assert assessor.assess(_q(), (CHILD,)).sufficient is True
def test_trace_records_the_stages_walked():
trace = LoopTrace()
run_turn(_state(), _q(), _Retriever((ADULT,)), _generator("ok [1]"), trace=trace)
assert trace.stages[0] == "understand"
assert "retrieve" in trace.stages
assert "assess" in trace.stages
assert trace.stages[-1] == "generate"
@@ -0,0 +1,265 @@
from pathlib import Path
from rag.artifacts import load_aliases
from rag.evaluation import CaseOrigin, EvaluationCase, EvaluationOutcome, summarize
from rag.in_memory import InMemoryLexicalRetriever, InMemoryParentStore, _char_ngrams
from rag.models import (
EvidenceDecision,
ParentDocument,
QueryIntent,
RetrievalDocument,
SearchHit,
SourceRef,
SubjectScope,
)
from rag.routing import (
CatalogDrugResolver,
DrugResolutionStatus,
QueryRoutingService,
)
from rag.service import EvidencePolicy, RetrievalService
SOURCE = SourceRef(
physical_page=112,
precision="region",
block_id="p112_t0",
bbox=(1, 2, 3, 4),
source_crop="crops/p112_t0.png",
)
VERIFIED_ENTITIES = (
Path(__file__).parents[3] / "ingestion/data/verified/drug_entities.json"
)
def table_service(*, visual: bool = False) -> RetrievalService:
row = RetrievalDocument(
doc_id="p112_t0::row::0",
parent_id="p112_t0",
drug_id="acetylcystein",
kind="table_row",
section_key="lieu_luong_va_cach_dung",
text="ACETYLCYSTEIN thể trọng 40 đến 49 kg thể tích 34 ml",
source_refs=(SOURCE,),
requires_visual_check=visual,
)
parent = ParentDocument(
parent_id="p112_t0",
kind="table",
text="| Thể trọng | Thể tích |\n| 40 - 49 kg | 34 ml |",
source_refs=(SOURCE,),
)
return RetrievalService(
InMemoryLexicalRetriever([row]),
InMemoryParentStore([parent]),
EvidencePolicy(minimum_score=0.01),
)
def test_row_hit_hydrates_complete_parent_and_keeps_citation():
result = table_service().retrieve("acetylcystein 45 kg bao nhiêu ml", "acetylcystein")
assert result.decision == EvidenceDecision.ANSWERABLE
assert result.evidence[0].hydrated_from_parent is True
assert result.evidence[0].text.startswith("| Thể trọng")
assert result.evidence[0].source_refs == (SOURCE,)
def test_visual_risk_routes_to_pdf_verifier():
result = table_service(visual=True).retrieve(
"acetylcystein 45 kg bao nhiêu ml", "acetylcystein",
)
assert result.decision == EvidenceDecision.VERIFY_PDF
assert result.reason == "visual_verification_required"
def test_missing_parent_abstains_instead_of_answering_from_row_fragment():
row = RetrievalDocument(
doc_id="row", parent_id="missing", drug_id="drug", kind="table_row",
section_key="dose", text="drug dose 10 mg", source_refs=(SOURCE,),
)
service = RetrievalService(
InMemoryLexicalRetriever([row]), InMemoryParentStore([]),
EvidencePolicy(minimum_score=0.01),
)
result = service.retrieve("drug dose", "drug")
assert result.decision == EvidenceDecision.ABSTAIN
assert result.reason == "parent_hydration_failed"
def test_missing_provenance_abstains():
document = RetrievalDocument(
doc_id="prose", drug_id="drug", kind="prose", section_key="dose",
text="drug dose 10 mg", source_refs=(),
)
service = RetrievalService(
InMemoryLexicalRetriever([document]), InMemoryParentStore([]),
EvidencePolicy(minimum_score=0.01),
)
result = service.retrieve("drug dose", "drug")
assert result.decision == EvidenceDecision.ABSTAIN
assert result.reason == "missing_provenance"
class FixedRetriever:
def __init__(self, hits: list[SearchHit]) -> None:
self._hits = hits
def search(self, query: str, drug_id: str, limit: int) -> list[SearchHit]:
del query, drug_id
return self._hits[:limit]
def test_near_tied_different_sources_are_returned_for_evidence_grading():
first = RetrievalDocument("a", "drug", "prose", "A", "dose", (SOURCE,))
second = RetrievalDocument("b", "drug", "prose", "B", "dose", (SOURCE,))
service = RetrievalService(
FixedRetriever([SearchHit(first, 0.50), SearchHit(second, 0.495)]),
InMemoryParentStore([]),
)
result = service.retrieve("dose", "drug")
assert result.decision == EvidenceDecision.ANSWERABLE
assert [item.evidence_id for item in result.evidence] == ["a", "b"]
def test_source_derived_cases_do_not_inflate_release_gate_metric():
outcomes = [
EvaluationOutcome(
EvaluationCase(
"expert-1", "q", "drug", "right", CaseOrigin.EXPERT,
SubjectScope.HUMAN,
),
("wrong",),
),
EvaluationOutcome(
EvaluationCase(
"generated-1", "q", "drug", "right", CaseOrigin.SOURCE_DERIVED,
SubjectScope.HUMAN,
),
("right",),
),
]
report = summarize(outcomes)
assert report["expert_release_gate"]["recall_at_1"] == 0.0
assert report["source_derived_diagnostic"]["recall_at_1"] == 1.0
assert report["manual_routing_diagnostic"]["cases"] == 0
def test_character_ngrams_preserve_word_order():
assert _char_ngrams("beta alpha") != _char_ngrams("alpha beta")
def test_drug_resolver_handles_a_typo_without_fixture_drug_id():
resolver = CatalogDrugResolver({"famciclovir": {"famciclovir"}})
result = resolver.resolve("famciclovia chỉnh liều khi ClCr 20")
assert result.status == DrugResolutionStatus.RESOLVED
assert result.drug_id == "famciclovir"
def test_drug_resolver_does_not_guess_when_query_mentions_two_drugs():
resolver = CatalogDrugResolver({
"oresol": {"oresol"},
"natri_clorid": {"natri clorid"},
})
result = resolver.resolve("oresol có bao nhiêu natri clorid")
assert result.status == DrugResolutionStatus.AMBIGUOUS
def test_verified_aliases_reach_common_parenthesized_drug_names():
resolver = CatalogDrugResolver(load_aliases(VERIFIED_ENTITIES))
assert resolver.resolve("Liều paracetamol cho người lớn").drug_id == (
"paracetamol_acetaminophen"
)
assert resolver.resolve("Chống chỉ định aspirin").drug_id == (
"acid_acetylsalicylic_aspirin"
)
assert resolver.resolve("Công thức oresol").drug_id == (
"thuoc_uong_bu_nuoc_va_ien_giai"
)
def test_verified_catalog_protects_canonical_substring_traps():
resolver = CatalogDrugResolver(load_aliases(VERIFIED_ENTITIES))
traps = {
"homatropin hydrobromid": "homatropin_hydrobromid",
"hydroclorothiazid": "hydroclorothiazid",
"flucloxacilin": "flucloxacilin",
"pseudoephedrin": "pseudoephedrin",
"ethinylestradiol": "ethinylestradiol",
"desloratadin": "desloratadin",
"ciprofloxacin": "ciprofloxacin",
"levofloxacin": "levofloxacin",
"esomeprazol": "esomeprazol",
"methylprednisolon": "methylprednisolon",
"medroxyprogesteron acetat": "medroxyprogesteron_acetat",
"methyltestosteron": "methyltestosteron",
"oxytetracyclin": "oxytetracyclin",
}
for query, expected_id in traps.items():
result = resolver.resolve(query)
assert result.status == DrugResolutionStatus.RESOLVED
assert result.drug_id == expected_id
def test_asymmetric_evidence_resolves_subject_and_component():
ors = RetrievalDocument(
doc_id="ors", drug_id="ors", kind="prose", section_key="formula",
text="Oresol chứa natri clorid", source_refs=(SOURCE,),
)
sodium = RetrievalDocument(
doc_id="sodium", drug_id="sodium", kind="prose", section_key="dose",
text="Natri clorid dùng đường truyền", source_refs=(SOURCE,),
)
routed = QueryRoutingService(
RetrievalService(
InMemoryLexicalRetriever([ors, sodium]), InMemoryParentStore([]),
EvidencePolicy(minimum_score=0.01),
),
CatalogDrugResolver({"ors": {"oresol"}, "sodium": {"natri clorid"}}),
)
result = routed.retrieve(
"Oresol có bao nhiêu natri clorid?",
SubjectScope.HUMAN,
QueryIntent.FACT_LOOKUP,
)
assert result.decision == EvidenceDecision.ANSWERABLE
assert result.resolved_drug_id == "ors"
def test_structured_scope_fails_closed_and_rejects_non_human_subject():
document = RetrievalDocument(
doc_id="dose", drug_id="famciclovir", drug_name="FAMCICLOVIR",
kind="prose", text="Famciclovir liều cho người lớn", section_key="dose",
source_refs=(SOURCE,),
)
routed = QueryRoutingService(
RetrievalService(
InMemoryLexicalRetriever([document]), InMemoryParentStore([]),
EvidencePolicy(minimum_score=0.01),
),
CatalogDrugResolver({"famciclovir": {"famciclovir"}}),
)
veterinary = routed.retrieve(
"Liều famciclovir cho mèo", SubjectScope.NON_HUMAN,
)
unknown = routed.retrieve("Liều famciclovir")
adult = routed.retrieve(
"Liều famciclovir cho người lớn", SubjectScope.HUMAN,
QueryIntent.FACT_LOOKUP,
)
assert veterinary.decision == EvidenceDecision.ABSTAIN
assert veterinary.reason == "out_of_scope_non_human"
assert unknown.decision == EvidenceDecision.ABSTAIN
assert unknown.reason == "subject_scope_unknown"
assert adult.decision == EvidenceDecision.ANSWERABLE
assert adult.resolved_drug_id == "famciclovir"
def test_recommendation_intent_is_refused_at_policy_boundary():
routed = QueryRoutingService(
table_service(), CatalogDrugResolver({"drug": {"drug"}}),
)
result = routed.retrieve(
"Nên dùng drug nào?", SubjectScope.HUMAN, QueryIntent.RECOMMENDATION,
)
assert result.decision == EvidenceDecision.ABSTAIN
assert result.reason == "recommendation_out_of_scope"
@@ -0,0 +1,76 @@
"""A section must be served in the order it was written.
Found 2026-08-05 by reading a real answer in the UI rather than a test:
`liều dùng paracetamol` opened mid-sentence on `5 - 12 tuổi:` and buried
`Liều lượng: Người lớn:` seven hundred words down. Qdrant scrolls in point-id
order and point ids are `uuid5(chunk_id)`, so PARACETAMOL's five dosing parts
came back **3, 4, 1, 2, 0**.
This is a clinical defect, not a cosmetic one: a reader who stops partway
through stops in the middle of a different population's dose.
"""
from __future__ import annotations
from adapters.qdrant import QdrantRetriever
class _ScrambledClient:
"""Returns parts out of order, the way a real scroll did."""
def __init__(self, part_indices: list[int], include_index: bool = True) -> None:
self._payloads = [
{
"chunk_id": f"paracetamol__lieu__{index}",
"drug_id": "paracetamol",
"section_key": "lieu_luong_va_cach_dung",
"chunk_kind": "prose",
"text": f"part {index}",
"source_refs": [{"physical_page": 1120, "precision": "page"}],
**({"part_index": index} if include_index else {}),
}
for index in part_indices
]
def scroll(self, **kwargs):
points = [type("P", (), {"payload": payload})() for payload in self._payloads]
return points, None
class _Embedder:
dimensions = 4
def embed_query(self, text: str) -> list[float]: # never used by this route
raise AssertionError("find_by_section must not embed anything")
def _hits(part_indices: list[int], include_index: bool = True) -> list[str]:
retriever = QdrantRetriever(
_ScrambledClient(part_indices, include_index), "duocthu_v1", _Embedder()
)
return [
hit.document.text
for hit in retriever.find_by_section("paracetamol", "lieu_luong_va_cach_dung")
]
def test_the_exact_scramble_observed_against_the_real_collection():
assert _hits([3, 4, 1, 2, 0]) == [
"part 0",
"part 1",
"part 2",
"part 3",
"part 4",
]
def test_an_already_ordered_section_is_left_alone():
assert _hits([0, 1, 2, 3]) == ["part 0", "part 1", "part 2", "part 3"]
def test_a_part_missing_its_index_is_kept_and_sorted_last():
"""Dropping it would silently shorten a dose list, which is the one
outcome worse than showing it out of order."""
texts = _hits([1, 0], include_index=False)
assert len(texts) == 2
assert set(texts) == {"part 0", "part 1"}
@@ -0,0 +1,218 @@
"""Section routing: the fix for hit@1 0.05 on `chong_chi_dinh`.
Measured 2026-08-04 on the real Cohere collection, letting vector similarity
choose the section answered contraindication questions correctly 1 time in 20.
These tests pin the two properties that make filtering safe: the longer phrase
always wins, and an unrecognised question routes nowhere rather than guessing.
"""
from __future__ import annotations
from rag.models import (
EvidenceDecision,
RetrievalDocument,
SearchHit,
SourceRef,
)
from rag.in_memory import InMemoryParentStore
from rag.sections import SectionResolver
from rag.service import EvidencePolicy, RetrievalService
SOURCE = SourceRef(physical_page=200, precision="page", printed_page=142)
def _doc(doc_id: str, section_key: str, text: str) -> RetrievalDocument:
return RetrievalDocument(
doc_id=doc_id,
parent_id=None,
drug_id="aspirin",
kind="prose",
section_key=section_key,
text=text,
source_refs=(SOURCE,),
requires_visual_check=False,
)
class SectionAwareRetriever:
"""Fake that records which route the service actually took."""
def __init__(self, docs: list[RetrievalDocument]) -> None:
self._docs = docs
self.search_calls: list[str] = []
self.section_calls: list[tuple[str, str]] = []
def search(
self, query: str, drug_id: str, limit: int # noqa: ARG002 — Retriever protocol
) -> list[SearchHit]:
self.search_calls.append(query)
# Deliberately wrong on purpose: the whole point is that the section
# route must not consult similarity at all.
return [SearchHit(self._docs[-1], 0.99)]
def find_by_section(self, drug_id: str, section_key: str) -> list[SearchHit]:
self.section_calls.append((drug_id, section_key))
return [
SearchHit(doc, 1.0) for doc in self._docs if doc.section_key == section_key
]
class SimilarityOnlyRetriever:
def __init__(self, docs: list[RetrievalDocument]) -> None:
self._docs = docs
self.search_calls: list[str] = []
def search(
self, query: str, drug_id: str, limit: int # noqa: ARG002 — Retriever protocol
) -> list[SearchHit]:
self.search_calls.append(query)
return [SearchHit(self._docs[0], 0.99)]
CONTRA = [
_doc("c1", "chong_chi_dinh", "Mẫn cảm với aspirin."),
_doc("c2", "chong_chi_dinh", "Loét dạ dày tá tràng đang tiến triển."),
_doc("c3", "chong_chi_dinh", "Hen do aspirin."),
_doc("c4", "chong_chi_dinh", "Suy gan nặng."),
_doc("c5", "chong_chi_dinh", "Trẻ em dưới 16 tuổi có sốt virus."),
]
INDICATION = [_doc("i1", "chi_dinh", "Giảm đau, hạ sốt, chống viêm.")]
PHARMACOLOGY = [_doc("p1", "duoc_ly_va_co_che_tac_dung", "Ức chế cyclooxygenase.")]
ALL_DOCS = CONTRA + INDICATION + PHARMACOLOGY
def _service(retriever, resolver: SectionResolver | None) -> RetrievalService:
return RetrievalService(
retriever,
InMemoryParentStore([]),
EvidencePolicy(minimum_score=0.01),
section_resolver=resolver,
)
class TestSectionResolver:
def test_contraindication_is_never_read_as_indication(self) -> None:
"""The one that measured 0.05. "chống chỉ định" contains "chỉ định"."""
resolver = SectionResolver()
assert resolver.resolve("Chống chỉ định của aspirin là gì?").section_key == (
"chong_chi_dinh"
)
assert resolver.resolve("Chỉ định của aspirin?").section_key == "chi_dinh"
def test_works_without_diacritics(self) -> None:
assert SectionResolver().resolve("aspirin chong chi dinh").section_key == (
"chong_chi_dinh"
)
def test_overdose_is_not_read_as_dose(self) -> None:
resolver = SectionResolver()
assert resolver.resolve("xử trí quá liều metformin").section_key == (
"qua_lieu_va_xu_tri"
)
assert resolver.resolve("liều dùng metformin").section_key == (
"lieu_luong_va_cach_dung"
)
def test_adr_management_is_not_read_as_adr_itself(self) -> None:
resolver = SectionResolver()
assert resolver.resolve("xử trí tác dụng phụ của prednisolon").section_key == (
"huong_dan_xu_tri_adr"
)
assert resolver.resolve("tác dụng phụ của prednisolon").section_key == (
"tac_dung_khong_mong_muon"
)
def test_incompatibility_is_not_read_as_interaction(self) -> None:
resolver = SectionResolver()
assert resolver.resolve("tương kỵ của ceftriaxon").section_key == "tuong_ky"
assert resolver.resolve("tương tác của ceftriaxon").section_key == (
"tuong_tac_thuoc"
)
def test_bare_lieu_resolves_without_capturing_overdose(self) -> None:
"""Found by testing on human-written golden questions, not templates.
4 of 16 said just "Liều Metformin cho người lớn?". Adding bare "liều"
is only safe because "quá liều" is longer and is tested first.
"""
resolver = SectionResolver()
assert resolver.resolve("Liều Metformin cho người lớn?").section_key == (
"lieu_luong_va_cach_dung"
)
assert resolver.resolve("quá liều paracetamol").section_key == (
"qua_lieu_va_xu_tri"
)
def test_colloquial_pregnancy_phrasing(self) -> None:
assert SectionResolver().resolve(
"Bà bầu dùng Ibuprofen được không?"
).section_key == "thoi_ky_mang_thai"
def test_unrecognised_question_routes_nowhere(self) -> None:
"""No match must not become a guess — the caller falls back."""
assert SectionResolver().resolve("thuốc này giá bao nhiêu") is None
assert SectionResolver().resolve("") is None
def test_new_section_needs_no_code_change(self) -> None:
resolver = SectionResolver({"invented_section": ("một mục hoàn toàn mới",)})
assert resolver.resolve("hỏi về một mục hoàn toàn mới").section_key == (
"invented_section"
)
class TestSectionRouting:
def test_named_section_bypasses_similarity_entirely(self) -> None:
retriever = SectionAwareRetriever(ALL_DOCS)
result = _service(retriever, SectionResolver()).retrieve(
"Chống chỉ định của aspirin là gì?", "aspirin"
)
assert retriever.section_calls == [("aspirin", "chong_chi_dinh")]
assert retriever.search_calls == []
assert result.decision == EvidenceDecision.ANSWERABLE
assert {item.evidence_id for item in result.evidence} == {
"c1", "c2", "c3", "c4", "c5",
}
def test_whole_section_is_returned_past_the_evidence_limit(self) -> None:
"""Five contraindications must not arrive as three."""
retriever = SectionAwareRetriever(ALL_DOCS)
service = RetrievalService(
retriever,
InMemoryParentStore([]),
EvidencePolicy(minimum_score=0.01, evidence_limit=3),
section_resolver=SectionResolver(),
)
result = service.retrieve("chống chỉ định aspirin", "aspirin")
assert len(result.evidence) == 5
def test_unnamed_section_falls_back_to_similarity(self) -> None:
retriever = SectionAwareRetriever(ALL_DOCS)
result = _service(retriever, SectionResolver()).retrieve(
"aspirin dùng cho bệnh nhân này thế nào", "aspirin"
)
assert retriever.section_calls == []
assert retriever.search_calls
assert result.decision == EvidenceDecision.ANSWERABLE
def test_retriever_without_the_capability_still_works(self) -> None:
retriever = SimilarityOnlyRetriever(ALL_DOCS)
result = _service(retriever, SectionResolver()).retrieve(
"Chống chỉ định của aspirin là gì?", "aspirin"
)
assert retriever.search_calls
assert result.decision == EvidenceDecision.ANSWERABLE
def test_no_resolver_keeps_the_old_behaviour(self) -> None:
retriever = SectionAwareRetriever(ALL_DOCS)
_service(retriever, None).retrieve("chống chỉ định aspirin", "aspirin")
assert retriever.section_calls == []
assert retriever.search_calls
def test_named_but_empty_section_falls_back(self) -> None:
"""A drug with no such section must not abstain — similarity still tries."""
retriever = SectionAwareRetriever(INDICATION + PHARMACOLOGY)
result = _service(retriever, SectionResolver()).retrieve(
"chống chỉ định aspirin", "aspirin"
)
assert retriever.section_calls == [("aspirin", "chong_chi_dinh")]
assert retriever.search_calls
assert result.decision == EvidenceDecision.ANSWERABLE