Add read-only production runtime audit

This commit is contained in:
2026-08-17 11:17:40 +07:00
parent 057d4ed9dc
commit a1de4715a4
106 changed files with 6869 additions and 1782 deletions
+62 -2
View File
@@ -127,7 +127,8 @@ def test_veterinary_phrase_abstains_even_if_the_model_missed_it():
agent = _agent(QueryFrame(turn_type="drug_attribute", drugs=("metformin",)))
reply = agent.handle("liều metformin cho chó bao nhiêu")
assert reply.decision == "abstain"
assert reply.reason == "out_of_scope"
assert reply.reason == "out_of_scope_non_human"
assert "chỉ bao phủ thuốc dùng cho người" in reply.answer
def test_unknown_drug_name_is_reported_not_substituted():
@@ -145,7 +146,7 @@ def test_no_drug_named_asks_which_one():
assert reply.reason == "no_drug"
def test_drug_attribute_without_an_attribute_does_not_fall_into_overview_retrieval():
def test_drug_attribute_without_an_attribute_keeps_ai_clarification_without_retrieval():
retrieval = _FixedRetrieval({})
answers = GroundedAnswerService(routing=None)
agent = RagAgent(
@@ -163,6 +164,65 @@ def test_drug_attribute_without_an_attribute_does_not_fall_into_overview_retriev
assert retrieval.calls == []
def test_bare_drug_overview_opens_section_picker_without_retrieval():
retrieval = _FixedRetrieval({})
agent = RagAgent(
_FixedUnderstander(QueryFrame(
turn_type="drug_overview", drugs=("metformin",)
)),
retrieval,
GroundedAnswerService(routing=None),
)
reply = agent.handle("Metformin", response_mode="monograph")
assert reply.decision == "clarify"
assert reply.reason == "select_drug_sections"
assert reply.drugs == ("metformin",)
assert retrieval.calls == []
def test_monograph_bare_drug_ignores_stale_inherited_attribute():
retrieval = _FixedRetrieval({})
agent = RagAgent(
_FixedUnderstander(QueryFrame(
turn_type="drug_attribute",
drugs=("metformin",),
attribute="chong_chi_dinh",
)),
retrieval,
GroundedAnswerService(routing=None),
)
reply = agent.handle("Metformin", response_mode="monograph")
assert reply.reason == "select_drug_sections"
assert retrieval.calls == []
def test_monograph_mode_keeps_explicit_attribute_on_ai_route():
result = RetrievalResult(
EvidenceDecision.ABSTAIN, "not_configured", resolved_drug_id="metformin"
)
retrieval = _FixedRetrieval({"metformin": result})
agent = RagAgent(
_FixedUnderstander(QueryFrame(
turn_type="drug_attribute",
drugs=("metformin",),
attribute="chong_chi_dinh",
)),
retrieval,
GroundedAnswerService(routing=None),
)
reply = agent.handle(
"Chống chỉ định của Metformin là gì?", response_mode="monograph"
)
assert reply.reason != "select_drug_sections"
assert retrieval.calls[0][1] == "chong_chi_dinh"
# --- the pediatric dosing gate. This code path gained its first test
# coverage on 2026-08-11, after driving production reproduced the same
# behaviour 5/5: the clarify question asked for both age and weight every
+62 -5
View File
@@ -5,7 +5,7 @@ from fastapi.testclient import TestClient
from adapters.prometheus import PrometheusMetrics
from adapters.postgres import FeedbackTraceNotFound, RetrievalTrace
from config import Settings
from main import create_app
from main import _route_label, create_app
from rag.agent import AgentReply
from rag.answer import DISCLAIMER, Citation, GroundedAnswerService
from rag.metrics import TRACE_WRITE_FAILED, InMemoryMetrics
@@ -150,6 +150,18 @@ def test_history_for_unknown_conversation_is_empty_not_an_error():
assert response.json() == {"items": []}
def test_history_rejects_an_oversized_conversation_id_before_querying_storage():
traces = FakeHistoryTraceWriter({})
app = create_app(settings=Settings(), trace_writer=traces)
response = TestClient(app).get(
"/v1/rag/history", params={"conversation_id": "x" * 129}
)
assert response.status_code == 422
assert traces.calls == []
def test_health_and_fail_closed_rag_response_are_traced():
traces = MemoryTraceWriter()
app = create_app(
@@ -204,6 +216,19 @@ def _metrics_app(**settings_kwargs):
)
def test_all_public_rag_endpoints_have_bounded_request_metric_labels():
paths = (
"/v1/rag/query",
"/v1/rag/suggest",
"/v1/rag/feedback",
"/v1/rag/history",
"/v1/rag/sections",
"/v1/rag/section-text",
)
assert {_route_label(path) for path in paths} == set(paths)
def test_metrics_stays_open_when_no_token_is_configured():
"""The default must not break the existing Compose scrape or local runs —
the endpoint is not internet-reachable in that topology."""
@@ -252,10 +277,15 @@ class FakeAgent:
def __init__(self, reply: AgentReply) -> None:
self._reply = reply
self.calls: list[tuple[str, str | None]] = []
self.calls: list[tuple[str, str | None, str]] = []
def handle(self, turn: str, conversation_id: str | None = None) -> AgentReply:
self.calls.append((turn, conversation_id))
def handle(
self,
turn: str,
conversation_id: str | None = None,
response_mode: str = "ai",
) -> AgentReply:
self.calls.append((turn, conversation_id, response_mode))
return self._reply
def complete(self, prefix: str, k: int = 8) -> list[str]:
@@ -291,7 +321,34 @@ def test_query_routes_through_the_agent_when_one_is_configured():
assert body["answer"] == "Liều 500 mg [1]."
assert body["resolved_drug_id"] == "metformin"
assert len(body["citations"]) == 1
assert agent.calls == [("Liều metformin?", "c1")]
assert agent.calls == [("Liều metformin?", "c1", "ai")]
def test_query_forwards_monograph_response_mode_to_agent():
agent = FakeAgent(AgentReply(
decision="clarify",
reason="select_drug_sections",
clarification="Chọn mục cần xem.",
drugs=("metformin",),
turn_type="drug_overview",
))
app = create_app(
settings=Settings(),
answer_service=GroundedAnswerService(FixedRouting()),
conversational=agent,
trace_writer=MemoryTraceWriter(),
)
response = TestClient(app).post("/v1/rag/query", json={
"query": "Metformin",
"subject_scope": "human",
"intent": "fact_lookup",
"response_mode": "monograph",
})
assert response.status_code == 200
assert response.json()["reason"] == "select_drug_sections"
assert agent.calls == [("Metformin", None, "monograph")]
def test_query_agent_clarification_is_surfaced_as_the_answer():
@@ -11,6 +11,7 @@ from rag.clinical import (
ConditionNormalizer,
ConditionQuery,
ConditionRelation,
HepaticContext,
PatientContext,
RenalContext,
)
@@ -20,6 +21,8 @@ from rag.understanding import (
LlmQueryUnderstander,
QueryFrame,
_apply_condition_candidate_cue,
_apply_contextual_candidate_safety,
_apply_general_condition_scope,
_apply_named_drug_cues,
_apply_reverse_relation_cues,
_merge_with_prior_frame,
@@ -72,9 +75,104 @@ def test_condition_normalizer_handles_professional_aliases_without_drug_mapping(
assert normalizer.normalize("THA dùng gì", "THA").normalized_condition == "tăng huyết áp"
assert normalizer.normalize("cao huyết áp", "cao huyết áp").normalized_condition == "tăng huyết áp"
assert normalizer.normalize("Gout", "gout").normalized_condition == "gút"
assert normalizer.detect_known_alias("BN viêm phổi dùng thuốc gì?").normalized_condition == "viêm phổi"
assert normalizer.normalize("bệnh lạ", "bệnh lạ").normalized_condition == "bệnh lạ"
def test_explicit_indication_relation_is_a_condition_candidate_lookup():
noisy = QueryFrame(
turn_type="condition_relation",
condition=ConditionNormalizer().normalize("bệnh gút", "gút"),
needs_clarify=True,
clarify_reason="Hỏi lại sai hướng",
)
frame = _apply_condition_candidate_cue(
noisy,
"Thuốc nào có chỉ định liên quan bệnh gút?",
ConditionNormalizer(),
)
assert frame.turn_type == "condition_to_drug"
assert frame.condition_relation == ConditionRelation.INDICATION
assert frame.needs_clarify is False
def test_general_condition_does_not_invent_patient_hepatic_context():
frame = QueryFrame(
turn_type="condition_to_drug",
indication="viêm gan B mạn",
condition=ConditionQuery(
original_query="Viêm gan B mạn dùng thuốc gì?",
normalized_condition="viêm gan B mạn",
),
patient_context=PatientContext(
primary_condition="viêm gan B mạn",
hepatic=HepaticContext(description="viêm gan B mạn"),
),
)
cleaned = _apply_general_condition_scope(
frame, "Viêm gan B mạn dùng thuốc gì?"
)
assert cleaned.patient_context.primary_condition == "viêm gan B mạn"
assert cleaned.patient_context.requires_safety_review is False
def test_patient_allergy_condition_lookup_keeps_safety_context():
patient = PatientContext(
primary_condition="viêm phổi", allergies=("penicillin",)
)
frame = QueryFrame(
turn_type="condition_to_drug",
condition=ConditionQuery(
original_query="BN dị ứng penicillin, viêm phổi dùng thuốc gì?",
normalized_condition="viêm phổi",
),
patient_context=patient,
)
kept = _apply_general_condition_scope(
frame, "BN dị ứng penicillin, viêm phổi dùng thuốc gì?"
)
assert kept.patient_context == patient
assert kept.patient_context.requires_safety_review is True
def test_candidate_safety_followup_stays_on_prior_condition_lookup():
prior = QueryFrame(
turn_type="condition_to_drug",
indication="tăng huyết áp",
condition=ConditionQuery(
original_query="BN bị tăng huyết áp",
normalized_condition="tăng huyết áp",
),
patient_context=PatientContext(
age_text="68 tuổi",
renal=RenalContext(description="CKD", ckd_stage="G4"),
),
)
noisy = QueryFrame(
turn_type="condition_relation",
condition_relation=ConditionRelation.CONTRAINDICATION,
depends_on_previous_turn=True,
patient_context=prior.patient_context,
needs_clarify=False,
)
corrected = _apply_contextual_candidate_safety(
noisy,
"Trong các thuốc trên cái nào cần lưu ý hơn với bệnh thận?",
prior,
)
assert corrected.turn_type == "condition_to_drug"
assert corrected.condition == prior.condition
assert corrected.condition_relation == ConditionRelation.INDICATION
def test_broad_condition_is_clarified_but_specific_subtype_is_not():
normalizer = ConditionNormalizer()
broad = normalizer.normalize("Viêm gan dùng thuốc gì?", "viêm gan")
@@ -7,6 +7,7 @@ from the real METFORMIN and PARACETAMOL sections in `duocthu_v1`.
from __future__ import annotations
import json
from dataclasses import replace
import pytest
@@ -516,6 +517,43 @@ def test_a_real_negative_verdict_is_still_an_unsupported_claim():
assert metrics.total(GENERATION_REJECTED, reason="request_budget_exhausted") == 0
def test_patient_candidate_list_retries_one_noisy_entailment_rejection():
metrics = InMemoryMetrics()
result = _result()
result = replace(
result,
evidence=(replace(result.evidence[0], drug_id="metformin"),),
)
generator = _Generator(
[
{"claims": [{"drug_id": "metformin", "text": "Người lớn uống 500 mg", "citations": [1]}],
"evidence_sufficient": True},
{"claims": [{"drug_id": "metformin", "text": "Người lớn uống 500 mg", "citations": [1]}],
"evidence_sufficient": True},
],
entailment_payload=[
{"entailed": False, "unsupported": [1]},
{"entailed": True, "unsupported": [], "complete": True},
],
)
service = GroundedAnswerService(_FixedRouting(result), generator, metrics)
grounded = service.answer_from_result(
"Trong các thuốc trên thuốc nào cần lưu ý hơn với bệnh thận?",
result,
list_mode=True,
patient_specific=True,
candidate_drug_ids=("metformin",),
prechecked=True,
)
assert grounded.generated is True
assert grounded.result.decision == EvidenceDecision.ANSWERABLE
assert generator._call == 2
assert generator._entailment_call == 2
assert metrics.total(GENERATION_REJECTED, reason="unsupported_claim") == 0
def test_entailment_check_is_skipped_when_there_are_no_claims():
"""No claims at all (2026-08-10: the structured-claims schema makes a
claim's `text` a required, non-empty field, so the old "answer is
@@ -16,6 +16,7 @@ from rag.understanding import (
SECTION_KEYS,
LlmQueryUnderstander,
QueryFrame,
_merge_with_prior_frame,
)
CATALOG = {
@@ -162,6 +163,29 @@ def test_single_section_named_is_unaffected_by_the_multi_section_clarify():
assert frame.needs_clarify is False
def test_multi_section_clarify_does_not_inherit_a_stale_prior_attribute():
prior = QueryFrame(
turn_type="drug_attribute",
drugs=("paracetamol_acetaminophen",),
attribute="lieu_luong_va_cach_dung",
needs_clarify=True,
clarify_reason="Anh/chị muốn tra gì?",
)
current = QueryFrame(
turn_type="drug_attribute",
drugs=("paracetamol_acetaminophen",),
attribute=None,
needs_clarify=True,
clarify_reason="Anh/chị muốn xem mục nào trước?",
quick_replies=("Chỉ định", "Chống chỉ định"),
)
merged = _merge_with_prior_frame(current, prior)
assert merged.attribute is None
assert merged.quick_replies == ("Chỉ định", "Chống chỉ định")
def test_exact_candidate_does_not_repeat_the_catalog_wide_fuzzy_scan():
resolver = _FakeResolver({"metformin": "metformin"})
understander = LlmQueryUnderstander(_FixedLlm({