Add read-only production runtime audit
This commit is contained in:
@@ -127,7 +127,8 @@ def test_veterinary_phrase_abstains_even_if_the_model_missed_it():
|
||||
agent = _agent(QueryFrame(turn_type="drug_attribute", drugs=("metformin",)))
|
||||
reply = agent.handle("liều metformin cho chó bao nhiêu")
|
||||
assert reply.decision == "abstain"
|
||||
assert reply.reason == "out_of_scope"
|
||||
assert reply.reason == "out_of_scope_non_human"
|
||||
assert "chỉ bao phủ thuốc dùng cho người" in reply.answer
|
||||
|
||||
|
||||
def test_unknown_drug_name_is_reported_not_substituted():
|
||||
@@ -145,7 +146,7 @@ def test_no_drug_named_asks_which_one():
|
||||
assert reply.reason == "no_drug"
|
||||
|
||||
|
||||
def test_drug_attribute_without_an_attribute_does_not_fall_into_overview_retrieval():
|
||||
def test_drug_attribute_without_an_attribute_keeps_ai_clarification_without_retrieval():
|
||||
retrieval = _FixedRetrieval({})
|
||||
answers = GroundedAnswerService(routing=None)
|
||||
agent = RagAgent(
|
||||
@@ -163,6 +164,65 @@ def test_drug_attribute_without_an_attribute_does_not_fall_into_overview_retriev
|
||||
assert retrieval.calls == []
|
||||
|
||||
|
||||
def test_bare_drug_overview_opens_section_picker_without_retrieval():
|
||||
retrieval = _FixedRetrieval({})
|
||||
agent = RagAgent(
|
||||
_FixedUnderstander(QueryFrame(
|
||||
turn_type="drug_overview", drugs=("metformin",)
|
||||
)),
|
||||
retrieval,
|
||||
GroundedAnswerService(routing=None),
|
||||
)
|
||||
|
||||
reply = agent.handle("Metformin", response_mode="monograph")
|
||||
|
||||
assert reply.decision == "clarify"
|
||||
assert reply.reason == "select_drug_sections"
|
||||
assert reply.drugs == ("metformin",)
|
||||
assert retrieval.calls == []
|
||||
|
||||
|
||||
def test_monograph_bare_drug_ignores_stale_inherited_attribute():
|
||||
retrieval = _FixedRetrieval({})
|
||||
agent = RagAgent(
|
||||
_FixedUnderstander(QueryFrame(
|
||||
turn_type="drug_attribute",
|
||||
drugs=("metformin",),
|
||||
attribute="chong_chi_dinh",
|
||||
)),
|
||||
retrieval,
|
||||
GroundedAnswerService(routing=None),
|
||||
)
|
||||
|
||||
reply = agent.handle("Metformin", response_mode="monograph")
|
||||
|
||||
assert reply.reason == "select_drug_sections"
|
||||
assert retrieval.calls == []
|
||||
|
||||
|
||||
def test_monograph_mode_keeps_explicit_attribute_on_ai_route():
|
||||
result = RetrievalResult(
|
||||
EvidenceDecision.ABSTAIN, "not_configured", resolved_drug_id="metformin"
|
||||
)
|
||||
retrieval = _FixedRetrieval({"metformin": result})
|
||||
agent = RagAgent(
|
||||
_FixedUnderstander(QueryFrame(
|
||||
turn_type="drug_attribute",
|
||||
drugs=("metformin",),
|
||||
attribute="chong_chi_dinh",
|
||||
)),
|
||||
retrieval,
|
||||
GroundedAnswerService(routing=None),
|
||||
)
|
||||
|
||||
reply = agent.handle(
|
||||
"Chống chỉ định của Metformin là gì?", response_mode="monograph"
|
||||
)
|
||||
|
||||
assert reply.reason != "select_drug_sections"
|
||||
assert retrieval.calls[0][1] == "chong_chi_dinh"
|
||||
|
||||
|
||||
# --- the pediatric dosing gate. This code path gained its first test
|
||||
# coverage on 2026-08-11, after driving production reproduced the same
|
||||
# behaviour 5/5: the clarify question asked for both age and weight every
|
||||
|
||||
@@ -5,7 +5,7 @@ from fastapi.testclient import TestClient
|
||||
from adapters.prometheus import PrometheusMetrics
|
||||
from adapters.postgres import FeedbackTraceNotFound, RetrievalTrace
|
||||
from config import Settings
|
||||
from main import create_app
|
||||
from main import _route_label, create_app
|
||||
from rag.agent import AgentReply
|
||||
from rag.answer import DISCLAIMER, Citation, GroundedAnswerService
|
||||
from rag.metrics import TRACE_WRITE_FAILED, InMemoryMetrics
|
||||
@@ -150,6 +150,18 @@ def test_history_for_unknown_conversation_is_empty_not_an_error():
|
||||
assert response.json() == {"items": []}
|
||||
|
||||
|
||||
def test_history_rejects_an_oversized_conversation_id_before_querying_storage():
|
||||
traces = FakeHistoryTraceWriter({})
|
||||
app = create_app(settings=Settings(), trace_writer=traces)
|
||||
|
||||
response = TestClient(app).get(
|
||||
"/v1/rag/history", params={"conversation_id": "x" * 129}
|
||||
)
|
||||
|
||||
assert response.status_code == 422
|
||||
assert traces.calls == []
|
||||
|
||||
|
||||
def test_health_and_fail_closed_rag_response_are_traced():
|
||||
traces = MemoryTraceWriter()
|
||||
app = create_app(
|
||||
@@ -204,6 +216,19 @@ def _metrics_app(**settings_kwargs):
|
||||
)
|
||||
|
||||
|
||||
def test_all_public_rag_endpoints_have_bounded_request_metric_labels():
|
||||
paths = (
|
||||
"/v1/rag/query",
|
||||
"/v1/rag/suggest",
|
||||
"/v1/rag/feedback",
|
||||
"/v1/rag/history",
|
||||
"/v1/rag/sections",
|
||||
"/v1/rag/section-text",
|
||||
)
|
||||
|
||||
assert {_route_label(path) for path in paths} == set(paths)
|
||||
|
||||
|
||||
def test_metrics_stays_open_when_no_token_is_configured():
|
||||
"""The default must not break the existing Compose scrape or local runs —
|
||||
the endpoint is not internet-reachable in that topology."""
|
||||
@@ -252,10 +277,15 @@ class FakeAgent:
|
||||
|
||||
def __init__(self, reply: AgentReply) -> None:
|
||||
self._reply = reply
|
||||
self.calls: list[tuple[str, str | None]] = []
|
||||
self.calls: list[tuple[str, str | None, str]] = []
|
||||
|
||||
def handle(self, turn: str, conversation_id: str | None = None) -> AgentReply:
|
||||
self.calls.append((turn, conversation_id))
|
||||
def handle(
|
||||
self,
|
||||
turn: str,
|
||||
conversation_id: str | None = None,
|
||||
response_mode: str = "ai",
|
||||
) -> AgentReply:
|
||||
self.calls.append((turn, conversation_id, response_mode))
|
||||
return self._reply
|
||||
|
||||
def complete(self, prefix: str, k: int = 8) -> list[str]:
|
||||
@@ -291,7 +321,34 @@ def test_query_routes_through_the_agent_when_one_is_configured():
|
||||
assert body["answer"] == "Liều 500 mg [1]."
|
||||
assert body["resolved_drug_id"] == "metformin"
|
||||
assert len(body["citations"]) == 1
|
||||
assert agent.calls == [("Liều metformin?", "c1")]
|
||||
assert agent.calls == [("Liều metformin?", "c1", "ai")]
|
||||
|
||||
|
||||
def test_query_forwards_monograph_response_mode_to_agent():
|
||||
agent = FakeAgent(AgentReply(
|
||||
decision="clarify",
|
||||
reason="select_drug_sections",
|
||||
clarification="Chọn mục cần xem.",
|
||||
drugs=("metformin",),
|
||||
turn_type="drug_overview",
|
||||
))
|
||||
app = create_app(
|
||||
settings=Settings(),
|
||||
answer_service=GroundedAnswerService(FixedRouting()),
|
||||
conversational=agent,
|
||||
trace_writer=MemoryTraceWriter(),
|
||||
)
|
||||
|
||||
response = TestClient(app).post("/v1/rag/query", json={
|
||||
"query": "Metformin",
|
||||
"subject_scope": "human",
|
||||
"intent": "fact_lookup",
|
||||
"response_mode": "monograph",
|
||||
})
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json()["reason"] == "select_drug_sections"
|
||||
assert agent.calls == [("Metformin", None, "monograph")]
|
||||
|
||||
|
||||
def test_query_agent_clarification_is_surfaced_as_the_answer():
|
||||
|
||||
@@ -11,6 +11,7 @@ from rag.clinical import (
|
||||
ConditionNormalizer,
|
||||
ConditionQuery,
|
||||
ConditionRelation,
|
||||
HepaticContext,
|
||||
PatientContext,
|
||||
RenalContext,
|
||||
)
|
||||
@@ -20,6 +21,8 @@ from rag.understanding import (
|
||||
LlmQueryUnderstander,
|
||||
QueryFrame,
|
||||
_apply_condition_candidate_cue,
|
||||
_apply_contextual_candidate_safety,
|
||||
_apply_general_condition_scope,
|
||||
_apply_named_drug_cues,
|
||||
_apply_reverse_relation_cues,
|
||||
_merge_with_prior_frame,
|
||||
@@ -72,9 +75,104 @@ def test_condition_normalizer_handles_professional_aliases_without_drug_mapping(
|
||||
assert normalizer.normalize("THA dùng gì", "THA").normalized_condition == "tăng huyết áp"
|
||||
assert normalizer.normalize("cao huyết áp", "cao huyết áp").normalized_condition == "tăng huyết áp"
|
||||
assert normalizer.normalize("Gout", "gout").normalized_condition == "gút"
|
||||
assert normalizer.detect_known_alias("BN viêm phổi dùng thuốc gì?").normalized_condition == "viêm phổi"
|
||||
assert normalizer.normalize("bệnh lạ", "bệnh lạ").normalized_condition == "bệnh lạ"
|
||||
|
||||
|
||||
def test_explicit_indication_relation_is_a_condition_candidate_lookup():
|
||||
noisy = QueryFrame(
|
||||
turn_type="condition_relation",
|
||||
condition=ConditionNormalizer().normalize("bệnh gút", "gút"),
|
||||
needs_clarify=True,
|
||||
clarify_reason="Hỏi lại sai hướng",
|
||||
)
|
||||
|
||||
frame = _apply_condition_candidate_cue(
|
||||
noisy,
|
||||
"Thuốc nào có chỉ định liên quan bệnh gút?",
|
||||
ConditionNormalizer(),
|
||||
)
|
||||
|
||||
assert frame.turn_type == "condition_to_drug"
|
||||
assert frame.condition_relation == ConditionRelation.INDICATION
|
||||
assert frame.needs_clarify is False
|
||||
|
||||
|
||||
def test_general_condition_does_not_invent_patient_hepatic_context():
|
||||
frame = QueryFrame(
|
||||
turn_type="condition_to_drug",
|
||||
indication="viêm gan B mạn",
|
||||
condition=ConditionQuery(
|
||||
original_query="Viêm gan B mạn dùng thuốc gì?",
|
||||
normalized_condition="viêm gan B mạn",
|
||||
),
|
||||
patient_context=PatientContext(
|
||||
primary_condition="viêm gan B mạn",
|
||||
hepatic=HepaticContext(description="viêm gan B mạn"),
|
||||
),
|
||||
)
|
||||
|
||||
cleaned = _apply_general_condition_scope(
|
||||
frame, "Viêm gan B mạn dùng thuốc gì?"
|
||||
)
|
||||
|
||||
assert cleaned.patient_context.primary_condition == "viêm gan B mạn"
|
||||
assert cleaned.patient_context.requires_safety_review is False
|
||||
|
||||
|
||||
def test_patient_allergy_condition_lookup_keeps_safety_context():
|
||||
patient = PatientContext(
|
||||
primary_condition="viêm phổi", allergies=("penicillin",)
|
||||
)
|
||||
frame = QueryFrame(
|
||||
turn_type="condition_to_drug",
|
||||
condition=ConditionQuery(
|
||||
original_query="BN dị ứng penicillin, viêm phổi dùng thuốc gì?",
|
||||
normalized_condition="viêm phổi",
|
||||
),
|
||||
patient_context=patient,
|
||||
)
|
||||
|
||||
kept = _apply_general_condition_scope(
|
||||
frame, "BN dị ứng penicillin, viêm phổi dùng thuốc gì?"
|
||||
)
|
||||
|
||||
assert kept.patient_context == patient
|
||||
assert kept.patient_context.requires_safety_review is True
|
||||
|
||||
|
||||
def test_candidate_safety_followup_stays_on_prior_condition_lookup():
|
||||
prior = QueryFrame(
|
||||
turn_type="condition_to_drug",
|
||||
indication="tăng huyết áp",
|
||||
condition=ConditionQuery(
|
||||
original_query="BN bị tăng huyết áp",
|
||||
normalized_condition="tăng huyết áp",
|
||||
),
|
||||
patient_context=PatientContext(
|
||||
age_text="68 tuổi",
|
||||
renal=RenalContext(description="CKD", ckd_stage="G4"),
|
||||
),
|
||||
)
|
||||
noisy = QueryFrame(
|
||||
turn_type="condition_relation",
|
||||
condition_relation=ConditionRelation.CONTRAINDICATION,
|
||||
depends_on_previous_turn=True,
|
||||
patient_context=prior.patient_context,
|
||||
needs_clarify=False,
|
||||
)
|
||||
|
||||
corrected = _apply_contextual_candidate_safety(
|
||||
noisy,
|
||||
"Trong các thuốc trên cái nào cần lưu ý hơn với bệnh thận?",
|
||||
prior,
|
||||
)
|
||||
|
||||
assert corrected.turn_type == "condition_to_drug"
|
||||
assert corrected.condition == prior.condition
|
||||
assert corrected.condition_relation == ConditionRelation.INDICATION
|
||||
|
||||
|
||||
def test_broad_condition_is_clarified_but_specific_subtype_is_not():
|
||||
normalizer = ConditionNormalizer()
|
||||
broad = normalizer.normalize("Viêm gan dùng thuốc gì?", "viêm gan")
|
||||
|
||||
@@ -7,6 +7,7 @@ from the real METFORMIN and PARACETAMOL sections in `duocthu_v1`.
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from dataclasses import replace
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -516,6 +517,43 @@ def test_a_real_negative_verdict_is_still_an_unsupported_claim():
|
||||
assert metrics.total(GENERATION_REJECTED, reason="request_budget_exhausted") == 0
|
||||
|
||||
|
||||
def test_patient_candidate_list_retries_one_noisy_entailment_rejection():
|
||||
metrics = InMemoryMetrics()
|
||||
result = _result()
|
||||
result = replace(
|
||||
result,
|
||||
evidence=(replace(result.evidence[0], drug_id="metformin"),),
|
||||
)
|
||||
generator = _Generator(
|
||||
[
|
||||
{"claims": [{"drug_id": "metformin", "text": "Người lớn uống 500 mg", "citations": [1]}],
|
||||
"evidence_sufficient": True},
|
||||
{"claims": [{"drug_id": "metformin", "text": "Người lớn uống 500 mg", "citations": [1]}],
|
||||
"evidence_sufficient": True},
|
||||
],
|
||||
entailment_payload=[
|
||||
{"entailed": False, "unsupported": [1]},
|
||||
{"entailed": True, "unsupported": [], "complete": True},
|
||||
],
|
||||
)
|
||||
service = GroundedAnswerService(_FixedRouting(result), generator, metrics)
|
||||
|
||||
grounded = service.answer_from_result(
|
||||
"Trong các thuốc trên thuốc nào cần lưu ý hơn với bệnh thận?",
|
||||
result,
|
||||
list_mode=True,
|
||||
patient_specific=True,
|
||||
candidate_drug_ids=("metformin",),
|
||||
prechecked=True,
|
||||
)
|
||||
|
||||
assert grounded.generated is True
|
||||
assert grounded.result.decision == EvidenceDecision.ANSWERABLE
|
||||
assert generator._call == 2
|
||||
assert generator._entailment_call == 2
|
||||
assert metrics.total(GENERATION_REJECTED, reason="unsupported_claim") == 0
|
||||
|
||||
|
||||
def test_entailment_check_is_skipped_when_there_are_no_claims():
|
||||
"""No claims at all (2026-08-10: the structured-claims schema makes a
|
||||
claim's `text` a required, non-empty field, so the old "answer is
|
||||
|
||||
@@ -16,6 +16,7 @@ from rag.understanding import (
|
||||
SECTION_KEYS,
|
||||
LlmQueryUnderstander,
|
||||
QueryFrame,
|
||||
_merge_with_prior_frame,
|
||||
)
|
||||
|
||||
CATALOG = {
|
||||
@@ -162,6 +163,29 @@ def test_single_section_named_is_unaffected_by_the_multi_section_clarify():
|
||||
assert frame.needs_clarify is False
|
||||
|
||||
|
||||
def test_multi_section_clarify_does_not_inherit_a_stale_prior_attribute():
|
||||
prior = QueryFrame(
|
||||
turn_type="drug_attribute",
|
||||
drugs=("paracetamol_acetaminophen",),
|
||||
attribute="lieu_luong_va_cach_dung",
|
||||
needs_clarify=True,
|
||||
clarify_reason="Anh/chị muốn tra gì?",
|
||||
)
|
||||
current = QueryFrame(
|
||||
turn_type="drug_attribute",
|
||||
drugs=("paracetamol_acetaminophen",),
|
||||
attribute=None,
|
||||
needs_clarify=True,
|
||||
clarify_reason="Anh/chị muốn xem mục nào trước?",
|
||||
quick_replies=("Chỉ định", "Chống chỉ định"),
|
||||
)
|
||||
|
||||
merged = _merge_with_prior_frame(current, prior)
|
||||
|
||||
assert merged.attribute is None
|
||||
assert merged.quick_replies == ("Chỉ định", "Chống chỉ định")
|
||||
|
||||
|
||||
def test_exact_candidate_does_not_repeat_the_catalog_wide_fuzzy_scan():
|
||||
resolver = _FakeResolver({"metformin": "metformin"})
|
||||
understander = LlmQueryUnderstander(_FixedLlm({
|
||||
|
||||
Reference in New Issue
Block a user