344 lines
12 KiB
Python
344 lines
12 KiB
Python
from fastapi.testclient import TestClient
|
|
|
|
from adapters.prometheus import PrometheusMetrics
|
|
from adapters.postgres import FeedbackTraceNotFound
|
|
from config import Settings
|
|
from main import create_app
|
|
from rag.agent import AgentReply
|
|
from rag.answer import DISCLAIMER, Citation, GroundedAnswerService
|
|
from rag.metrics import TRACE_WRITE_FAILED, InMemoryMetrics
|
|
from rag.models import EvidenceDecision, QueryIntent, RetrievalResult, SubjectScope
|
|
|
|
|
|
class FixedRouting:
|
|
def retrieve(self, query, subject_scope, intent):
|
|
assert query == "Liều thuốc?"
|
|
assert subject_scope == SubjectScope.HUMAN
|
|
assert intent == QueryIntent.FACT_LOOKUP
|
|
return RetrievalResult(EvidenceDecision.ABSTAIN, "drug_not_resolved")
|
|
|
|
|
|
class MemoryTraceWriter:
|
|
def __init__(self):
|
|
self.rows = []
|
|
self.feedback = []
|
|
|
|
def save(self, **fields):
|
|
self.rows.append(fields)
|
|
return "trace-1"
|
|
|
|
def save_feedback(self, **fields):
|
|
self.feedback.append(fields)
|
|
return "feedback-1"
|
|
|
|
|
|
def test_feedback_is_linked_to_the_answer_trace():
|
|
traces = MemoryTraceWriter()
|
|
app = create_app(settings=Settings(), trace_writer=traces)
|
|
response = TestClient(app).post("/v1/rag/feedback", json={
|
|
"trace_id": "8f33dd3e-9000-4384-ae69-33da2629e29b",
|
|
"rating": "not_helpful",
|
|
"comment": " Thiếu lưu ý suy thận. ",
|
|
"conversation_id": "case-1",
|
|
})
|
|
|
|
assert response.status_code == 200
|
|
assert response.json() == {"feedback_id": "feedback-1", "status": "saved"}
|
|
assert traces.feedback == [{
|
|
"trace_id": "8f33dd3e-9000-4384-ae69-33da2629e29b",
|
|
"rating": "not_helpful",
|
|
"comment": "Thiếu lưu ý suy thận.",
|
|
"conversation_id": "case-1",
|
|
}]
|
|
|
|
|
|
def test_feedback_rejects_an_unpersisted_trace():
|
|
class MissingTraceWriter(MemoryTraceWriter):
|
|
def save_feedback(self, **fields):
|
|
raise FeedbackTraceNotFound(fields["trace_id"])
|
|
|
|
app = create_app(settings=Settings(), trace_writer=MissingTraceWriter())
|
|
response = TestClient(app).post("/v1/rag/feedback", json={
|
|
"trace_id": "8f33dd3e-9000-4384-ae69-33da2629e29b",
|
|
"rating": "helpful",
|
|
})
|
|
|
|
assert response.status_code == 404
|
|
assert response.json() == {"detail": "trace_not_found"}
|
|
|
|
|
|
def test_health_and_fail_closed_rag_response_are_traced():
|
|
traces = MemoryTraceWriter()
|
|
app = create_app(
|
|
settings=Settings(),
|
|
answer_service=GroundedAnswerService(FixedRouting()),
|
|
trace_writer=traces,
|
|
)
|
|
client = TestClient(app)
|
|
assert client.get("/health").json() == {"status": "ok"}
|
|
response = client.post("/v1/rag/query", json={
|
|
"query": "Liều thuốc?",
|
|
"subject_scope": "human",
|
|
"intent": "fact_lookup",
|
|
})
|
|
assert response.status_code == 200
|
|
assert response.json()["decision"] == "abstain"
|
|
assert response.json()["trace_id"] == "trace-1"
|
|
assert traces.rows[0]["reason"] == "drug_not_resolved"
|
|
|
|
|
|
def test_every_query_response_carries_the_disclaimer_including_an_abstain():
|
|
"""The API-visible half of the guardrail.
|
|
|
|
`docs/architecture.md` specifies the disclaimer at several layers and the
|
|
web banner was the only one present, so any consumer other than that one
|
|
UI received medical content with nothing attached. An abstain is the case
|
|
worth pinning: it is easy to treat as "not really an answer", and it is
|
|
still the system responding to a clinical question.
|
|
"""
|
|
app = create_app(
|
|
settings=Settings(),
|
|
answer_service=GroundedAnswerService(FixedRouting()),
|
|
trace_writer=MemoryTraceWriter(),
|
|
)
|
|
response = TestClient(app).post("/v1/rag/query", json={
|
|
"query": "Liều thuốc?",
|
|
"subject_scope": "human",
|
|
"intent": "fact_lookup",
|
|
})
|
|
|
|
body = response.json()
|
|
assert body["decision"] == "abstain"
|
|
assert body["disclaimer"] == DISCLAIMER
|
|
assert "không thay thế chỉ định" in body["disclaimer"]
|
|
|
|
|
|
def _metrics_app(**settings_kwargs):
|
|
return create_app(
|
|
settings=Settings(**settings_kwargs),
|
|
answer_service=GroundedAnswerService(FixedRouting()),
|
|
trace_writer=MemoryTraceWriter(),
|
|
)
|
|
|
|
|
|
def test_metrics_stays_open_when_no_token_is_configured():
|
|
"""The default must not break the existing Compose scrape or local runs —
|
|
the endpoint is not internet-reachable in that topology."""
|
|
response = TestClient(_metrics_app()).get("/metrics")
|
|
assert response.status_code in (200, 404) # 404 only when no exporter
|
|
|
|
|
|
def test_metrics_requires_the_token_once_one_is_configured():
|
|
client = TestClient(_metrics_app(metrics_token="s3cret"))
|
|
|
|
assert client.get("/metrics").status_code == 401
|
|
assert client.get(
|
|
"/metrics", headers={"Authorization": "Bearer wrong"}
|
|
).status_code == 401
|
|
# A correct token gets past the guard; whether an exporter is attached is
|
|
# a separate concern, so 404 is an acceptable non-401 here.
|
|
assert client.get(
|
|
"/metrics", headers={"Authorization": "Bearer s3cret"}
|
|
).status_code in (200, 404)
|
|
|
|
|
|
def test_metrics_token_is_not_accepted_from_a_query_string():
|
|
"""Secrets in URLs end up in access logs and referrers, so only the
|
|
Authorization header is honoured."""
|
|
client = TestClient(_metrics_app(metrics_token="s3cret"))
|
|
assert client.get("/metrics?token=s3cret").status_code == 401
|
|
|
|
|
|
def test_query_requires_structured_scope_and_intent():
|
|
app = create_app(
|
|
settings=Settings(),
|
|
answer_service=GroundedAnswerService(FixedRouting()),
|
|
trace_writer=MemoryTraceWriter(),
|
|
)
|
|
response = TestClient(app).post("/v1/rag/query", json={"query": "Liều?"})
|
|
assert response.status_code == 422
|
|
|
|
|
|
# --- F-03: the app.state.conversational slot is the new RagAgent, wired live ---
|
|
|
|
|
|
class FakeAgent:
|
|
"""Stands in for `RagAgent` — proves `routers/rag.py` calls `.handle()`
|
|
and maps `AgentReply` correctly, not that the agent's own routing logic
|
|
is correct (that's `test_agent.py`)."""
|
|
|
|
def __init__(self, reply: AgentReply) -> None:
|
|
self._reply = reply
|
|
self.calls: list[tuple[str, str | None]] = []
|
|
|
|
def handle(self, turn: str, conversation_id: str | None = None) -> AgentReply:
|
|
self.calls.append((turn, conversation_id))
|
|
return self._reply
|
|
|
|
def complete(self, prefix: str, k: int = 8) -> list[str]:
|
|
return ["Metformin"] if prefix else []
|
|
|
|
|
|
def _citation() -> Citation:
|
|
return Citation(
|
|
chunk_id="metformin::lieu::0", printed_page_start=714, printed_page_end=714,
|
|
physical_page=812,
|
|
)
|
|
|
|
|
|
def test_query_routes_through_the_agent_when_one_is_configured():
|
|
agent = FakeAgent(AgentReply(
|
|
decision="answerable", reason="grounded_evidence_available",
|
|
answer="Liều 500 mg [1].", citations=(_citation(),),
|
|
drugs=("metformin",), turn_type="drug_attribute", generated=True,
|
|
))
|
|
app = create_app(
|
|
settings=Settings(),
|
|
answer_service=GroundedAnswerService(FixedRouting()),
|
|
conversational=agent,
|
|
trace_writer=MemoryTraceWriter(),
|
|
)
|
|
response = TestClient(app).post("/v1/rag/query", json={
|
|
"query": "Liều metformin?", "subject_scope": "human",
|
|
"intent": "fact_lookup", "conversation_id": "c1",
|
|
})
|
|
body = response.json()
|
|
assert response.status_code == 200
|
|
assert body["decision"] == "answerable"
|
|
assert body["answer"] == "Liều 500 mg [1]."
|
|
assert body["resolved_drug_id"] == "metformin"
|
|
assert len(body["citations"]) == 1
|
|
assert agent.calls == [("Liều metformin?", "c1")]
|
|
|
|
|
|
def test_query_agent_clarification_is_surfaced_as_the_answer():
|
|
agent = FakeAgent(AgentReply(
|
|
decision="clarify", reason="no_drug",
|
|
clarification="Anh/chị muốn tra thuốc nào?",
|
|
))
|
|
app = create_app(
|
|
settings=Settings(),
|
|
answer_service=GroundedAnswerService(FixedRouting()),
|
|
conversational=agent,
|
|
trace_writer=MemoryTraceWriter(),
|
|
)
|
|
response = TestClient(app).post("/v1/rag/query", json={
|
|
"query": "liều dùng bao nhiêu", "subject_scope": "human",
|
|
"intent": "fact_lookup",
|
|
})
|
|
body = response.json()
|
|
assert body["decision"] == "clarify"
|
|
assert body["answer"] == "Anh/chị muốn tra thuốc nào?"
|
|
assert body["resolved_drug_id"] is None
|
|
|
|
|
|
def test_suggest_delegates_to_the_agent_when_configured():
|
|
app = create_app(
|
|
settings=Settings(),
|
|
conversational=FakeAgent(AgentReply(decision="answerable", reason="x")),
|
|
trace_writer=MemoryTraceWriter(),
|
|
)
|
|
response = TestClient(app).get("/v1/rag/suggest", params={"q": "met"})
|
|
assert response.json() == {"suggestions": ["Metformin"]}
|
|
|
|
|
|
def test_suggest_with_no_agent_configured_returns_empty():
|
|
app = create_app(settings=Settings(), trace_writer=MemoryTraceWriter())
|
|
response = TestClient(app).get("/v1/rag/suggest", params={"q": "met"})
|
|
assert response.json() == {"suggestions": []}
|
|
|
|
|
|
# --- F-09: trace persistence is fail-open ------------------------------------
|
|
|
|
|
|
class _RaisingTraceWriter:
|
|
"""Stands in for a Postgres outage: `save()` always raises."""
|
|
|
|
def save(self, **fields):
|
|
raise ConnectionError("could not connect to postgres")
|
|
|
|
|
|
def test_a_trace_write_failure_does_not_turn_a_good_answer_into_a_500():
|
|
agent = FakeAgent(AgentReply(
|
|
decision="answerable", reason="grounded_evidence_available",
|
|
answer="Liều 500 mg [1].", citations=(_citation(),),
|
|
drugs=("metformin",), turn_type="drug_attribute", generated=True,
|
|
))
|
|
metrics = InMemoryMetrics()
|
|
app = create_app(
|
|
settings=Settings(),
|
|
answer_service=GroundedAnswerService(FixedRouting()),
|
|
conversational=agent,
|
|
trace_writer=_RaisingTraceWriter(),
|
|
metrics=metrics,
|
|
)
|
|
response = TestClient(app).post("/v1/rag/query", json={
|
|
"query": "Liều metformin?", "subject_scope": "human", "intent": "fact_lookup",
|
|
})
|
|
body = response.json()
|
|
|
|
assert response.status_code == 200
|
|
assert body["answer"] == "Liều 500 mg [1]."
|
|
assert body["trace_id"] # a locally-generated fallback id, still present
|
|
assert metrics.total(TRACE_WRITE_FAILED) == 1
|
|
|
|
|
|
class _AnyRouting:
|
|
def retrieve(self, query, subject_scope, intent):
|
|
return RetrievalResult(EvidenceDecision.ABSTAIN, "drug_not_resolved")
|
|
|
|
|
|
def test_correlation_id_round_trips_through_headers_body_and_trace_row():
|
|
traces = MemoryTraceWriter()
|
|
app = create_app(
|
|
settings=Settings(embedding_provider="disabled"),
|
|
answer_service=GroundedAnswerService(_AnyRouting()),
|
|
trace_writer=traces,
|
|
)
|
|
|
|
response = TestClient(app).post(
|
|
"/v1/rag/query",
|
|
headers={"X-Correlation-ID": "req-test-1"},
|
|
json={
|
|
"query": "Paracetamol dose?",
|
|
"subject_scope": "human",
|
|
"intent": "fact_lookup",
|
|
},
|
|
)
|
|
|
|
assert response.status_code == 200
|
|
assert response.headers["x-correlation-id"] == "req-test-1"
|
|
assert response.json()["correlation_id"] == "req-test-1"
|
|
assert traces.rows[0]["correlation_id"] == "req-test-1"
|
|
assert "otel_trace_id" in traces.rows[0]
|
|
|
|
|
|
def test_metrics_endpoint_exposes_request_decision_and_stage_histograms():
|
|
metrics = PrometheusMetrics()
|
|
app = create_app(
|
|
settings=Settings(embedding_provider="disabled"),
|
|
answer_service=GroundedAnswerService(_AnyRouting()),
|
|
trace_writer=MemoryTraceWriter(),
|
|
metrics=metrics,
|
|
)
|
|
client = TestClient(app)
|
|
|
|
response = client.post(
|
|
"/v1/rag/query",
|
|
json={
|
|
"query": "Paracetamol dose?",
|
|
"subject_scope": "human",
|
|
"intent": "fact_lookup",
|
|
},
|
|
)
|
|
assert response.status_code == 200
|
|
|
|
scrape = client.get("/metrics")
|
|
assert scrape.status_code == 200
|
|
body = scrape.text
|
|
assert 'duocthu_requests_total{method="POST",route="/v1/rag/query",status="2xx"}' in body
|
|
assert 'duocthu_decision_total{decision="abstain",reason="drug_not_resolved"}' in body
|
|
assert 'duocthu_stage_duration_seconds_count{outcome="ok",stage="receive"}' in body
|
|
assert 'duocthu_stage_duration_seconds_count{outcome="ok",stage="persistence"}' in body
|
|
assert 'duocthu_stage_duration_seconds_count{outcome="ok",stage="response"}' in body
|