Add Langfuse as a self-hosted eval and trace viewer

This commit is contained in:
2026-08-21 10:35:25 +07:00
parent 4490a1abf0
commit 53582b6030
12 changed files with 1129 additions and 280 deletions
+89
View File
@@ -162,6 +162,66 @@ def test_history_rejects_an_oversized_conversation_id_before_querying_storage():
assert traces.calls == []
def test_transcript_returns_user_and_assistant_turns_oldest_first():
when = datetime(2026, 8, 14, 10, 0, tzinfo=timezone.utc)
traces = FakeHistoryTraceWriter({
# list_by_conversation contract is newest-first, same as /history.
"case-1": [
RetrievalTrace(
trace_id="t2", query="Chống chỉ định metformin?",
subject_scope="human", intent="fact_lookup",
decision="answerable", reason="grounded_evidence_available",
resolved_drug_id="metformin", citations=(),
conversation_id="case-1", created_at=when,
response_payload={
"answer": "Suy thận nặng.",
"resolved_drug_id": "metformin",
"citations": [{"chunk_id": "metformin::cci::0"}],
"generated": True,
"quick_replies": [],
"blocks": [],
"answer_mode": "concise",
"answer_plan": None,
"candidate_assessments": [],
"disclaimer": "disclaimer text",
},
),
RetrievalTrace(
trace_id="t1", query="Chỉ định metformin?",
subject_scope="human", intent="fact_lookup",
decision="answerable", reason="grounded_evidence_available",
resolved_drug_id="metformin", citations=(),
conversation_id="case-1", created_at=when,
response_payload=None,
),
],
})
app = create_app(settings=Settings(), trace_writer=traces)
response = TestClient(app).get("/v1/rag/transcript", params={"conversation_id": "case-1"})
assert response.status_code == 200
body = response.json()
# t1 (oldest) has no persisted answer -> user turn only. t2 has one -> both.
roles = [(m["trace_id"], m["role"]) for m in body["messages"]]
assert roles == [("t1", "user"), ("t2", "user"), ("t2", "assistant")]
assistant = body["messages"][2]
assert assistant["content"] == "Suy thận nặng."
assert assistant["citations"] == [{"chunk_id": "metformin::cci::0"}]
assert assistant["generated"] is True
def test_transcript_with_empty_conversation_id_returns_no_messages():
traces = FakeHistoryTraceWriter({})
app = create_app(settings=Settings(), trace_writer=traces)
response = TestClient(app).get("/v1/rag/transcript", params={"conversation_id": " "})
assert response.status_code == 200
assert response.json() == {"messages": []}
assert traces.calls == []
def test_health_and_fail_closed_rag_response_are_traced():
traces = MemoryTraceWriter()
app = create_app(
@@ -324,6 +384,35 @@ def test_query_routes_through_the_agent_when_one_is_configured():
assert agent.calls == [("Liều metformin?", "c1", "ai")]
def test_query_persists_the_full_answer_for_later_transcript_replay():
"""The `/transcript` endpoint can only redraw a conversation if this
survives to storage — a plain trace row (decision/reason only) is not
enough to show what the AI actually said."""
agent = FakeAgent(AgentReply(
decision="answerable", reason="grounded_evidence_available",
answer="Liều 500 mg [1].", citations=(_citation(),),
drugs=("metformin",), turn_type="drug_attribute", generated=True,
))
traces = MemoryTraceWriter()
app = create_app(
settings=Settings(),
answer_service=GroundedAnswerService(FixedRouting()),
conversational=agent,
trace_writer=traces,
)
TestClient(app).post("/v1/rag/query", json={
"query": "Liều metformin?", "subject_scope": "human",
"intent": "fact_lookup", "conversation_id": "c1",
})
payload = traces.rows[-1]["response_payload"]
assert payload["answer"] == "Liều 500 mg [1]."
assert payload["resolved_drug_id"] == "metformin"
assert payload["generated"] is True
assert len(payload["citations"]) == 1
assert payload["citations"][0]["chunk_id"] == "metformin::lieu::0"
def test_query_forwards_monograph_response_mode_to_agent():
agent = FakeAgent(AgentReply(
decision="clarify",