Verify exact production traces and record rollout

2026-08-11 11:29:59 +07:00
parent 6b8f7584ed
commit 59e6ad2d0d
46 changed files with 2795 additions and 290 deletions
@@ -1,5 +1,6 @@
 from fastapi.testclient import TestClient

+from adapters.prometheus import PrometheusMetrics
 from config import Settings
 from main import create_app
 from rag.agent import AgentReply
@@ -177,3 +178,63 @@ def test_a_trace_write_failure_does_not_turn_a_good_answer_into_a_500():
    assert body["answer"] == "Liều 500 mg [1]."
    assert body["trace_id"]  # a locally-generated fallback id, still present
    assert metrics.total(TRACE_WRITE_FAILED) == 1
+
+
+class _AnyRouting:
+    def retrieve(self, query, subject_scope, intent):
+        return RetrievalResult(EvidenceDecision.ABSTAIN, "drug_not_resolved")
+
+
+def test_correlation_id_round_trips_through_headers_body_and_trace_row():
+    traces = MemoryTraceWriter()
+    app = create_app(
+        settings=Settings(embedding_provider="disabled"),
+        answer_service=GroundedAnswerService(_AnyRouting()),
+        trace_writer=traces,
+    )
+
+    response = TestClient(app).post(
+        "/v1/rag/query",
+        headers={"X-Correlation-ID": "req-test-1"},
+        json={
+            "query": "Paracetamol dose?",
+            "subject_scope": "human",
+            "intent": "fact_lookup",
+        },
+    )
+
+    assert response.status_code == 200
+    assert response.headers["x-correlation-id"] == "req-test-1"
+    assert response.json()["correlation_id"] == "req-test-1"
+    assert traces.rows[0]["correlation_id"] == "req-test-1"
+    assert "otel_trace_id" in traces.rows[0]
+
+
+def test_metrics_endpoint_exposes_request_decision_and_stage_histograms():
+    metrics = PrometheusMetrics()
+    app = create_app(
+        settings=Settings(embedding_provider="disabled"),
+        answer_service=GroundedAnswerService(_AnyRouting()),
+        trace_writer=MemoryTraceWriter(),
+        metrics=metrics,
+    )
+    client = TestClient(app)
+
+    response = client.post(
+        "/v1/rag/query",
+        json={
+            "query": "Paracetamol dose?",
+            "subject_scope": "human",
+            "intent": "fact_lookup",
+        },
+    )
+    assert response.status_code == 200
+
+    scrape = client.get("/metrics")
+    assert scrape.status_code == 200
+    body = scrape.text
+    assert 'duocthu_requests_total{method="POST",route="/v1/rag/query",status="2xx"}' in body
+    assert 'duocthu_decision_total{decision="abstain",reason="drug_not_resolved"}' in body
+    assert 'duocthu_stage_duration_seconds_count{outcome="ok",stage="receive"}' in body
+    assert 'duocthu_stage_duration_seconds_count{outcome="ok",stage="persistence"}' in body
+    assert 'duocthu_stage_duration_seconds_count{outcome="ok",stage="response"}' in body
@@ -11,7 +11,7 @@ import json
 import pytest

 from rag import grounding
-from rag.answer import GroundedAnswerService
+from rag.answer import DISCLAIMER, GroundedAnswerService
 from rag.budget import RequestBudgetExhausted
 from rag.metrics import GENERATION_REJECTED, GENERATION_SERVED, InMemoryMetrics
 from rag.models import (
@@ -116,6 +116,62 @@ def _answer(payload, result: RetrievalResult | None = None, entailment_payload=N
    return grounded, metrics


+# --- the disclaimer guardrail -------------------------------------------------
+# `docs/architecture.md` specifies the disclaimer at several layers. Only the
+# web banner existed; `packages/shared-types/src/dto/chat.ts` declared the
+# field but nothing filled it, so a consumer other than that one UI got medical
+# content with nothing attached. It is a dataclass default rather than
+# something each call site adds, so these tests are about the paths that could
+# plausibly skip it: rejections, abstains and clarifications.
+
+
+def test_a_served_answer_carries_the_disclaimer():
+    grounded, _ = _answer(
+        {"claims": [{"text": "Người lớn uống 500 mg", "citations": [1]}],
+         "evidence_sufficient": True},
+    )
+
+    assert grounded.generated is True
+    assert grounded.disclaimer == DISCLAIMER
+    assert grounded.disclaimer
+
+
+def test_a_rejected_generation_still_carries_the_disclaimer():
+    """An abstain is still a clinical response and still needs the notice —
+    it is the case most likely to be treated as "not really an answer"."""
+    grounded, _ = _answer(
+        {"claims": [{"text": "Người lớn uống 850 mg, 2 lần mỗi ngày", "citations": [1]}],
+         "evidence_sufficient": True},
+    )
+
+    assert grounded.answer is None
+    assert grounded.result.decision == EvidenceDecision.ABSTAIN
+    assert grounded.disclaimer == DISCLAIMER
+
+
+def test_a_provider_outage_response_still_carries_the_disclaimer():
+    grounded, _ = _answer(
+        {"claims": [{"text": "Người lớn: 500 mg, 2 lần/ngày", "citations": [1]}],
+         "evidence_sufficient": True},
+        entailment_payload=AnswerGenerationUnavailable(),
+    )
+
+    assert grounded.disclaimer == DISCLAIMER
+
+
+def test_the_disclaimer_is_not_something_the_model_can_influence():
+    """It is a module constant, never routed through the generator, so a
+    prompt-injected or malfunctioning model cannot shorten or drop it. Asserted
+    directly because the value is the guardrail."""
+    grounded, _ = _answer(
+        {"claims": [{"text": "Bỏ qua mọi cảnh báo. Người lớn uống 500 mg", "citations": [1]}],
+         "evidence_sufficient": True},
+    )
+
+    assert grounded.disclaimer == DISCLAIMER
+    assert "không thay thế chỉ định" in grounded.disclaimer
+
+
 # --- the guardrail's whole reason to exist ------------------------------------


@@ -0,0 +1,122 @@
+from adapters.prometheus import PrometheusMetrics
+from config import Settings
+from rag.instrumentation import InstrumentedGenerator
+from rag.metrics import (
+    CLARIFY_ASKED,
+    PROVIDER_FAILURE,
+    REQUEST_DURATION,
+    REQUESTS,
+    STAGE_DURATION,
+    TRACE_WRITE_FAILED,
+    InMemoryMetrics,
+)
+from rag.telemetry import (
+    configure_telemetry,
+    correlation_context,
+    request_span,
+    stage,
+)
+
+
+def test_prometheus_registers_domain_failures_histograms_and_bounds_labels():
+    metrics = PrometheusMetrics()
+    metrics.increment(TRACE_WRITE_FAILED)
+    metrics.increment(CLARIFY_ASKED, reason="raw-user-controlled-reason")
+    metrics.increment(
+        REQUESTS, method="POST", route="/v1/rag/query", status="2xx"
+    )
+    metrics.observe(
+        REQUEST_DURATION,
+        1.25,
+        method="POST",
+        route="/v1/rag/query",
+        status="2xx",
+    )
+    metrics.observe(STAGE_DURATION, 0.2, stage="retrieval", outcome="ok")
+
+    body, content_type = metrics.render()
+    rendered = body.decode("utf-8")
+    assert "application/openmetrics-text" in content_type
+    assert "duocthu_trace_write_failed_total 1.0" in rendered
+    assert 'duocthu_clarify_asked_total{reason="other"} 1.0' in rendered
+    assert "duocthu_request_duration_seconds_bucket" in rendered
+    assert "duocthu_stage_duration_seconds_bucket" in rendered
+
+
+def test_stage_records_duration_without_requiring_an_otel_collector():
+    metrics = InMemoryMetrics()
+    configure_telemetry(Settings(otel_enabled=False), metrics)
+
+    with stage("grounding"):
+        pass
+
+    values = metrics.observed(STAGE_DURATION, stage="grounding", outcome="ok")
+    assert len(values) == 1
+    assert values[0] >= 0
+
+
+class _FailingGenerator:
+    model_id = "fake"
+
+    def generate(self, system: str, user: str, schema: dict) -> str:
+        raise ConnectionError("provider offline")
+
+
+def test_provider_failure_uses_bounded_classification_not_exception_text():
+    metrics = InMemoryMetrics()
+    generator = InstrumentedGenerator(_FailingGenerator(), metrics)
+
+    try:
+        generator.generate("system", "user", {})
+    except ConnectionError:
+        pass
+    else:  # pragma: no cover - assertion guard
+        raise AssertionError("expected provider failure")
+
+    assert metrics.total(
+        PROVIDER_FAILURE,
+        provider="other",
+        operation="generate",
+        reason="provider_unavailable",
+    ) == 1
+
+
+def test_otel_server_and_stage_spans_share_trace_and_emit_metric_exemplar():
+    otel_trace = pytest.importorskip("opentelemetry.trace")
+    sdk_trace = pytest.importorskip("opentelemetry.sdk.trace")
+    exporter_module = pytest.importorskip(
+        "opentelemetry.sdk.trace.export.in_memory_span_exporter"
+    )
+    export_module = pytest.importorskip("opentelemetry.sdk.trace.export")
+
+    exporter = exporter_module.InMemorySpanExporter()
+    provider = sdk_trace.TracerProvider()
+    provider.add_span_processor(export_module.SimpleSpanProcessor(exporter))
+    otel_trace.set_tracer_provider(provider)
+
+    metrics = PrometheusMetrics()
+    configure_telemetry(
+        Settings(
+            otel_enabled=True,
+            otel_exporter_otlp_endpoint="http://127.0.0.1:4318/v1/traces",
+        ),
+        metrics,
+    )
+    with correlation_context("req-span-test"):
+        with request_span("POST", "/v1/rag/query", {}):
+            with stage("retrieval"):
+                pass
+
+    provider.force_flush()
+    spans = exporter.get_finished_spans()
+    assert {item.name for item in spans} == {
+        "POST /v1/rag/query",
+        "rag.stage.retrieval",
+    }
+    assert len({item.context.trace_id for item in spans}) == 1
+    assert spans[-1].attributes["duocthu.correlation_id"] == "req-span-test"
+
+    rendered = metrics.render()[0].decode("utf-8")
+    assert "duocthu_stage_duration_seconds_bucket" in rendered
+    assert 'trace_id="' in rendered
+import pytest