Verify exact production traces and record rollout

2026-08-11 11:29:59 +07:00
parent 6b8f7584ed
commit 59e6ad2d0d
46 changed files with 2795 additions and 290 deletions
@@ -0,0 +1,122 @@
+from adapters.prometheus import PrometheusMetrics
+from config import Settings
+from rag.instrumentation import InstrumentedGenerator
+from rag.metrics import (
+    CLARIFY_ASKED,
+    PROVIDER_FAILURE,
+    REQUEST_DURATION,
+    REQUESTS,
+    STAGE_DURATION,
+    TRACE_WRITE_FAILED,
+    InMemoryMetrics,
+)
+from rag.telemetry import (
+    configure_telemetry,
+    correlation_context,
+    request_span,
+    stage,
+)
+
+
+def test_prometheus_registers_domain_failures_histograms_and_bounds_labels():
+    metrics = PrometheusMetrics()
+    metrics.increment(TRACE_WRITE_FAILED)
+    metrics.increment(CLARIFY_ASKED, reason="raw-user-controlled-reason")
+    metrics.increment(
+        REQUESTS, method="POST", route="/v1/rag/query", status="2xx"
+    )
+    metrics.observe(
+        REQUEST_DURATION,
+        1.25,
+        method="POST",
+        route="/v1/rag/query",
+        status="2xx",
+    )
+    metrics.observe(STAGE_DURATION, 0.2, stage="retrieval", outcome="ok")
+
+    body, content_type = metrics.render()
+    rendered = body.decode("utf-8")
+    assert "application/openmetrics-text" in content_type
+    assert "duocthu_trace_write_failed_total 1.0" in rendered
+    assert 'duocthu_clarify_asked_total{reason="other"} 1.0' in rendered
+    assert "duocthu_request_duration_seconds_bucket" in rendered
+    assert "duocthu_stage_duration_seconds_bucket" in rendered
+
+
+def test_stage_records_duration_without_requiring_an_otel_collector():
+    metrics = InMemoryMetrics()
+    configure_telemetry(Settings(otel_enabled=False), metrics)
+
+    with stage("grounding"):
+        pass
+
+    values = metrics.observed(STAGE_DURATION, stage="grounding", outcome="ok")
+    assert len(values) == 1
+    assert values[0] >= 0
+
+
+class _FailingGenerator:
+    model_id = "fake"
+
+    def generate(self, system: str, user: str, schema: dict) -> str:
+        raise ConnectionError("provider offline")
+
+
+def test_provider_failure_uses_bounded_classification_not_exception_text():
+    metrics = InMemoryMetrics()
+    generator = InstrumentedGenerator(_FailingGenerator(), metrics)
+
+    try:
+        generator.generate("system", "user", {})
+    except ConnectionError:
+        pass
+    else:  # pragma: no cover - assertion guard
+        raise AssertionError("expected provider failure")
+
+    assert metrics.total(
+        PROVIDER_FAILURE,
+        provider="other",
+        operation="generate",
+        reason="provider_unavailable",
+    ) == 1
+
+
+def test_otel_server_and_stage_spans_share_trace_and_emit_metric_exemplar():
+    otel_trace = pytest.importorskip("opentelemetry.trace")
+    sdk_trace = pytest.importorskip("opentelemetry.sdk.trace")
+    exporter_module = pytest.importorskip(
+        "opentelemetry.sdk.trace.export.in_memory_span_exporter"
+    )
+    export_module = pytest.importorskip("opentelemetry.sdk.trace.export")
+
+    exporter = exporter_module.InMemorySpanExporter()
+    provider = sdk_trace.TracerProvider()
+    provider.add_span_processor(export_module.SimpleSpanProcessor(exporter))
+    otel_trace.set_tracer_provider(provider)
+
+    metrics = PrometheusMetrics()
+    configure_telemetry(
+        Settings(
+            otel_enabled=True,
+            otel_exporter_otlp_endpoint="http://127.0.0.1:4318/v1/traces",
+        ),
+        metrics,
+    )
+    with correlation_context("req-span-test"):
+        with request_span("POST", "/v1/rag/query", {}):
+            with stage("retrieval"):
+                pass
+
+    provider.force_flush()
+    spans = exporter.get_finished_spans()
+    assert {item.name for item in spans} == {
+        "POST /v1/rag/query",
+        "rag.stage.retrieval",
+    }
+    assert len({item.context.trace_id for item in spans}) == 1
+    assert spans[-1].attributes["duocthu.correlation_id"] == "req-span-test"
+
+    rendered = metrics.render()[0].decode("utf-8")
+    assert "duocthu_stage_duration_seconds_bucket" in rendered
+    assert 'trace_id="' in rendered
+import pytest