123 lines
3.8 KiB
Python
123 lines
3.8 KiB
Python
from adapters.prometheus import PrometheusMetrics
|
|
from config import Settings
|
|
from rag.instrumentation import InstrumentedGenerator
|
|
from rag.metrics import (
|
|
CLARIFY_ASKED,
|
|
PROVIDER_FAILURE,
|
|
REQUEST_DURATION,
|
|
REQUESTS,
|
|
STAGE_DURATION,
|
|
TRACE_WRITE_FAILED,
|
|
InMemoryMetrics,
|
|
)
|
|
from rag.telemetry import (
|
|
configure_telemetry,
|
|
correlation_context,
|
|
request_span,
|
|
stage,
|
|
)
|
|
|
|
|
|
def test_prometheus_registers_domain_failures_histograms_and_bounds_labels():
|
|
metrics = PrometheusMetrics()
|
|
metrics.increment(TRACE_WRITE_FAILED)
|
|
metrics.increment(CLARIFY_ASKED, reason="raw-user-controlled-reason")
|
|
metrics.increment(
|
|
REQUESTS, method="POST", route="/v1/rag/query", status="2xx"
|
|
)
|
|
metrics.observe(
|
|
REQUEST_DURATION,
|
|
1.25,
|
|
method="POST",
|
|
route="/v1/rag/query",
|
|
status="2xx",
|
|
)
|
|
metrics.observe(STAGE_DURATION, 0.2, stage="retrieval", outcome="ok")
|
|
|
|
body, content_type = metrics.render()
|
|
rendered = body.decode("utf-8")
|
|
assert "application/openmetrics-text" in content_type
|
|
assert "duocthu_trace_write_failed_total 1.0" in rendered
|
|
assert 'duocthu_clarify_asked_total{reason="other"} 1.0' in rendered
|
|
assert "duocthu_request_duration_seconds_bucket" in rendered
|
|
assert "duocthu_stage_duration_seconds_bucket" in rendered
|
|
|
|
|
|
def test_stage_records_duration_without_requiring_an_otel_collector():
|
|
metrics = InMemoryMetrics()
|
|
configure_telemetry(Settings(otel_enabled=False), metrics)
|
|
|
|
with stage("grounding"):
|
|
pass
|
|
|
|
values = metrics.observed(STAGE_DURATION, stage="grounding", outcome="ok")
|
|
assert len(values) == 1
|
|
assert values[0] >= 0
|
|
|
|
|
|
class _FailingGenerator:
|
|
model_id = "fake"
|
|
|
|
def generate(self, system: str, user: str, schema: dict) -> str:
|
|
raise ConnectionError("provider offline")
|
|
|
|
|
|
def test_provider_failure_uses_bounded_classification_not_exception_text():
|
|
metrics = InMemoryMetrics()
|
|
generator = InstrumentedGenerator(_FailingGenerator(), metrics)
|
|
|
|
try:
|
|
generator.generate("system", "user", {})
|
|
except ConnectionError:
|
|
pass
|
|
else: # pragma: no cover - assertion guard
|
|
raise AssertionError("expected provider failure")
|
|
|
|
assert metrics.total(
|
|
PROVIDER_FAILURE,
|
|
provider="other",
|
|
operation="generate",
|
|
reason="provider_unavailable",
|
|
) == 1
|
|
|
|
|
|
def test_otel_server_and_stage_spans_share_trace_and_emit_metric_exemplar():
|
|
otel_trace = pytest.importorskip("opentelemetry.trace")
|
|
sdk_trace = pytest.importorskip("opentelemetry.sdk.trace")
|
|
exporter_module = pytest.importorskip(
|
|
"opentelemetry.sdk.trace.export.in_memory_span_exporter"
|
|
)
|
|
export_module = pytest.importorskip("opentelemetry.sdk.trace.export")
|
|
|
|
exporter = exporter_module.InMemorySpanExporter()
|
|
provider = sdk_trace.TracerProvider()
|
|
provider.add_span_processor(export_module.SimpleSpanProcessor(exporter))
|
|
otel_trace.set_tracer_provider(provider)
|
|
|
|
metrics = PrometheusMetrics()
|
|
configure_telemetry(
|
|
Settings(
|
|
otel_enabled=True,
|
|
otel_exporter_otlp_endpoint="http://127.0.0.1:4318/v1/traces",
|
|
),
|
|
metrics,
|
|
)
|
|
with correlation_context("req-span-test"):
|
|
with request_span("POST", "/v1/rag/query", {}):
|
|
with stage("retrieval"):
|
|
pass
|
|
|
|
provider.force_flush()
|
|
spans = exporter.get_finished_spans()
|
|
assert {item.name for item in spans} == {
|
|
"POST /v1/rag/query",
|
|
"rag.stage.retrieval",
|
|
}
|
|
assert len({item.context.trace_id for item in spans}) == 1
|
|
assert spans[-1].attributes["duocthu.correlation_id"] == "req-span-test"
|
|
|
|
rendered = metrics.render()[0].decode("utf-8")
|
|
assert "duocthu_stage_duration_seconds_bucket" in rendered
|
|
assert 'trace_id="' in rendered
|
|
import pytest
|