Verify exact production traces and record rollout

apps/ai-service/rag/metrics.py

@@ -18,18 +18,28 @@ from typing import Protocol
 class Metrics(Protocol):
     def increment(self, name: str, **labels: str) -> None: ...
 
+    def observe(self, name: str, value: float, **labels: str) -> None: ...
+
 
 class NullMetrics:
     def increment(self, name: str, **labels: str) -> None:  # noqa: ARG002
         # Deliberately inert: the default when no metrics stack is configured.
         return None
 
+    def observe(
+        self, name: str, value: float, **labels: str  # noqa: ARG002
+    ) -> None:
+        return None
+
 
 class InMemoryMetrics:
     """Reference implementation of the contract; also what tests assert on."""
 
     def __init__(self) -> None:
         self.counts: dict[tuple[str, tuple[tuple[str, str], ...]], int] = {}
+        self.observations: dict[
+            tuple[str, tuple[tuple[str, str], ...]], list[float]
+        ] = {}
 
     def increment(self, name: str, **labels: str) -> None:
         key = (name, tuple(sorted(labels.items())))
@@ -40,6 +50,22 @@ class InMemoryMetrics:
             return self.counts.get((name, tuple(sorted(labels.items()))), 0)
         return sum(count for (n, _), count in self.counts.items() if n == name)
 
+    def observe(self, name: str, value: float, **labels: str) -> None:
+        key = (name, tuple(sorted(labels.items())))
+        self.observations.setdefault(key, []).append(value)
+
+    def observed(self, name: str, **labels: str) -> tuple[float, ...]:
+        if labels:
+            return tuple(
+                self.observations.get((name, tuple(sorted(labels.items()))), [])
+            )
+        return tuple(
+            value
+            for (metric_name, _), values in self.observations.items()
+            if metric_name == name
+            for value in values
+        )
+
 
 RETRIEVAL_ROUTE = "duocthu_retrieval_route_total"
 ABSTENTION = "duocthu_abstention_total"
@@ -60,3 +86,12 @@ FOLLOWUP_INHERITED = "duocthu_followup_inherited_total"
 # degradation actually happens, since a silent fail-open with no counter is
 # indistinguishable from tracing quietly working.
 TRACE_WRITE_FAILED = "duocthu_trace_write_failed_total"
+
+# HTTP and request-pipeline telemetry. Every label is a bounded vocabulary:
+# route is a route template (never a raw path), status is a class (2xx/4xx/5xx),
+# and stage/provider/reason values are normalized by the Prometheus adapter.
+REQUESTS = "duocthu_requests_total"
+REQUEST_DURATION = "duocthu_request_duration_seconds"
+STAGE_DURATION = "duocthu_stage_duration_seconds"
+DECISION = "duocthu_decision_total"
+PROVIDER_FAILURE = "duocthu_provider_failure_total"