98 lines
3.8 KiB
Python
98 lines
3.8 KiB
Python
"""Domain counters, defined here so the numbers on a dashboard are the
|
|
numbers the domain actually decided.
|
|
|
|
Kept behind a tiny protocol rather than importing `prometheus_client` into
|
|
`rag/`: the domain records that a generation was refused for an ungrounded
|
|
number, and the process that happens to expose Prometheus does the exporting.
|
|
`NullMetrics` is the default, so tests and any deployment without a metrics
|
|
stack run unchanged.
|
|
|
|
The counter that matters is `generation_rejected` — it is the measured form of
|
|
the claim that the answer layer cannot state a figure the book does not.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from typing import Protocol
|
|
|
|
|
|
class Metrics(Protocol):
|
|
def increment(self, name: str, **labels: str) -> None: ...
|
|
|
|
def observe(self, name: str, value: float, **labels: str) -> None: ...
|
|
|
|
|
|
class NullMetrics:
|
|
def increment(self, name: str, **labels: str) -> None: # noqa: ARG002
|
|
# Deliberately inert: the default when no metrics stack is configured.
|
|
return None
|
|
|
|
def observe(
|
|
self, name: str, value: float, **labels: str # noqa: ARG002
|
|
) -> None:
|
|
return None
|
|
|
|
|
|
class InMemoryMetrics:
|
|
"""Reference implementation of the contract; also what tests assert on."""
|
|
|
|
def __init__(self) -> None:
|
|
self.counts: dict[tuple[str, tuple[tuple[str, str], ...]], int] = {}
|
|
self.observations: dict[
|
|
tuple[str, tuple[tuple[str, str], ...]], list[float]
|
|
] = {}
|
|
|
|
def increment(self, name: str, **labels: str) -> None:
|
|
key = (name, tuple(sorted(labels.items())))
|
|
self.counts[key] = self.counts.get(key, 0) + 1
|
|
|
|
def total(self, name: str, **labels: str) -> int:
|
|
if labels:
|
|
return self.counts.get((name, tuple(sorted(labels.items()))), 0)
|
|
return sum(count for (n, _), count in self.counts.items() if n == name)
|
|
|
|
def observe(self, name: str, value: float, **labels: str) -> None:
|
|
key = (name, tuple(sorted(labels.items())))
|
|
self.observations.setdefault(key, []).append(value)
|
|
|
|
def observed(self, name: str, **labels: str) -> tuple[float, ...]:
|
|
if labels:
|
|
return tuple(
|
|
self.observations.get((name, tuple(sorted(labels.items()))), [])
|
|
)
|
|
return tuple(
|
|
value
|
|
for (metric_name, _), values in self.observations.items()
|
|
if metric_name == name
|
|
for value in values
|
|
)
|
|
|
|
|
|
RETRIEVAL_ROUTE = "duocthu_retrieval_route_total"
|
|
ABSTENTION = "duocthu_abstention_total"
|
|
GENERATION_REJECTED = "duocthu_generation_rejected_total"
|
|
GENERATION_SERVED = "duocthu_generation_served_total"
|
|
ANSWER_EXTRACTIVE = "duocthu_answer_extractive_total"
|
|
|
|
# Conversational loop. `CLARIFY_ASKED` is the counter that shows the system
|
|
# asking instead of guessing — the behaviour a reviewer will probe first.
|
|
CLARIFY_ASKED = "duocthu_clarify_asked_total"
|
|
LOOP_ROUNDS = "duocthu_loop_retrieval_rounds_total"
|
|
LOOP_REFINED = "duocthu_loop_refined_total"
|
|
LOOP_REPAIRED = "duocthu_loop_repaired_total"
|
|
FOLLOWUP_INHERITED = "duocthu_followup_inherited_total"
|
|
|
|
# Trace persistence is fail-open (F-09): a Postgres outage must not turn an
|
|
# already-computed, safe answer into a 500. This counts how often that
|
|
# degradation actually happens, since a silent fail-open with no counter is
|
|
# indistinguishable from tracing quietly working.
|
|
TRACE_WRITE_FAILED = "duocthu_trace_write_failed_total"
|
|
|
|
# HTTP and request-pipeline telemetry. Every label is a bounded vocabulary:
|
|
# route is a route template (never a raw path), status is a class (2xx/4xx/5xx),
|
|
# and stage/provider/reason values are normalized by the Prometheus adapter.
|
|
REQUESTS = "duocthu_requests_total"
|
|
REQUEST_DURATION = "duocthu_request_duration_seconds"
|
|
STAGE_DURATION = "duocthu_stage_duration_seconds"
|
|
DECISION = "duocthu_decision_total"
|
|
PROVIDER_FAILURE = "duocthu_provider_failure_total"
|