57 lines
2.2 KiB
Python
57 lines
2.2 KiB
Python
"""Domain counters, defined here so the numbers on a dashboard are the
|
|
numbers the domain actually decided.
|
|
|
|
Kept behind a tiny protocol rather than importing `prometheus_client` into
|
|
`rag/`: the domain records that a generation was refused for an ungrounded
|
|
number, and the process that happens to expose Prometheus does the exporting.
|
|
`NullMetrics` is the default, so tests and any deployment without a metrics
|
|
stack run unchanged.
|
|
|
|
The counter that matters is `generation_rejected` — it is the measured form of
|
|
the claim that the answer layer cannot state a figure the book does not.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from typing import Protocol
|
|
|
|
|
|
class Metrics(Protocol):
|
|
def increment(self, name: str, **labels: str) -> None: ...
|
|
|
|
|
|
class NullMetrics:
|
|
def increment(self, name: str, **labels: str) -> None: # noqa: ARG002
|
|
# Deliberately inert: the default when no metrics stack is configured.
|
|
return None
|
|
|
|
|
|
class InMemoryMetrics:
|
|
"""Reference implementation of the contract; also what tests assert on."""
|
|
|
|
def __init__(self) -> None:
|
|
self.counts: dict[tuple[str, tuple[tuple[str, str], ...]], int] = {}
|
|
|
|
def increment(self, name: str, **labels: str) -> None:
|
|
key = (name, tuple(sorted(labels.items())))
|
|
self.counts[key] = self.counts.get(key, 0) + 1
|
|
|
|
def total(self, name: str, **labels: str) -> int:
|
|
if labels:
|
|
return self.counts.get((name, tuple(sorted(labels.items()))), 0)
|
|
return sum(count for (n, _), count in self.counts.items() if n == name)
|
|
|
|
|
|
RETRIEVAL_ROUTE = "duocthu_retrieval_route_total"
|
|
ABSTENTION = "duocthu_abstention_total"
|
|
GENERATION_REJECTED = "duocthu_generation_rejected_total"
|
|
GENERATION_SERVED = "duocthu_generation_served_total"
|
|
ANSWER_EXTRACTIVE = "duocthu_answer_extractive_total"
|
|
|
|
# Conversational loop. `CLARIFY_ASKED` is the counter that shows the system
|
|
# asking instead of guessing — the behaviour a reviewer will probe first.
|
|
CLARIFY_ASKED = "duocthu_clarify_asked_total"
|
|
LOOP_ROUNDS = "duocthu_loop_retrieval_rounds_total"
|
|
LOOP_REFINED = "duocthu_loop_refined_total"
|
|
LOOP_REPAIRED = "duocthu_loop_repaired_total"
|
|
FOLLOWUP_INHERITED = "duocthu_followup_inherited_total"
|