77 lines
2.7 KiB
Python
77 lines
2.7 KiB
Python
"""Exports the domain's counters; the only module that names prometheus_client.
|
|
|
|
`rag/metrics.py` defines what is counted and why. This decides how it leaves
|
|
the process, and is imported lazily so the service runs — and the suite passes
|
|
— with no metrics stack installed.
|
|
|
|
Counter names carry a `duocthu_` prefix and a `_total` suffix because that is
|
|
what Prometheus expects of a counter; the dashboard queries them by name.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from typing import Any
|
|
|
|
from rag.metrics import (
|
|
ABSTENTION,
|
|
ANSWER_EXTRACTIVE,
|
|
GENERATION_REJECTED,
|
|
GENERATION_SERVED,
|
|
RETRIEVAL_ROUTE,
|
|
)
|
|
|
|
_LABELS: dict[str, tuple[str, ...]] = {
|
|
ABSTENTION: ("reason",),
|
|
GENERATION_REJECTED: ("reason",),
|
|
RETRIEVAL_ROUTE: ("route",),
|
|
GENERATION_SERVED: (),
|
|
ANSWER_EXTRACTIVE: (),
|
|
}
|
|
|
|
_HELP = {
|
|
ABSTENTION: "Answers refused, by the reason retrieval gave.",
|
|
GENERATION_REJECTED: (
|
|
"Generations discarded before reaching the caller. `reason=\"ungrounded_number\"` "
|
|
"counts answers that stated a figure absent from the cited source; "
|
|
"`reason=\"uncited_claim\"` counts claims with no valid citation at all; "
|
|
"`reason=\"unsupported_claim\"` counts claims the entailment pass judged "
|
|
"not actually stated by the block they cite."
|
|
),
|
|
GENERATION_SERVED: "Generations that passed grounding verification and were served.",
|
|
ANSWER_EXTRACTIVE: "Answers served as verbatim source text.",
|
|
RETRIEVAL_ROUTE: "Retrievals by route: section filter, or similarity fallback.",
|
|
}
|
|
|
|
|
|
class PrometheusMetrics:
|
|
"""Domain `Metrics` backed by a Prometheus registry."""
|
|
|
|
def __init__(self, registry: Any | None = None) -> None:
|
|
from prometheus_client import CollectorRegistry, Counter
|
|
|
|
self._registry = registry or CollectorRegistry()
|
|
self._counters = {
|
|
name: Counter(name, _HELP[name], labels, registry=self._registry)
|
|
for name, labels in _LABELS.items()
|
|
}
|
|
|
|
@property
|
|
def registry(self) -> Any:
|
|
return self._registry
|
|
|
|
def increment(self, name: str, **labels: str) -> None:
|
|
counter = self._counters.get(name)
|
|
if counter is None:
|
|
return
|
|
# An unexpected label would raise at scrape time, far from its cause.
|
|
# Metrics must not be able to break a clinical answer, so a mismatch
|
|
# drops the sample rather than the request.
|
|
expected = set(_LABELS[name])
|
|
if set(labels) != expected:
|
|
return
|
|
(counter.labels(**labels) if labels else counter).inc()
|
|
|
|
def render(self) -> tuple[bytes, str]:
|
|
from prometheus_client import CONTENT_TYPE_LATEST, generate_latest
|
|
|
|
return generate_latest(self._registry), CONTENT_TYPE_LATEST
|