Verify exact production traces and record rollout

apps/ai-service/rag/answer.py

@@ -99,12 +99,30 @@ class AnswerPlan:
     needs_warning: bool = False
 
 
+# A fixed, non-LLM string. `docs/architecture.md`'s guardrail section
+# specifies the disclaimer at several layers, and the web banner
+# (`packages/ui/src/DisclaimerBanner.tsx`) was the only one in place: the
+# `disclaimer` field declared in `packages/shared-types/src/dto/chat.ts` was
+# never populated, so any consumer other than this one web UI received medical
+# content with nothing attached. Keeping it out of the prompt is deliberate —
+# a disclaimer the model writes is one the model can also reword, shorten or
+# omit, and it would then have to be verified like any other generated claim.
+DISCLAIMER = (
+    "Nội dung được trích từ Dược thư Quốc gia Việt Nam 2018, phục vụ tra cứu "
+    "chuyên môn và không thay thế chỉ định của bác sĩ hoặc dược sĩ lâm sàng."
+)
+
+
 @dataclass(frozen=True)
 class GroundedAnswer:
     result: RetrievalResult
     answer: str | None
     citations: tuple[Citation, ...] = ()
     generated: bool = False
+    # Carried on the payload rather than added by the caller, so no response
+    # path can be built that omits it — including abstains and clarifications,
+    # which are also medical content in the sense that matters here.
+    disclaimer: str = DISCLAIMER
     # Set when the model decided the turn is under-specified and asked back
     # (e.g. a dose question with no age/weight). The answer field carries the
     # question; the caller renders it as a clarification, not a final answer.

apps/ai-service/rag/grounding.py

@@ -41,6 +41,8 @@ from __future__ import annotations
 import re
 from dataclasses import dataclass
 
+from .telemetry import traced_stage
+
 # A digit run with internal separators kept: "500", "7,5", "1.000".
 # Ranges ("4 - 6 giờ") yield two tokens, and each is checked on its own.
 _NUMBER = re.compile(r"\d+(?:[.,]\d+)*")
@@ -127,6 +129,7 @@ def split_claims(answer: str, evidence_count: int) -> tuple[Claim, ...]:
     return tuple(claims)
 
 
+@traced_stage("grounding")
 def verify(answer: str, evidence_texts: tuple[str, ...]) -> GroundingReport:
     """Whether `answer` states only figures and sources traceable to the
     specific evidence block(s) cited immediately after each claim.

apps/ai-service/rag/instrumentation.py

@@ -0,0 +1,240 @@
+"""Non-invasive instrumentation wrappers for the live RAG object graph.
+
+Claude owns ``rag/agent.py`` and ``rag/answer.py`` in the shared worktree.
+Subclasses here add spans at their stable method boundaries without changing
+those files or duplicating their domain decisions.
+"""
+from __future__ import annotations
+
+from typing import Any
+
+from .agent import RagAgent
+from .answer import GroundedAnswerService
+from .metrics import CLARIFY_ASKED, Metrics, PROVIDER_FAILURE, RETRIEVAL_ROUTE
+from .service import RetrievalService
+from .telemetry import (
+    annotate_current_span,
+    current_stage,
+    dependency_span,
+    stage,
+)
+
+
+def _provider_name(delegate: Any) -> str:
+    name = delegate.__class__.__name__.casefold()
+    if "cohere" in name:
+        return "bedrock_cohere"
+    if "converse" in name:
+        return "bedrock_converse"
+    if "claude" in name:
+        return "bedrock_claude"
+    return "other"
+
+
+def _failure_reason(exc: BaseException) -> str:
+    name = exc.__class__.__name__.casefold()
+    if "timeout" in name:
+        return "timeout"
+    if "budget" in name:
+        return "request_budget_exhausted"
+    if "unavailable" in name or "connection" in name:
+        return "provider_unavailable"
+    return "error"
+
+
+class InstrumentedGenerator:
+    def __init__(self, delegate: Any, metrics: Metrics) -> None:
+        self._delegate = delegate
+        self._metrics = metrics
+        self._provider = _provider_name(delegate)
+
+    @property
+    def model_id(self) -> str:
+        return self._delegate.model_id
+
+    def generate(self, system: str, user: str, schema: dict) -> str:
+        operation = {
+            "understanding": "understand",
+            "generation": "generate",
+            "entailment": "entailment",
+        }.get(current_stage(), "generate")
+        with dependency_span(self._provider, operation):
+            try:
+                return self._delegate.generate(system, user, schema)
+            except Exception as exc:
+                self._metrics.increment(
+                    PROVIDER_FAILURE,
+                    provider=self._provider,
+                    operation=operation,
+                    reason=_failure_reason(exc),
+                )
+                raise
+
+
+class InstrumentedEmbedder:
+    def __init__(self, delegate: Any, metrics: Metrics) -> None:
+        self._delegate = delegate
+        self._metrics = metrics
+
+    @property
+    def dimensions(self) -> int:
+        return self._delegate.dimensions
+
+    @property
+    def model_id(self) -> str:
+        return self._delegate.model_id
+
+    def embed_query(self, text: str) -> list[float]:
+        with dependency_span("bedrock_cohere", "embed"):
+            try:
+                return self._delegate.embed_query(text)
+            except Exception as exc:
+                self._metrics.increment(
+                    PROVIDER_FAILURE,
+                    provider="bedrock_cohere",
+                    operation="embed",
+                    reason=_failure_reason(exc),
+                )
+                raise
+
+
+class InstrumentedReranker:
+    def __init__(self, delegate: Any, metrics: Metrics) -> None:
+        self._delegate = delegate
+        self._metrics = metrics
+
+    def rerank(
+        self, query: str, documents: list[str], top_n: int | None = None
+    ) -> list[int]:
+        with dependency_span("bedrock_cohere", "rerank"):
+            try:
+                return self._delegate.rerank(query, documents, top_n=top_n)
+            except Exception as exc:
+                self._metrics.increment(
+                    PROVIDER_FAILURE,
+                    provider="bedrock_cohere",
+                    operation="rerank",
+                    reason=_failure_reason(exc),
+                )
+                raise
+
+
+class InstrumentedRetrievalService(RetrievalService):
+    def __init__(self, *args, metrics: Metrics, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        self._observability_metrics = metrics
+
+    def retrieve_framed(self, *args, **kwargs):
+        with stage("retrieval"):
+            section_key = args[1] if len(args) > 1 else kwargs.get("section_key")
+            is_overview = args[3] if len(args) > 3 else kwargs.get("is_overview", False)
+            route = "section" if section_key else (
+                "overview" if is_overview else "similarity"
+            )
+            self._observability_metrics.increment(RETRIEVAL_ROUTE, route=route)
+            try:
+                result = super().retrieve_framed(*args, **kwargs)
+            except Exception as exc:
+                self._record_retrieval_failure(exc)
+                raise
+            self._annotate_result(result)
+            return result
+
+    def retrieve(self, *args, **kwargs):
+        with stage("retrieval"):
+            self._observability_metrics.increment(RETRIEVAL_ROUTE, route="other")
+            try:
+                result = super().retrieve(*args, **kwargs)
+            except Exception as exc:
+                self._record_retrieval_failure(exc)
+                raise
+            self._annotate_result(result)
+            return result
+
+    def retrieve_by_indication(self, *args, **kwargs):
+        with stage("retrieval"):
+            self._observability_metrics.increment(RETRIEVAL_ROUTE, route="indication")
+            try:
+                result = super().retrieve_by_indication(*args, **kwargs)
+            except Exception as exc:
+                self._record_retrieval_failure(exc)
+                raise
+            self._annotate_result(result)
+            return result
+
+    def _record_retrieval_failure(self, exc: BaseException) -> None:
+        self._observability_metrics.increment(
+            PROVIDER_FAILURE,
+            provider="qdrant",
+            operation="retrieve",
+            reason=_failure_reason(exc),
+        )
+
+    @staticmethod
+    def _annotate_result(result) -> None:
+        annotate_current_span(
+            **{
+                "duocthu.decision": result.decision.value,
+                "duocthu.reason": result.reason,
+                "duocthu.evidence_count": len(result.evidence),
+            }
+        )
+
+    def _rerank(self, query, hits):
+        with stage("rerank", configured=self._reranker is not None):
+            return super()._rerank(query, hits)
+
+    def _decide(self, evidence, is_drug_overview=False):
+        with stage("evidence", evidence_count=len(evidence)):
+            return super()._decide(evidence, is_drug_overview=is_drug_overview)
+
+
+class InstrumentedGroundedAnswerService(GroundedAnswerService):
+    def _generate(self, *args, **kwargs):
+        with stage("generation"):
+            return super()._generate(*args, **kwargs)
+
+    def _verify_entailment(self, *args, **kwargs):
+        with stage("entailment"):
+            return super()._verify_entailment(*args, **kwargs)
+
+
+class InstrumentedQueryUnderstander:
+    def __init__(self, delegate: Any) -> None:
+        self._delegate = delegate
+
+    def understand(self, *args, **kwargs):
+        with stage("understanding"):
+            frame = self._delegate.understand(*args, **kwargs)
+            annotate_current_span(
+                **{
+                    "duocthu.turn_type": frame.turn_type,
+                    "duocthu.needs_clarify": frame.needs_clarify,
+                    "duocthu.system_error": frame.system_error,
+                }
+            )
+            return frame
+
+
+class InstrumentedRagAgent(RagAgent):
+    def __init__(self, *args, metrics: Metrics, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        self._observability_metrics = metrics
+
+    def handle(self, *args, **kwargs):
+        reply = super().handle(*args, **kwargs)
+        if reply.decision == "clarify":
+            self._observability_metrics.increment(CLARIFY_ASKED, reason=reply.reason)
+        return reply
+
+    def _get_history(self, *args, **kwargs):
+        with stage("context"):
+            return super()._get_history(*args, **kwargs)
+
+    def _route(self, *args, **kwargs):
+        with stage("routing"):
+            return super()._route(*args, **kwargs)
+
+    def _remember(self, *args, **kwargs):
+        with stage("persistence"):
+            return super()._remember(*args, **kwargs)

apps/ai-service/rag/metrics.py

@@ -18,18 +18,28 @@ from typing import Protocol
 class Metrics(Protocol):
     def increment(self, name: str, **labels: str) -> None: ...
 
+    def observe(self, name: str, value: float, **labels: str) -> None: ...
+
 
 class NullMetrics:
     def increment(self, name: str, **labels: str) -> None:  # noqa: ARG002
         # Deliberately inert: the default when no metrics stack is configured.
         return None
 
+    def observe(
+        self, name: str, value: float, **labels: str  # noqa: ARG002
+    ) -> None:
+        return None
+
 
 class InMemoryMetrics:
     """Reference implementation of the contract; also what tests assert on."""
 
     def __init__(self) -> None:
         self.counts: dict[tuple[str, tuple[tuple[str, str], ...]], int] = {}
+        self.observations: dict[
+            tuple[str, tuple[tuple[str, str], ...]], list[float]
+        ] = {}
 
     def increment(self, name: str, **labels: str) -> None:
         key = (name, tuple(sorted(labels.items())))
@@ -40,6 +50,22 @@ class InMemoryMetrics:
             return self.counts.get((name, tuple(sorted(labels.items()))), 0)
         return sum(count for (n, _), count in self.counts.items() if n == name)
 
+    def observe(self, name: str, value: float, **labels: str) -> None:
+        key = (name, tuple(sorted(labels.items())))
+        self.observations.setdefault(key, []).append(value)
+
+    def observed(self, name: str, **labels: str) -> tuple[float, ...]:
+        if labels:
+            return tuple(
+                self.observations.get((name, tuple(sorted(labels.items()))), [])
+            )
+        return tuple(
+            value
+            for (metric_name, _), values in self.observations.items()
+            if metric_name == name
+            for value in values
+        )
+
 
 RETRIEVAL_ROUTE = "duocthu_retrieval_route_total"
 ABSTENTION = "duocthu_abstention_total"
@@ -60,3 +86,12 @@ FOLLOWUP_INHERITED = "duocthu_followup_inherited_total"
 # degradation actually happens, since a silent fail-open with no counter is
 # indistinguishable from tracing quietly working.
 TRACE_WRITE_FAILED = "duocthu_trace_write_failed_total"
+
+# HTTP and request-pipeline telemetry. Every label is a bounded vocabulary:
+# route is a route template (never a raw path), status is a class (2xx/4xx/5xx),
+# and stage/provider/reason values are normalized by the Prometheus adapter.
+REQUESTS = "duocthu_requests_total"
+REQUEST_DURATION = "duocthu_request_duration_seconds"
+STAGE_DURATION = "duocthu_stage_duration_seconds"
+DECISION = "duocthu_decision_total"
+PROVIDER_FAILURE = "duocthu_provider_failure_total"

apps/ai-service/rag/telemetry.py

@@ -0,0 +1,217 @@
+"""Request correlation, OpenTelemetry spans and stage timing.
+
+This module deliberately has a no-op default. The RAG domain remains runnable
+without the OpenTelemetry packages or collector, while a configured runtime
+gets one trace context shared by FastAPI's async middleware and its sync
+thread-pool endpoint.
+"""
+from __future__ import annotations
+
+import re
+import uuid
+from contextlib import contextmanager
+from contextvars import ContextVar
+from functools import wraps
+from time import perf_counter
+from typing import Any, Iterator, Mapping
+
+from .metrics import Metrics, NullMetrics, STAGE_DURATION
+
+_CORRELATION_ID: ContextVar[str | None] = ContextVar(
+    "duocthu_correlation_id", default=None
+)
+_STAGE: ContextVar[str | None] = ContextVar("duocthu_stage", default=None)
+_SAFE_CORRELATION = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$")
+_metrics: Metrics = NullMetrics()
+
+try:
+    from opentelemetry import trace
+
+    _tracer = trace.get_tracer("duocthu.ai-service")
+except ImportError:  # pragma: no cover - exercised only in minimal installs
+    trace = None
+    _tracer = None
+
+
+def configure_telemetry(settings: Any, metrics: Metrics | None = None) -> bool:
+    """Configure one OTLP tracer provider; return whether tracing is active.
+
+    Metrics are configured independently and continue to work when tracing is
+    disabled or optional OpenTelemetry packages are absent.
+    """
+    global _metrics, _tracer
+    _metrics = metrics or NullMetrics()
+    if not getattr(settings, "otel_enabled", False):
+        return False
+
+    try:
+        from opentelemetry import trace as otel_trace
+        from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
+        from opentelemetry.sdk.resources import Resource
+        from opentelemetry.sdk.trace import TracerProvider
+        from opentelemetry.sdk.trace.export import BatchSpanProcessor
+        from opentelemetry.sdk.trace.sampling import ParentBased, TraceIdRatioBased
+    except ImportError:
+        return False
+
+    current = otel_trace.get_tracer_provider()
+    # OpenTelemetry only permits setting the global provider once. Reuse a
+    # provider installed by a host/test instead of replacing it and warning.
+    if isinstance(current, TracerProvider):
+        _tracer = current.get_tracer("duocthu.ai-service")
+        return True
+
+    provider = TracerProvider(
+        resource=Resource.create(
+            {
+                "service.name": getattr(settings, "otel_service_name", "ai-service"),
+                "service.version": "0.1.0",
+                "deployment.environment": getattr(settings, "environment", "local"),
+            }
+        ),
+        sampler=ParentBased(
+            TraceIdRatioBased(float(getattr(settings, "otel_sample_ratio", 1.0)))
+        ),
+    )
+    exporter = OTLPSpanExporter(endpoint=settings.otel_exporter_otlp_endpoint)
+    provider.add_span_processor(BatchSpanProcessor(exporter))
+    otel_trace.set_tracer_provider(provider)
+    _tracer = provider.get_tracer("duocthu.ai-service")
+    return True
+
+
+def normalize_correlation_id(candidate: str | None) -> str:
+    value = (candidate or "").strip()
+    return value if _SAFE_CORRELATION.fullmatch(value) else str(uuid.uuid4())
+
+
+@contextmanager
+def correlation_context(candidate: str | None) -> Iterator[str]:
+    correlation_id = normalize_correlation_id(candidate)
+    token = _CORRELATION_ID.set(correlation_id)
+    try:
+        yield correlation_id
+    finally:
+        _CORRELATION_ID.reset(token)
+
+
+def current_correlation_id() -> str:
+    value = _CORRELATION_ID.get()
+    return value or str(uuid.uuid4())
+
+
+def current_trace_id() -> str | None:
+    if trace is None:
+        return None
+    context = trace.get_current_span().get_span_context()
+    if not context.is_valid:
+        return None
+    return f"{context.trace_id:032x}"
+
+
+def current_stage() -> str | None:
+    return _STAGE.get()
+
+
+@contextmanager
+def request_span(
+    method: str, route: str, headers: Mapping[str, str]
+) -> Iterator[Any]:
+    """Start a server span and extract an incoming W3C ``traceparent``."""
+    if _tracer is None:
+        yield None
+        return
+    try:
+        from opentelemetry import propagate
+        from opentelemetry.trace import SpanKind
+    except ImportError:  # pragma: no cover - minimal install fallback
+        yield None
+        return
+
+    parent = propagate.extract(headers)
+    with _tracer.start_as_current_span(
+        f"{method} {route}",
+        context=parent,
+        kind=SpanKind.SERVER,
+        attributes={
+            "http.request.method": method,
+            "http.route": route,
+            "duocthu.correlation_id": current_correlation_id(),
+        },
+    ) as span:
+        yield span
+
+
+@contextmanager
+def dependency_span(provider: str, operation: str) -> Iterator[Any]:
+    """Trace one external dependency call without creating another histogram."""
+    if _tracer is None:
+        yield None
+        return
+    with _tracer.start_as_current_span(
+        f"provider.{provider}.{operation}",
+        attributes={
+            "duocthu.provider": provider,
+            "duocthu.operation": operation,
+        },
+    ) as span:
+        yield span
+
+
+@contextmanager
+def stage(name: str, **attributes: Any) -> Iterator[Any]:
+    """Create a child span and observe a low-cardinality stage histogram."""
+    started = perf_counter()
+    outcome = "ok"
+    span = None
+    token = _STAGE.set(name)
+    try:
+        if _tracer is None:
+            yield None
+        else:
+            safe_attributes = {
+                key: value
+                for key, value in attributes.items()
+                if value is not None and isinstance(value, (str, bool, int, float))
+            }
+            safe_attributes["duocthu.stage"] = name
+            with _tracer.start_as_current_span(
+                f"rag.stage.{name}", attributes=safe_attributes
+            ) as span:
+                yield span
+    except BaseException as exc:
+        outcome = "cancelled" if exc.__class__.__name__ == "CancelledError" else "error"
+        if span is not None:
+            span.set_attribute("duocthu.outcome", outcome)
+        raise
+    finally:
+        elapsed = perf_counter() - started
+        if span is not None:
+            span.set_attribute("duocthu.outcome", outcome)
+            span.set_attribute("duocthu.duration_ms", elapsed * 1000)
+        _metrics.observe(STAGE_DURATION, elapsed, stage=name, outcome=outcome)
+        _STAGE.reset(token)
+
+
+def annotate_current_span(**attributes: Any) -> None:
+    if trace is None:
+        return
+    span = trace.get_current_span()
+    if not span.is_recording():
+        return
+    for key, value in attributes.items():
+        if value is not None and isinstance(value, (str, bool, int, float)):
+            span.set_attribute(key, value)
+
+
+def traced_stage(name: str):
+    """Decorator form for pure functions that should remain otherwise untouched."""
+    def decorate(function):
+        @wraps(function)
+        def wrapped(*args, **kwargs):
+            with stage(name):
+                return function(*args, **kwargs)
+
+        return wrapped
+
+    return decorate