Make a Langfuse trace worth opening: question, answer, session, no probe noise

This commit is contained in:
2026-08-21 14:45:32 +07:00
parent 53582b6030
commit f3eaab0948
16 changed files with 733 additions and 5 deletions
@@ -0,0 +1,82 @@
"""Post scores to a Langfuse project, or do nothing at all.
Traces already reach Langfuse as OTLP spans (see rag/telemetry.py). A *score*
is the other half: it attaches a number to one of those traces, which is what
turns Langfuse from a trace viewer into a place where answer quality is
visible over time.
Two producers exist:
- `scripts/score_evals_ragas.py` posts Ragas metrics for eval runs.
- This adapter posts the user's thumb up/down from `/v1/rag/feedback`,
which is the only signal in the whole system that comes from a human
rather than a model judging a model.
Deliberately best-effort. Feedback is stored in Postgres first and that write
is what must not fail; a Langfuse outage may not turn a working thumbs-up into
a 503 for the user. Every failure here is swallowed and counted, never raised.
Scores attach by OpenTelemetry trace id (32 hex), NOT by the `trace_id` UUID
that identifies the Postgres row -- Langfuse has never seen that one. The
caller is responsible for looking the OTel id up.
"""
from __future__ import annotations
import base64
import json
import urllib.request
from typing import Any
class LangfuseScoreClient:
"""Minimal client: one endpoint, one verb, no dependency on the SDK."""
def __init__(self, base_url: str, public_key: str, secret_key: str, timeout: float = 5.0):
self.endpoint = f"{base_url.rstrip('/')}/api/public/scores"
token = base64.b64encode(f"{public_key}:{secret_key}".encode()).decode()
self._auth = f"Basic {token}"
self._timeout = timeout
@classmethod
def from_settings(cls, settings: Any) -> "LangfuseScoreClient | None":
"""Return a client, or None when Langfuse is not fully configured.
Same all-or-nothing rule as the span exporter: a base URL without keys
would authenticate nothing, so treat a partial config as "off" rather
than posting requests that can only 401.
"""
base_url = str(getattr(settings, "langfuse_base_url", "") or "").strip()
public_key = str(getattr(settings, "langfuse_public_key", "") or "").strip()
secret_key = str(getattr(settings, "langfuse_secret_key", "") or "").strip()
if not (base_url and public_key and secret_key):
return None
return cls(base_url, public_key, secret_key)
def post_score(
self,
*,
otel_trace_id: str,
name: str,
value: float,
comment: str | None = None,
) -> bool:
"""True if Langfuse accepted the score. Never raises."""
body: dict[str, Any] = {
"traceId": otel_trace_id,
"name": name,
"value": value,
"dataType": "NUMERIC",
}
if comment:
body["comment"] = comment
request = urllib.request.Request(
self.endpoint,
data=json.dumps(body).encode("utf-8"),
headers={"Content-Type": "application/json", "Authorization": self._auth},
method="POST",
)
try:
with urllib.request.urlopen(request, timeout=self._timeout):
return True
except Exception: # noqa: BLE001 - see module docstring: never raise
return False
+22
View File
@@ -191,6 +191,28 @@ class PostgresTraceRepository:
raise FeedbackTraceNotFound(trace_id)
return str(row[0])
def otel_trace_id_for(self, trace_id: str) -> str | None:
"""The OpenTelemetry id of a persisted answer, for attaching a score.
Separate from `save_feedback` rather than folded into its RETURNING
clause: that method's `-> str` shape is part of the `TraceWriter`
protocol the router depends on, and widening it to carry an unrelated
id would make every implementation and test carry it too. Feedback is
rare enough that a second short query costs nothing.
None when the row has no OTel id -- true for every answer produced
while tracing was off, which is most of the history predating
2026-08-21. Callers skip posting rather than inventing one.
"""
import psycopg
with psycopg.connect(self._dsn, connect_timeout=5) as connection:
row = connection.execute(
"SELECT otel_trace_id FROM rag_retrieval_trace WHERE trace_id = %s",
(trace_id,),
).fetchone()
return str(row[0]) if row and row[0] else None
class PostgresConversationStore:
"""Durable, cross-worker replacement for `RagAgent`'s in-process
+8
View File
@@ -65,6 +65,14 @@ class Settings(BaseSettings):
otel_service_name: str = "ai-service"
otel_exporter_otlp_endpoint: str = "http://localhost:4318/v1/traces"
otel_sample_ratio: float = Field(default=1.0, ge=0.0, le=1.0)
# Langfuse ingests the SAME spans over OTLP as a second exporter, so it
# adds an LLM-shaped view (per-trace answers, eval scores) without
# replacing Tempo or touching any instrumentation. All three must be set
# for the exporter to be added; keys are secrets and live only in .env /
# the cluster Secret, never in values.yaml.
langfuse_base_url: str = ""
langfuse_public_key: str = ""
langfuse_secret_key: str = ""
entities_path: Path = _default_entities_path()
# F-08: a per-turn budget across RagAgent's sequential Bedrock calls
# (understand, generate, one entailment check on the live agent path).
+21 -1
View File
@@ -1,11 +1,13 @@
from __future__ import annotations
from contextlib import nullcontext
from hmac import compare_digest
from time import perf_counter
from typing import Any
from fastapi import FastAPI, Request, Response
from adapters.langfuse_scores import LangfuseScoreClient
from adapters.postgres import PostgresTraceRepository
from bootstrap import build_runtime
from config import Settings, get_settings
@@ -38,6 +40,10 @@ def create_app(
app.state.conversational = conversational
app.state.trace_writer = trace_writer
app.state.metrics = metrics
# None unless Langfuse is fully configured; `/v1/rag/feedback` then skips
# mirroring the user's verdict there. Built from the same settings the
# span exporter reads, so the two can never point at different projects.
app.state.langfuse_scores = LangfuseScoreClient.from_settings(configured)
# The raw Qdrant retriever, not routed through RagAgent — Feature-List
# #4/#23's section-list and verbatim-section-text endpoints are plain
# payload-filtered reads with no LLM/generation step, so nothing about
@@ -51,8 +57,19 @@ def create_app(
method = request.method.upper()
started = perf_counter()
status_code = 500
# Kubernetes probes and the Prometheus scrape run every few seconds
# forever and carry no information a trace could show. Left traced they
# were ~90% of all spans within a day, burying the RAG turns anyone
# actually opens a trace viewer to read, and costing storage and
# ingestion for nothing. Their Prometheus counters below are untouched,
# which is the right place to watch probe health anyway.
span_context = (
nullcontext(None)
if route in _UNTRACED_ROUTES
else request_span(method, route, request.headers)
)
with correlation_context(request.headers.get("x-correlation-id")) as correlation_id:
with request_span(method, route, request.headers) as span:
with span_context as span:
try:
response = await call_next(request)
status_code = response.status_code
@@ -126,6 +143,9 @@ def create_app(
return app
_UNTRACED_ROUTES = frozenset({"/health", "/ready", "/metrics"})
def _route_label(path: str) -> str:
known = {
"/health", "/ready", "/metrics", "/v1/rag/query", "/v1/rag/suggest",
+27
View File
@@ -7,6 +7,7 @@ thread-pool endpoint.
"""
from __future__ import annotations
import base64
import re
import uuid
from contextlib import contextmanager
@@ -75,11 +76,37 @@ def configure_telemetry(settings: Any, metrics: Metrics | None = None) -> bool:
)
exporter = OTLPSpanExporter(endpoint=settings.otel_exporter_otlp_endpoint)
provider.add_span_processor(BatchSpanProcessor(exporter))
for extra in _langfuse_exporters(settings, OTLPSpanExporter):
provider.add_span_processor(BatchSpanProcessor(extra))
otel_trace.set_tracer_provider(provider)
_tracer = provider.get_tracer("duocthu.ai-service")
return True
def _langfuse_exporters(settings: Any, exporter_cls: Any) -> list[Any]:
"""Zero or one extra OTLP exporter pointed at Langfuse.
Langfuse accepts standard OTLP/HTTP on `/api/public/otel/v1/traces`,
authenticated with HTTP Basic using the project's public key as the
username and the secret key as the password. Returning a list keeps the
caller unchanged when Langfuse is not configured -- which is the default,
and the case for every local run without a `.env`.
"""
base_url = str(getattr(settings, "langfuse_base_url", "") or "").rstrip("/")
public_key = str(getattr(settings, "langfuse_public_key", "") or "")
secret_key = str(getattr(settings, "langfuse_secret_key", "") or "")
if not (base_url and public_key and secret_key):
return []
token = base64.b64encode(f"{public_key}:{secret_key}".encode()).decode()
return [
exporter_cls(
endpoint=f"{base_url}/api/public/otel/v1/traces",
headers={"Authorization": f"Basic {token}"},
)
]
def normalize_correlation_id(candidate: str | None) -> str:
value = (candidate or "").strip()
return value if _SAFE_CORRELATION.fullmatch(value) else str(uuid.uuid4())
+81
View File
@@ -161,9 +161,24 @@ def _section_retriever(request: Request) -> SectionListRetriever:
router = APIRouter(prefix="/v1/rag", tags=["rag"])
def _clip(text: str | None, limit: int = 2000) -> str:
"""Bounded text for a span attribute.
Span attributes are not a place for unbounded content: every answer would
otherwise be sent twice per turn (Tempo and Langfuse) and stored forever.
2000 characters is past the end of a normal formulary answer, so this
almost never truncates -- it exists for the pathological case.
"""
value = (text or "").strip()
return value if len(value) <= limit else value[: limit - 1] + ""
@router.post("/feedback", response_model=RagFeedbackResponse)
def save_feedback(
payload: RagFeedbackRequest,
request: Request,
traces: Annotated[TraceWriter, Depends(_trace_writer)],
) -> RagFeedbackResponse:
comment = payload.comment.strip() if payload.comment else None
@@ -178,9 +193,53 @@ def save_feedback(
raise HTTPException(status_code=404, detail="trace_not_found") from exc
except Exception as exc:
raise HTTPException(status_code=503, detail="feedback_store_unavailable") from exc
_mirror_feedback_to_langfuse(request, traces, str(payload.trace_id), payload.rating, comment)
return RagFeedbackResponse(feedback_id=feedback_id)
def _mirror_feedback_to_langfuse(
request: Request,
traces: TraceWriter,
trace_id: str,
rating: str,
comment: str | None,
) -> None:
"""Attach the user's verdict to its Langfuse trace, or quietly do nothing.
This is the only quality signal in the system that comes from a human
instead of a model judging a model, so it is worth having next to the
Ragas scores rather than only in Postgres.
Runs AFTER the Postgres write and cannot affect its outcome: the durable
record is what matters, and a Langfuse outage must never turn a working
thumbs-up into a 503. Everything here is guarded -- a missing client, an
answer from before tracing was on, a database hiccup on the lookup, or a
refused post all end the same way: silently.
1.0/0.0 rather than a label so it averages: the mean of `user_feedback`
over a period reads directly as "share of answers users found helpful".
"""
client = getattr(request.app.state, "langfuse_scores", None)
if client is None:
return
lookup = getattr(traces, "otel_trace_id_for", None)
if lookup is None:
return
try:
otel_trace_id = lookup(trace_id)
except Exception: # noqa: BLE001 - feedback is already safely stored
return
if not otel_trace_id:
return
client.post_score(
otel_trace_id=otel_trace_id,
name="user_feedback",
value=1.0 if rating == "helpful" else 0.0,
comment=comment,
)
class HistoryItem(BaseModel):
trace_id: str
query: str
@@ -614,6 +673,28 @@ def query_rag(
"duocthu.citation_count": len(citations),
"duocthu.generated": generated,
"duocthu.persisted_trace_id": trace_id,
# Langfuse-mapped names (see its OpenTelemetry attribute mapping).
# Without these a trace shows stage timings but neither the
# question nor the answer, which is most of what makes a trace
# worth opening -- and scores attached to it unreadable, since
# "faithfulness 0.42" means nothing without the answer it judged.
#
# NOTE this is a deliberate change of posture: these spans also go
# to Tempo, and the README previously stated that full prompts and
# responses are not stored. Both destinations are self-hosted on
# this cluster and the corpus is a published formulary, so the
# exposure is bounded -- but it is a change, not an oversight, and
# the text is truncated rather than unbounded.
"langfuse.trace.name": f"rag.{decision}",
"langfuse.observation.input": _clip(payload.query),
"langfuse.observation.output": _clip(answer),
"langfuse.trace.metadata.decision": decision,
"langfuse.trace.metadata.reason": reason,
"langfuse.trace.metadata.resolved_drug_id": resolved_drug_id or "",
# Groups a multi-turn conversation into one Langfuse session, so a
# clarify turn and the answer that follows it read together
# instead of as two unrelated traces.
"langfuse.session.id": payload.conversation_id or "",
}
)
with stage("response"):
+35 -1
View File
@@ -25,6 +25,7 @@ import importlib.util
import json
import sys
import time
import urllib.request
from pathlib import Path
from typing import Any
@@ -44,6 +45,34 @@ def _load_battery_module():
return module
def _post_capturing_trace(
url: str, payload: dict[str, Any], timeout: float
) -> tuple[dict[str, Any], str | None]:
"""`run_manual_battery._post`, but also returning the OpenTelemetry trace id.
Langfuse attaches a score to a trace by its OTel trace id (32 hex chars),
NOT by the `traceId` in the response body -- that one is ai-service's own
Postgres row id, a UUID, and Langfuse has never heard of it. ai-service
sets the OTel id on the `X-Trace-ID` response header and the web BFF
forwards it, so the eval runner can record it per case and the Ragas
scorer can post scores that land on the right trace.
Duplicating the POST rather than widening `_post`'s return type: three
other call sites depend on that signature, and this script deliberately
reuses the battery's *checker* -- forking the definition of "correct" is
the thing worth avoiding, not four lines of urllib.
"""
request = urllib.request.Request(
url,
data=json.dumps(payload, ensure_ascii=False).encode("utf-8"),
headers={"Content-Type": "application/json; charset=utf-8"},
method="POST",
)
with urllib.request.urlopen(request, timeout=timeout) as response:
body = json.loads(response.read().decode("utf-8"))
return body, response.headers.get("X-Trace-ID")
def run_suite(
battery: Any,
name: str,
@@ -63,8 +92,9 @@ def run_suite(
conversation_id = f"{base}-{run_id}"
payload = {"content": case["query"], "conversationId": conversation_id}
started = time.monotonic()
otel_trace_id = None
try:
raw = battery._post(endpoint, payload, timeout)
raw, otel_trace_id = _post_capturing_trace(endpoint, payload, timeout)
response = battery._normalise_response(raw)
failures = battery._check(case, response)
error = None
@@ -78,6 +108,10 @@ def run_suite(
{
"suite": name,
"case": case,
# None when the request failed outright, or when the
# deployment has tracing off -- the scorer treats a
# missing id as "score locally, do not push".
"otel_trace_id": otel_trace_id,
"passed": ok,
"failures": failures,
"elapsed_seconds": elapsed,
+92 -1
View File
@@ -49,7 +49,11 @@ from __future__ import annotations
import argparse
import asyncio
import base64
import json
import os
import urllib.error
import urllib.request
import warnings
from pathlib import Path
from typing import Any
@@ -79,13 +83,86 @@ def _contexts_for(citations: list[dict[str, Any]]) -> list[str]:
return contexts
class LangfuseScores:
"""Posts one score per metric per case to Langfuse, or does nothing.
Scores attach to a trace by its OpenTelemetry id, which `run_all_evals.py`
records as `otel_trace_id` from the `X-Trace-ID` response header. A row
without one (request failed, or the deployment had tracing off) is skipped
rather than posted against a guessed id -- a score on the wrong trace is
worse than no score, because nothing later distinguishes it from a real one.
Failures here never abort scoring: the local .jsonl is the source of truth
and a Langfuse outage must not cost a whole Bedrock-funded run.
"""
def __init__(self, base_url: str, public_key: str, secret_key: str, run_name: str):
self.endpoint = f"{base_url.rstrip('/')}/api/public/scores"
token = base64.b64encode(f"{public_key}:{secret_key}".encode()).decode()
self.auth = f"Basic {token}"
self.run_name = run_name
self.posted = 0
self.failed = 0
self.no_trace = 0
@classmethod
def from_env(cls, run_name: str) -> "LangfuseScores | None":
base_url = os.environ.get("LANGFUSE_BASE_URL", "").strip()
public_key = os.environ.get("LANGFUSE_PUBLIC_KEY", "").strip()
secret_key = os.environ.get("LANGFUSE_SECRET_KEY", "").strip()
if not (base_url and public_key and secret_key):
return None
return cls(base_url, public_key, secret_key, run_name)
def post(self, trace_id: str | None, case_id: str, metric: str, value: float) -> None:
if not trace_id:
self.no_trace += 1
return
body = {
"traceId": trace_id,
"name": metric,
"value": value,
"dataType": "NUMERIC",
# The run name groups one eval run's scores so two runs of the same
# suite stay comparable instead of averaging into each other.
"comment": f"ragas/{self.run_name} case={case_id}",
}
request = urllib.request.Request(
self.endpoint,
data=json.dumps(body).encode("utf-8"),
headers={"Content-Type": "application/json", "Authorization": self.auth},
method="POST",
)
try:
with urllib.request.urlopen(request, timeout=20):
self.posted += 1
except Exception as exc: # noqa: BLE001 - reported in the summary
self.failed += 1
if self.failed <= 3: # a broken endpoint says it once, not 90 times
print(f" ! langfuse score post failed: {exc!r}", flush=True)
async def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--input", type=Path, required=True, help="run_all_evals.py output")
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--limit", type=int)
parser.add_argument(
"--run-name",
default="manual",
help="labels this run's scores in Langfuse so runs stay comparable",
)
args = parser.parse_args()
langfuse = LangfuseScores.from_env(args.run_name)
print(
f"langfuse: posting scores to {langfuse.endpoint}"
if langfuse
else "langfuse: not configured (set LANGFUSE_BASE_URL/_PUBLIC_KEY/_SECRET_KEY "
"to push scores); scoring locally only",
flush=True,
)
from langchain_aws import BedrockEmbeddings, ChatBedrockConverse
from ragas import SingleTurnSample
from ragas.embeddings import LangchainEmbeddingsWrapper
@@ -138,13 +215,21 @@ async def main() -> int:
response=answer,
retrieved_contexts=contexts,
)
result: dict[str, Any] = {"id": case_id, "query": case.get("query", "")}
otel_trace_id = row.get("otel_trace_id")
result: dict[str, Any] = {
"id": case_id,
"query": case.get("query", ""),
"otel_trace_id": otel_trace_id,
}
for name, metric in metrics.items():
try:
result[name] = float(await metric.single_turn_ascore(sample))
except Exception as exc: # noqa: BLE001 - recorded, not hidden
result[name] = None
result[f"{name}_error"] = repr(exc)[:200]
else:
if langfuse:
langfuse.post(otel_trace_id, case_id, name, result[name])
scored.append(result)
handle.write(json.dumps(result, ensure_ascii=False) + "\n")
handle.flush()
@@ -168,6 +253,12 @@ async def main() -> int:
)
else:
print(f" {name:<18} no successful scores")
if langfuse:
print(
f" langfuse posted={langfuse.posted} "
f"failed={langfuse.failed} no_trace_id={langfuse.no_trace}"
)
return 0
+101
View File
@@ -120,3 +120,104 @@ def test_otel_server_and_stage_spans_share_trace_and_emit_metric_exemplar():
assert "duocthu_stage_duration_seconds_bucket" in rendered
assert 'trace_id="' in rendered
import pytest
def test_langfuse_exporter_added_only_when_fully_configured():
"""Langfuse is opt-in: all three settings, or no extra exporter at all.
A half-configured Langfuse (say a base URL pasted in but no keys yet)
must not produce an exporter that would fail auth on every batch.
"""
from rag.telemetry import _langfuse_exporters
captured = []
def fake_exporter(**kwargs):
captured.append(kwargs)
return object()
# Explicit empty values, not Settings() defaults: Settings reads the
# developer's real .env, which on a configured machine already carries
# live Langfuse keys and would make this assertion pass or fail by
# accident depending on whose machine runs it.
unset = Settings(
langfuse_base_url="", langfuse_public_key="", langfuse_secret_key=""
)
assert _langfuse_exporters(unset, fake_exporter) == []
assert (
_langfuse_exporters(
Settings(
langfuse_base_url="https://langfuse.example",
langfuse_public_key="",
langfuse_secret_key="",
),
fake_exporter,
)
== []
)
assert captured == []
result = _langfuse_exporters(
Settings(
langfuse_base_url="https://langfuse.example/",
langfuse_public_key="pk-lf-public",
langfuse_secret_key="sk-lf-secret",
),
fake_exporter,
)
assert len(result) == 1
assert len(captured) == 1
# Trailing slash stripped, so the path is not doubled.
assert captured[0]["endpoint"] == "https://langfuse.example/api/public/otel/v1/traces"
# Basic auth is public-key-as-username, secret-key-as-password.
import base64
expected = base64.b64encode(b"pk-lf-public:sk-lf-secret").decode()
assert captured[0]["headers"] == {"Authorization": f"Basic {expected}"}
def test_langfuse_score_client_is_all_or_nothing():
"""Same partial-config rule as the span exporter, for the same reason."""
from adapters.langfuse_scores import LangfuseScoreClient
unset = Settings(
langfuse_base_url="", langfuse_public_key="", langfuse_secret_key=""
)
assert LangfuseScoreClient.from_settings(unset) is None
assert (
LangfuseScoreClient.from_settings(
Settings(
langfuse_base_url="https://langfuse.example",
langfuse_public_key="pk-lf-x",
langfuse_secret_key="",
)
)
is None
)
client = LangfuseScoreClient.from_settings(
Settings(
langfuse_base_url="https://langfuse.example/",
langfuse_public_key="pk-lf-public",
langfuse_secret_key="sk-lf-secret",
)
)
assert client is not None
# Trailing slash stripped, so the path is not doubled.
assert client.endpoint == "https://langfuse.example/api/public/scores"
def test_langfuse_score_post_never_raises(monkeypatch):
"""A Langfuse outage must not propagate into the feedback request path."""
from adapters import langfuse_scores
client = langfuse_scores.LangfuseScoreClient(
"https://langfuse.example", "pk", "sk"
)
def boom(*args, **kwargs):
raise OSError("connection refused")
monkeypatch.setattr(langfuse_scores.urllib.request, "urlopen", boom)
assert client.post_score(otel_trace_id="a" * 32, name="user_feedback", value=1.0) is False