Make a Langfuse trace worth opening: question, answer, session, no probe noise

2026-08-21 14:45:32 +07:00
parent 53582b6030
commit f3eaab0948
16 changed files with 733 additions and 5 deletions
@@ -25,6 +25,7 @@ import importlib.util
 import json
 import sys
 import time
+import urllib.request
 from pathlib import Path
 from typing import Any

@@ -44,6 +45,34 @@ def _load_battery_module():
    return module


+def _post_capturing_trace(
+    url: str, payload: dict[str, Any], timeout: float
+) -> tuple[dict[str, Any], str | None]:
+    """`run_manual_battery._post`, but also returning the OpenTelemetry trace id.
+
+    Langfuse attaches a score to a trace by its OTel trace id (32 hex chars),
+    NOT by the `traceId` in the response body -- that one is ai-service's own
+    Postgres row id, a UUID, and Langfuse has never heard of it. ai-service
+    sets the OTel id on the `X-Trace-ID` response header and the web BFF
+    forwards it, so the eval runner can record it per case and the Ragas
+    scorer can post scores that land on the right trace.
+
+    Duplicating the POST rather than widening `_post`'s return type: three
+    other call sites depend on that signature, and this script deliberately
+    reuses the battery's *checker* -- forking the definition of "correct" is
+    the thing worth avoiding, not four lines of urllib.
+    """
+    request = urllib.request.Request(
+        url,
+        data=json.dumps(payload, ensure_ascii=False).encode("utf-8"),
+        headers={"Content-Type": "application/json; charset=utf-8"},
+        method="POST",
+    )
+    with urllib.request.urlopen(request, timeout=timeout) as response:
+        body = json.loads(response.read().decode("utf-8"))
+        return body, response.headers.get("X-Trace-ID")
+
+
 def run_suite(
    battery: Any,
    name: str,
@@ -63,8 +92,9 @@ def run_suite(
            conversation_id = f"{base}-{run_id}"
            payload = {"content": case["query"], "conversationId": conversation_id}
            started = time.monotonic()
+            otel_trace_id = None
            try:
-                raw = battery._post(endpoint, payload, timeout)
+                raw, otel_trace_id = _post_capturing_trace(endpoint, payload, timeout)
                response = battery._normalise_response(raw)
                failures = battery._check(case, response)
                error = None
@@ -78,6 +108,10 @@ def run_suite(
                    {
                        "suite": name,
                        "case": case,
+                        # None when the request failed outright, or when the
+                        # deployment has tracing off -- the scorer treats a
+                        # missing id as "score locally, do not push".
+                        "otel_trace_id": otel_trace_id,
                        "passed": ok,
                        "failures": failures,
                        "elapsed_seconds": elapsed,
@@ -49,7 +49,11 @@ from __future__ import annotations

 import argparse
 import asyncio
+import base64
 import json
+import os
+import urllib.error
+import urllib.request
 import warnings
 from pathlib import Path
 from typing import Any
@@ -79,13 +83,86 @@ def _contexts_for(citations: list[dict[str, Any]]) -> list[str]:
    return contexts


+class LangfuseScores:
+    """Posts one score per metric per case to Langfuse, or does nothing.
+
+    Scores attach to a trace by its OpenTelemetry id, which `run_all_evals.py`
+    records as `otel_trace_id` from the `X-Trace-ID` response header. A row
+    without one (request failed, or the deployment had tracing off) is skipped
+    rather than posted against a guessed id -- a score on the wrong trace is
+    worse than no score, because nothing later distinguishes it from a real one.
+
+    Failures here never abort scoring: the local .jsonl is the source of truth
+    and a Langfuse outage must not cost a whole Bedrock-funded run.
+    """
+
+    def __init__(self, base_url: str, public_key: str, secret_key: str, run_name: str):
+        self.endpoint = f"{base_url.rstrip('/')}/api/public/scores"
+        token = base64.b64encode(f"{public_key}:{secret_key}".encode()).decode()
+        self.auth = f"Basic {token}"
+        self.run_name = run_name
+        self.posted = 0
+        self.failed = 0
+        self.no_trace = 0
+
+    @classmethod
+    def from_env(cls, run_name: str) -> "LangfuseScores | None":
+        base_url = os.environ.get("LANGFUSE_BASE_URL", "").strip()
+        public_key = os.environ.get("LANGFUSE_PUBLIC_KEY", "").strip()
+        secret_key = os.environ.get("LANGFUSE_SECRET_KEY", "").strip()
+        if not (base_url and public_key and secret_key):
+            return None
+        return cls(base_url, public_key, secret_key, run_name)
+
+    def post(self, trace_id: str | None, case_id: str, metric: str, value: float) -> None:
+        if not trace_id:
+            self.no_trace += 1
+            return
+        body = {
+            "traceId": trace_id,
+            "name": metric,
+            "value": value,
+            "dataType": "NUMERIC",
+            # The run name groups one eval run's scores so two runs of the same
+            # suite stay comparable instead of averaging into each other.
+            "comment": f"ragas/{self.run_name} case={case_id}",
+        }
+        request = urllib.request.Request(
+            self.endpoint,
+            data=json.dumps(body).encode("utf-8"),
+            headers={"Content-Type": "application/json", "Authorization": self.auth},
+            method="POST",
+        )
+        try:
+            with urllib.request.urlopen(request, timeout=20):
+                self.posted += 1
+        except Exception as exc:  # noqa: BLE001 - reported in the summary
+            self.failed += 1
+            if self.failed <= 3:  # a broken endpoint says it once, not 90 times
+                print(f"  ! langfuse score post failed: {exc!r}", flush=True)
+
+
 async def main() -> int:
    parser = argparse.ArgumentParser()
    parser.add_argument("--input", type=Path, required=True, help="run_all_evals.py output")
    parser.add_argument("--output", type=Path, required=True)
    parser.add_argument("--limit", type=int)
+    parser.add_argument(
+        "--run-name",
+        default="manual",
+        help="labels this run's scores in Langfuse so runs stay comparable",
+    )
    args = parser.parse_args()

+    langfuse = LangfuseScores.from_env(args.run_name)
+    print(
+        f"langfuse: posting scores to {langfuse.endpoint}"
+        if langfuse
+        else "langfuse: not configured (set LANGFUSE_BASE_URL/_PUBLIC_KEY/_SECRET_KEY "
+        "to push scores); scoring locally only",
+        flush=True,
+    )
+
    from langchain_aws import BedrockEmbeddings, ChatBedrockConverse
    from ragas import SingleTurnSample
    from ragas.embeddings import LangchainEmbeddingsWrapper
@@ -138,13 +215,21 @@ async def main() -> int:
                response=answer,
                retrieved_contexts=contexts,
            )
-            result: dict[str, Any] = {"id": case_id, "query": case.get("query", "")}
+            otel_trace_id = row.get("otel_trace_id")
+            result: dict[str, Any] = {
+                "id": case_id,
+                "query": case.get("query", ""),
+                "otel_trace_id": otel_trace_id,
+            }
            for name, metric in metrics.items():
                try:
                    result[name] = float(await metric.single_turn_ascore(sample))
                except Exception as exc:  # noqa: BLE001 - recorded, not hidden
                    result[name] = None
                    result[f"{name}_error"] = repr(exc)[:200]
+                else:
+                    if langfuse:
+                        langfuse.post(otel_trace_id, case_id, name, result[name])
            scored.append(result)
            handle.write(json.dumps(result, ensure_ascii=False) + "\n")
            handle.flush()
@@ -168,6 +253,12 @@ async def main() -> int:
            )
        else:
            print(f"  {name:<18} no successful scores")
+
+    if langfuse:
+        print(
+            f"  langfuse           posted={langfuse.posted} "
+            f"failed={langfuse.failed} no_trace_id={langfuse.no_trace}"
+        )
    return 0