Make a Langfuse trace worth opening: question, answer, session, no probe noise

apps/ai-service/scripts/run_all_evals.py

@@ -25,6 +25,7 @@ import importlib.util
 import json
 import sys
 import time
+import urllib.request
 from pathlib import Path
 from typing import Any
 
@@ -44,6 +45,34 @@ def _load_battery_module():
     return module
 
 
+def _post_capturing_trace(
+    url: str, payload: dict[str, Any], timeout: float
+) -> tuple[dict[str, Any], str | None]:
+    """`run_manual_battery._post`, but also returning the OpenTelemetry trace id.
+
+    Langfuse attaches a score to a trace by its OTel trace id (32 hex chars),
+    NOT by the `traceId` in the response body -- that one is ai-service's own
+    Postgres row id, a UUID, and Langfuse has never heard of it. ai-service
+    sets the OTel id on the `X-Trace-ID` response header and the web BFF
+    forwards it, so the eval runner can record it per case and the Ragas
+    scorer can post scores that land on the right trace.
+
+    Duplicating the POST rather than widening `_post`'s return type: three
+    other call sites depend on that signature, and this script deliberately
+    reuses the battery's *checker* -- forking the definition of "correct" is
+    the thing worth avoiding, not four lines of urllib.
+    """
+    request = urllib.request.Request(
+        url,
+        data=json.dumps(payload, ensure_ascii=False).encode("utf-8"),
+        headers={"Content-Type": "application/json; charset=utf-8"},
+        method="POST",
+    )
+    with urllib.request.urlopen(request, timeout=timeout) as response:
+        body = json.loads(response.read().decode("utf-8"))
+        return body, response.headers.get("X-Trace-ID")
+
+
 def run_suite(
     battery: Any,
     name: str,
@@ -63,8 +92,9 @@ def run_suite(
             conversation_id = f"{base}-{run_id}"
             payload = {"content": case["query"], "conversationId": conversation_id}
             started = time.monotonic()
+            otel_trace_id = None
             try:
-                raw = battery._post(endpoint, payload, timeout)
+                raw, otel_trace_id = _post_capturing_trace(endpoint, payload, timeout)
                 response = battery._normalise_response(raw)
                 failures = battery._check(case, response)
                 error = None
@@ -78,6 +108,10 @@ def run_suite(
                     {
                         "suite": name,
                         "case": case,
+                        # None when the request failed outright, or when the
+                        # deployment has tracing off -- the scorer treats a
+                        # missing id as "score locally, do not push".
+                        "otel_trace_id": otel_trace_id,
                         "passed": ok,
                         "failures": failures,
                         "elapsed_seconds": elapsed,