Make a Langfuse trace worth opening: question, answer, session, no probe noise

apps/ai-service/scripts/score_evals_ragas.py

@@ -49,7 +49,11 @@ from __future__ import annotations
 
 import argparse
 import asyncio
+import base64
 import json
+import os
+import urllib.error
+import urllib.request
 import warnings
 from pathlib import Path
 from typing import Any
@@ -79,13 +83,86 @@ def _contexts_for(citations: list[dict[str, Any]]) -> list[str]:
     return contexts
 
 
+class LangfuseScores:
+    """Posts one score per metric per case to Langfuse, or does nothing.
+
+    Scores attach to a trace by its OpenTelemetry id, which `run_all_evals.py`
+    records as `otel_trace_id` from the `X-Trace-ID` response header. A row
+    without one (request failed, or the deployment had tracing off) is skipped
+    rather than posted against a guessed id -- a score on the wrong trace is
+    worse than no score, because nothing later distinguishes it from a real one.
+
+    Failures here never abort scoring: the local .jsonl is the source of truth
+    and a Langfuse outage must not cost a whole Bedrock-funded run.
+    """
+
+    def __init__(self, base_url: str, public_key: str, secret_key: str, run_name: str):
+        self.endpoint = f"{base_url.rstrip('/')}/api/public/scores"
+        token = base64.b64encode(f"{public_key}:{secret_key}".encode()).decode()
+        self.auth = f"Basic {token}"
+        self.run_name = run_name
+        self.posted = 0
+        self.failed = 0
+        self.no_trace = 0
+
+    @classmethod
+    def from_env(cls, run_name: str) -> "LangfuseScores | None":
+        base_url = os.environ.get("LANGFUSE_BASE_URL", "").strip()
+        public_key = os.environ.get("LANGFUSE_PUBLIC_KEY", "").strip()
+        secret_key = os.environ.get("LANGFUSE_SECRET_KEY", "").strip()
+        if not (base_url and public_key and secret_key):
+            return None
+        return cls(base_url, public_key, secret_key, run_name)
+
+    def post(self, trace_id: str | None, case_id: str, metric: str, value: float) -> None:
+        if not trace_id:
+            self.no_trace += 1
+            return
+        body = {
+            "traceId": trace_id,
+            "name": metric,
+            "value": value,
+            "dataType": "NUMERIC",
+            # The run name groups one eval run's scores so two runs of the same
+            # suite stay comparable instead of averaging into each other.
+            "comment": f"ragas/{self.run_name} case={case_id}",
+        }
+        request = urllib.request.Request(
+            self.endpoint,
+            data=json.dumps(body).encode("utf-8"),
+            headers={"Content-Type": "application/json", "Authorization": self.auth},
+            method="POST",
+        )
+        try:
+            with urllib.request.urlopen(request, timeout=20):
+                self.posted += 1
+        except Exception as exc:  # noqa: BLE001 - reported in the summary
+            self.failed += 1
+            if self.failed <= 3:  # a broken endpoint says it once, not 90 times
+                print(f"  ! langfuse score post failed: {exc!r}", flush=True)
+
+
 async def main() -> int:
     parser = argparse.ArgumentParser()
     parser.add_argument("--input", type=Path, required=True, help="run_all_evals.py output")
     parser.add_argument("--output", type=Path, required=True)
     parser.add_argument("--limit", type=int)
+    parser.add_argument(
+        "--run-name",
+        default="manual",
+        help="labels this run's scores in Langfuse so runs stay comparable",
+    )
     args = parser.parse_args()
 
+    langfuse = LangfuseScores.from_env(args.run_name)
+    print(
+        f"langfuse: posting scores to {langfuse.endpoint}"
+        if langfuse
+        else "langfuse: not configured (set LANGFUSE_BASE_URL/_PUBLIC_KEY/_SECRET_KEY "
+        "to push scores); scoring locally only",
+        flush=True,
+    )
+
     from langchain_aws import BedrockEmbeddings, ChatBedrockConverse
     from ragas import SingleTurnSample
     from ragas.embeddings import LangchainEmbeddingsWrapper
@@ -138,13 +215,21 @@ async def main() -> int:
                 response=answer,
                 retrieved_contexts=contexts,
             )
-            result: dict[str, Any] = {"id": case_id, "query": case.get("query", "")}
+            otel_trace_id = row.get("otel_trace_id")
+            result: dict[str, Any] = {
+                "id": case_id,
+                "query": case.get("query", ""),
+                "otel_trace_id": otel_trace_id,
+            }
             for name, metric in metrics.items():
                 try:
                     result[name] = float(await metric.single_turn_ascore(sample))
                 except Exception as exc:  # noqa: BLE001 - recorded, not hidden
                     result[name] = None
                     result[f"{name}_error"] = repr(exc)[:200]
+                else:
+                    if langfuse:
+                        langfuse.post(otel_trace_id, case_id, name, result[name])
             scored.append(result)
             handle.write(json.dumps(result, ensure_ascii=False) + "\n")
             handle.flush()
@@ -168,6 +253,12 @@ async def main() -> int:
             )
         else:
             print(f"  {name:<18} no successful scores")
+
+    if langfuse:
+        print(
+            f"  langfuse           posted={langfuse.posted} "
+            f"failed={langfuse.failed} no_trace_id={langfuse.no_trace}"
+        )
     return 0