Add Langfuse as a self-hosted eval and trace viewer
This commit is contained in:
@@ -0,0 +1,175 @@
|
||||
"""Score recorded eval responses with Ragas, using Bedrock as the judge.
|
||||
|
||||
`run_all_evals.py` answers "did the service break a rule" -- decision, citation
|
||||
presence, drug provenance. It cannot answer "was the answer any good", so a
|
||||
change to retrieval or prompting can degrade quality while every invariant
|
||||
still passes. This fills that gap.
|
||||
|
||||
Three metrics, chosen because the datasets carry no reference answers and any
|
||||
metric needing one (context recall, answer correctness) would be unmeasurable:
|
||||
|
||||
faithfulness -- is every claim in the answer supported by the cited
|
||||
evidence? The hallucination check.
|
||||
context_precision -- were the retrieved chunks actually relevant, or did
|
||||
useful evidence arrive buried in noise? A retrieval
|
||||
check, which faithfulness alone cannot see: an answer
|
||||
can be perfectly faithful to one good chunk that
|
||||
arrived alongside nine useless ones.
|
||||
answer_relevancy -- does the answer address the question asked? Catches a
|
||||
grounded, well-cited answer to a different question.
|
||||
|
||||
CONTEXT MUST CARRY THE DRUG NAME. Each citation's `evidence_text` is the raw
|
||||
section prose, which frequently never repeats the drug it belongs to ("Tăng
|
||||
huyết áp (dùng đơn trị liệu...)"). The service knows the drug from a separate
|
||||
field; a judge handed the bare text does not. Scoring a multi-drug answer that
|
||||
way on 2026-08-18 produced faithfulness 0.251 -- every claim marked
|
||||
unsupported because no context could be attributed to any drug -- and the same
|
||||
run scored 1.000 once `[drug_name]` was prefixed. That was a defect in the
|
||||
measurement, not the service, and it is exactly the kind of error that gets
|
||||
reported as a model regression. Hence `_contexts_for`.
|
||||
|
||||
Only `answerable` turns are scored: an abstain or a clarify has no claims to
|
||||
be faithful to, and scoring them would drag the mean around with values that
|
||||
mean nothing.
|
||||
|
||||
Runs against recorded output, so it never re-queries production and can be
|
||||
re-run offline as often as needed.
|
||||
|
||||
Requires ragas + langchain-aws, which conflict with this service's own pinned
|
||||
dependencies -- install them in a separate virtualenv and run this with that
|
||||
interpreter. See evals/README-ragas.md.
|
||||
|
||||
Usage:
|
||||
<ragas-venv>/python scripts/score_evals_ragas.py \\
|
||||
--input /tmp/evals/production60.jsonl \\
|
||||
--output /tmp/evals/production60.ragas.jsonl
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import warnings
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
warnings.filterwarnings("ignore", category=DeprecationWarning)
|
||||
|
||||
ANSWER_MODEL = "qwen.qwen3-next-80b-a3b"
|
||||
# NOT cohere-v4, which production uses for retrieval: langchain-aws cannot
|
||||
# parse v4's response envelope and raises a bare KeyError(0). v3-multilingual
|
||||
# is the right substitute anyway -- this embedder only measures how close the
|
||||
# answer sits to the question, never touching the indexed corpus, so it does
|
||||
# not need to match the retrieval model. Multilingual matters more here, the
|
||||
# corpus and questions both being Vietnamese.
|
||||
EMBED_MODEL = "cohere.embed-multilingual-v3"
|
||||
REGION = "us-east-1"
|
||||
|
||||
|
||||
def _contexts_for(citations: list[dict[str, Any]]) -> list[str]:
|
||||
"""Label every context with its drug -- see the module docstring."""
|
||||
contexts = []
|
||||
for citation in citations:
|
||||
text = (citation.get("evidence_text") or "").strip()
|
||||
if not text:
|
||||
continue
|
||||
name = citation.get("drug_name") or citation.get("drug_id") or ""
|
||||
contexts.append(f"[{name}] {text}" if name else text)
|
||||
return contexts
|
||||
|
||||
|
||||
async def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--input", type=Path, required=True, help="run_all_evals.py output")
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
parser.add_argument("--limit", type=int)
|
||||
args = parser.parse_args()
|
||||
|
||||
from langchain_aws import BedrockEmbeddings, ChatBedrockConverse
|
||||
from ragas import SingleTurnSample
|
||||
from ragas.embeddings import LangchainEmbeddingsWrapper
|
||||
from ragas.llms import LangchainLLMWrapper
|
||||
from ragas.metrics import (
|
||||
Faithfulness,
|
||||
LLMContextPrecisionWithoutReference,
|
||||
ResponseRelevancy,
|
||||
)
|
||||
|
||||
judge = LangchainLLMWrapper(
|
||||
ChatBedrockConverse(model=ANSWER_MODEL, region_name=REGION, temperature=0)
|
||||
)
|
||||
embedder = LangchainEmbeddingsWrapper(
|
||||
BedrockEmbeddings(model_id=EMBED_MODEL, region_name=REGION)
|
||||
)
|
||||
metrics = {
|
||||
"faithfulness": Faithfulness(llm=judge),
|
||||
"context_precision": LLMContextPrecisionWithoutReference(llm=judge),
|
||||
"answer_relevancy": ResponseRelevancy(llm=judge, embeddings=embedder),
|
||||
}
|
||||
|
||||
rows = [
|
||||
json.loads(line)
|
||||
for line in args.input.read_text(encoding="utf-8").splitlines()
|
||||
if line.strip()
|
||||
]
|
||||
if args.limit:
|
||||
rows = rows[: args.limit]
|
||||
|
||||
scored: list[dict[str, Any]] = []
|
||||
skipped = 0
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
with args.output.open("w", encoding="utf-8") as handle:
|
||||
for index, row in enumerate(rows, start=1):
|
||||
case, response = row.get("case", {}), row.get("response") or {}
|
||||
case_id = case.get("id", f"row{index}")
|
||||
decision = response.get("decision")
|
||||
answer = (response.get("answer") or "").strip()
|
||||
contexts = _contexts_for(response.get("citations") or [])
|
||||
|
||||
if decision != "answerable" or not answer or not contexts:
|
||||
skipped += 1
|
||||
print(f"[{index:02d}/{len(rows)}] {case_id} SKIP ({decision})", flush=True)
|
||||
continue
|
||||
|
||||
sample = SingleTurnSample(
|
||||
user_input=case.get("query", ""),
|
||||
response=answer,
|
||||
retrieved_contexts=contexts,
|
||||
)
|
||||
result: dict[str, Any] = {"id": case_id, "query": case.get("query", "")}
|
||||
for name, metric in metrics.items():
|
||||
try:
|
||||
result[name] = float(await metric.single_turn_ascore(sample))
|
||||
except Exception as exc: # noqa: BLE001 - recorded, not hidden
|
||||
result[name] = None
|
||||
result[f"{name}_error"] = repr(exc)[:200]
|
||||
scored.append(result)
|
||||
handle.write(json.dumps(result, ensure_ascii=False) + "\n")
|
||||
handle.flush()
|
||||
print(
|
||||
f"[{index:02d}/{len(rows)}] {case_id} "
|
||||
+ " ".join(
|
||||
f"{n}={result[n]:.2f}" if result[n] is not None else f"{n}=ERR"
|
||||
for n in metrics
|
||||
),
|
||||
flush=True,
|
||||
)
|
||||
|
||||
print(f"\n=== {args.input.name}: scored {len(scored)}, skipped {skipped} ===")
|
||||
for name in metrics:
|
||||
values = [r[name] for r in scored if r.get(name) is not None]
|
||||
if values:
|
||||
worst = min(values)
|
||||
print(
|
||||
f" {name:<18} mean={sum(values) / len(values):.3f} "
|
||||
f"min={worst:.3f} n={len(values)}"
|
||||
)
|
||||
else:
|
||||
print(f" {name:<18} no successful scores")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(asyncio.run(main()))
|
||||
Reference in New Issue
Block a user