Add production condition retrieval smoke test

apps/ai-service/rag/condition_evaluation.py

@@ -0,0 +1,111 @@
+"""Deterministic metrics for the condition-to-drug vertical slice.
+
+This intentionally does not use one overall LLM judge.  Callers populate an
+outcome from structured frames, retrieval metadata, candidate claims and
+citations; every metric below is then an auditable exact comparison.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class ConditionEvaluationOutcome:
+    case_id: str
+    expected_intent: str
+    actual_intent: str
+    expected_condition: str | None
+    actual_condition: str | None
+    expected_clarification: bool
+    actual_clarification: bool
+    expected_relation: str
+    actual_relation: str
+    expected_drug_ids: tuple[str, ...] = ()
+    retrieved_drug_ids: tuple[str, ...] = ()
+    generated_drug_ids: tuple[str, ...] = ()
+    retrieved_section_keys: tuple[str, ...] = ()
+    citation_validity: tuple[bool, ...] = ()
+    grounded_claims: tuple[bool, ...] = ()
+    expected_patient_fields: tuple[tuple[str, str], ...] = ()
+    actual_patient_fields: tuple[tuple[str, str], ...] = ()
+    expected_safety_facets: tuple[str, ...] = ()
+    retrieved_safety_facets: tuple[str, ...] = ()
+
+
+def summarize_condition_outcomes(
+    outcomes: list[ConditionEvaluationOutcome], *, retrieval_k: int = 8
+) -> dict[str, float | int | None]:
+    if not outcomes:
+        return {"cases": 0}
+
+    retrieval_rows = [row for row in outcomes if row.expected_drug_ids]
+    generated_count = sum(len(row.generated_drug_ids) for row in outcomes)
+    unsupported_count = sum(
+        sum(drug not in set(row.retrieved_drug_ids) for drug in row.generated_drug_ids)
+        for row in outcomes
+    )
+    citation_values = [value for row in outcomes for value in row.citation_validity]
+    grounded_values = [value for row in outcomes for value in row.grounded_claims]
+    patient_rows = [row for row in outcomes if row.expected_patient_fields]
+    safety_rows = [row for row in outcomes if row.expected_safety_facets]
+
+    return {
+        "cases": len(outcomes),
+        "intent_accuracy": _mean(
+            row.actual_intent == row.expected_intent for row in outcomes
+        ),
+        "condition_normalization_accuracy": _mean(
+            row.actual_condition == row.expected_condition
+            for row in outcomes
+            if row.expected_condition is not None
+        ),
+        "ambiguity_clarification_accuracy": _mean(
+            row.actual_clarification == row.expected_clarification
+            for row in outcomes
+        ),
+        f"indication_recall_at_{retrieval_k}": _mean(
+            bool(set(row.expected_drug_ids) & set(row.retrieved_drug_ids[:retrieval_k]))
+            for row in retrieval_rows
+        ),
+        f"drug_precision_at_{retrieval_k}": _mean(
+            len(set(row.expected_drug_ids) & set(row.retrieved_drug_ids[:retrieval_k]))
+            / max(1, len(row.retrieved_drug_ids[:retrieval_k]))
+            for row in retrieval_rows
+        ),
+        "section_correctness": _mean(
+            all(section == "chi_dinh" for section in row.retrieved_section_keys)
+            for row in retrieval_rows
+        ),
+        "relation_correctness": _mean(
+            row.actual_relation == row.expected_relation for row in outcomes
+        ),
+        "unsupported_drug_rate": (
+            unsupported_count / generated_count if generated_count else 0.0
+        ),
+        "citation_correctness": _mean(citation_values),
+        "groundedness": _mean(grounded_values),
+        "patient_context_extraction_accuracy": _mean(
+            _field_accuracy(row.expected_patient_fields, row.actual_patient_fields)
+            for row in patient_rows
+        ),
+        "safety_evidence_retrieval_accuracy": _mean(
+            len(set(row.expected_safety_facets) & set(row.retrieved_safety_facets))
+            / len(set(row.expected_safety_facets))
+            for row in safety_rows
+        ),
+    }
+
+
+def _field_accuracy(
+    expected: tuple[tuple[str, str], ...], actual: tuple[tuple[str, str], ...]
+) -> float:
+    expected_map = dict(expected)
+    actual_map = dict(actual)
+    return sum(actual_map.get(key) == value for key, value in expected_map.items()) / max(
+        1, len(expected_map)
+    )
+
+
+def _mean(values) -> float | None:
+    rows = list(values)
+    return round(sum(rows) / len(rows), 4) if rows else None