Log the 2026-08-24 session: F3 fix live, audit filled, corpus re-ingest scoped

apps/ai-service/rag/understanding.py

@@ -203,6 +203,18 @@ class QueryFrame:
     # ordinary clarifying question in the API response and trace — this lets
     # `RagAgent._route` surface the real cause instead.
     system_error: str | None = None
+    # The part of the turn the Dược thư cannot answer AT ALL, as the model
+    # named it — a property the book does not record (giá, nơi bán, bảo hiểm)
+    # or a comparative judgement it never makes ("hãng nào tốt nhất"). Set
+    # means refuse, and `_parse` forces `turn_type` to "out_of_scope" on it.
+    #
+    # Why a field and not another phrase list: `attribute` is validated against
+    # SECTION_KEYS and anything unrecognised collapses to None, which made
+    # "user asked for a section but did not say which" and "user asked for
+    # something the book has no section for" indistinguishable — both became
+    # `attribute=None` and both clarified. Found live 2026-08-24: "Paracetamol
+    # giá bao nhiêu?" answered "Bạn muốn hỏi liều cho người lớn hay trẻ em?".
+    unsupported_request: str | None = None
     raw: dict = field(default_factory=dict, compare=False)
 
 
@@ -213,6 +225,19 @@ FRAME_SCHEMA = {
     "drugs": ["drug_id exactly as it appears in the provided catalog list"],
     "unknown_drugs": ["a drug name the user mentioned that is NOT in the catalog"],
     "attribute": "one of the section keys provided, or null",
+    "unsupported_request": (
+        "Null in the ordinary case. Set it ONLY when the turn asks for "
+        "something the Dược thư does not contain at all, and name that thing "
+        "briefly in Vietnamese. Two kinds qualify: (a) a commercial or "
+        "administrative property the book never records — giá/giá tiền, nơi "
+        "bán/mua ở đâu, bảo hiểm chi trả, hạn dùng của một hộp cụ thể; (b) a "
+        "comparative or evaluative judgement the book never makes — 'hãng nào "
+        "tốt nhất', 'thuốc nào hay hơn', 'nên chọn loại nào'. "
+        "IMPORTANT — do NOT set it for trade names: the monograph HAS a "
+        "'Tên thương mại' section, so 'Paracetamol của hãng nào', 'biệt dược "
+        "của X' are ordinary in-scope lookups (attribute=ten_thuong_mai). "
+        "Only RANKING trade names is unsupported, not listing them."
+    ),
     "population": "tre_em | tre_so_sinh | nguoi_lon | nguoi_cao_tuoi | phu_nu_co_thai | phu_nu_cho_con_bu | suy_than | suy_gan | null",
     "weight_kg": (
         "number if a body weight is given, else null. Vietnamese casual speech "
@@ -412,6 +437,10 @@ Quy tắc bắt buộc:
   thực sự muốn hỏi điều gì (vd "Anh/chị muốn hỏi đường dùng nào ạ?"), không tự
   suy đoán lại giá trị cũ.
 - Chào hỏi/vu vơ -> "smalltalk". Ngoài phạm vi chuyên luận thuốc -> "out_of_scope".
+- Nếu câu hỏi nhắm vào thứ Dược thư không ghi (giá tiền, nơi mua, bảo hiểm) hoặc
+  đòi xếp hạng hơn kém ("hãng nào tốt nhất", "thuốc nào hay hơn"), đặt
+  "unsupported_request" nêu ngắn gọn thứ đó. Tên biệt dược CÓ trong sách (mục
+  "Tên thương mại"), nên hỏi biệt dược là hợp lệ — không đặt unsupported_request.
 - Khi needs_clarify=true, kèm "quick_replies": 2-4 phương án NGẮN cho câu hỏi lại
   đó, CHỈ khi nó thực sự có vài lựa chọn rời rạc tự nhiên (vd đối tượng: "Người
   lớn"/"Trẻ em"). Để mảng rỗng nếu cần một giá trị cụ thể không có lựa chọn ngắn
@@ -658,6 +687,24 @@ class LlmQueryUnderstander:
         if turn_type not in TURN_TYPES:
             turn_type = "drug_attribute" if drugs else "out_of_scope"
         needs_clarify = data.get("needs_clarify") is True
+        # The scope gate, applied deterministically rather than trusted to the
+        # model's own `turn_type`. When the turn asks for something the book
+        # does not contain, refusing is the only correct outcome — Feature-List
+        # F3 requires 100% of out-of-scope turns to be refused, and the failure
+        # this fixes was precisely the model saying `drug_attribute` while
+        # leaving `attribute` null, which downstream read as "which section did
+        # you mean?" and asked the user a question the book cannot answer.
+        #
+        # Deliberately unconditional: it fires even when a valid `attribute`
+        # was also parsed. A turn mixing an answerable section with an
+        # unanswerable property ("giá bao nhiêu và liều người lớn?") is refused
+        # whole rather than half-answered. Over-refusing is the safe direction
+        # for a safety threshold; the 90-case suite is the guard against
+        # over-refusing in practice.
+        unsupported_request = _clean_str(data.get("unsupported_request"))
+        if unsupported_request:
+            turn_type = "out_of_scope"
+            needs_clarify = False
         clarify_reason = _clean_str(data.get("clarify_reason"))
         quick_replies = (
             _clean_quick_replies(data.get("quick_replies"))
@@ -700,6 +747,7 @@ class LlmQueryUnderstander:
             needs_clarify=needs_clarify,
             clarify_reason=clarify_reason,
             quick_replies=quick_replies,
+            unsupported_request=unsupported_request,
             raw=data if isinstance(data, dict) else {},
         )
 
@@ -750,6 +798,17 @@ def _apply_reverse_relation_cues(frame: QueryFrame, turn: str) -> QueryFrame:
     )
 
 
+# Shared with `_apply_condition_candidate_cue` below: a turn naming these
+# signals is describing one patient's own combined profile ("BN X kèm Y"),
+# not asking the model to pick between unrelated conditions.
+_PATIENT_CONTEXT_CUES = (
+    " bn ", " benh nhan ", " nguoi benh ", " kem ", " di ung ",
+    " dang dung ", " mang thai ", " cho con bu ", " tuoi ", " kg ",
+    " ckd ", " suy than ", " suy gan ", " child pugh ", " egfr ",
+    " creatinin ", " ast ", " alt ",
+)
+
+
 def _apply_condition_candidate_cue(
     frame: QueryFrame, turn: str, normalizer: ConditionNormalizer
 ) -> QueryFrame:
@@ -774,6 +833,23 @@ def _apply_condition_candidate_cue(
     )
     if not any(cue in text for cue in candidate_cues):
         return frame
+    if condition.ambiguous and any(cue in text for cue in _PATIENT_CONTEXT_CUES):
+        # Found live 2026-08-20 (eval case P08): "BN tăng huyết áp kèm xơ gan
+        # Child-Pugh B dùng thuốc nào cần lưu ý?" reliably clarified instead
+        # of answering, 4/4 reproductions. The raw understanding call reads
+        # a comorbidity ("kèm xơ gan...") as a FORK in what the question
+        # means ("thuốc nào cần lưu ý" vs "thuốc nào gây tăng huyết áp") and
+        # marks the condition ambiguous with its own clarify_question — but
+        # this turn already told us which drug lane it wants (a candidate
+        # cue matched, e.g. "thuốc nào cần"), so the fork the model raised
+        # is not genuine: `_apply_general_condition_scope` already treats
+        # this same cue set as "this is one patient's profile, not a choice
+        # between diseases", and `frame.patient_context` (hepatic/renal/etc,
+        # parsed separately and left untouched here) is exactly what lets
+        # `_condition_to_drug`'s `assess_patient_candidates` answer safely
+        # instead — clearing the stale ambiguity is what lets a turn reach
+        # that path instead of dead-ending in a clarify loop.
+        condition = replace(condition, ambiguous=False, clarify_question=None)
     return replace(
         frame,
         turn_type="condition_to_drug",
@@ -799,13 +875,7 @@ def _apply_general_condition_scope(frame: QueryFrame, turn: str) -> QueryFrame:
     if frame.turn_type not in {"condition_to_drug", "symptom_to_drug"}:
         return frame
     text = f" {normalize_name(turn)} "
-    patient_cues = (
-        " bn ", " benh nhan ", " nguoi benh ", " kem ", " di ung ",
-        " dang dung ", " mang thai ", " cho con bu ", " tuoi ", " kg ",
-        " ckd ", " suy than ", " suy gan ", " child pugh ", " egfr ",
-        " creatinin ", " ast ", " alt ",
-    )
-    if any(cue in text for cue in patient_cues):
+    if any(cue in text for cue in _PATIENT_CONTEXT_CUES):
         return frame
     primary = (
         frame.condition.normalized_condition

apps/ai-service/routers/rag.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import json
 import uuid
 from typing import Annotated, Any, Literal, Protocol
 
@@ -161,6 +162,18 @@ def _section_retriever(request: Request) -> SectionListRetriever:
 router = APIRouter(prefix="/v1/rag", tags=["rag"])
 
 
+def _json_attr(text: str) -> str:
+    """Langfuse expects `observation.input`/`output` as a JSON string.
+
+    Passing raw text silently maps to nothing: verified live 2026-08-21, the
+    span carried the attribute and Langfuse showed empty input/output columns
+    while `langfuse.session.id` -- a plain string by contract -- mapped fine
+    from the same annotate call. No error anywhere; the field simply stays
+    blank, which is the failure mode worth remembering.
+    """
+    return json.dumps(text, ensure_ascii=False)
+
+
 def _clip(text: str | None, limit: int = 2000) -> str:
     """Bounded text for a span attribute.
 
@@ -686,8 +699,8 @@ def query_rag(
             # exposure is bounded -- but it is a change, not an oversight, and
             # the text is truncated rather than unbounded.
             "langfuse.trace.name": f"rag.{decision}",
-            "langfuse.observation.input": _clip(payload.query),
-            "langfuse.observation.output": _clip(answer),
+            "langfuse.observation.input": _json_attr(_clip(payload.query)),
+            "langfuse.observation.output": _json_attr(_clip(answer)),
             "langfuse.trace.metadata.decision": decision,
             "langfuse.trace.metadata.reason": reason,
             "langfuse.trace.metadata.resolved_drug_id": resolved_drug_id or "",

apps/ai-service/scripts/paraphrase_probe.py

@@ -0,0 +1,139 @@
+"""Ask the same clinical question several ways; the answer must not change.
+
+The 90-case suite pins one exact wording per case, and `rag/understanding.py`
+routes on hardcoded Vietnamese phrase lists (`candidate_cues`, `patient_cues`).
+Together those make it possible to pass eval while a user who phrases the same
+question differently gets a different outcome -- the system would be memorising
+the test, not understanding the request. Nothing in the existing harness can
+detect that, because every case is a single phrasing.
+
+This probe closes that hole. Each group below is ONE clinical intent written
+several ways by hand (not model-generated: a model asked to paraphrase tends to
+preserve the distinctive words that drive the routing, which is exactly what
+must vary). A group is CONSISTENT when every phrasing lands on the same
+decision. Which decision is right is a separate question -- this measures
+stability, not correctness, and instability is a defect regardless of which
+answer is the good one.
+
+Usage:
+    python scripts/paraphrase_probe.py --base-url https://realvuxbaro.me
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import time
+import urllib.request
+from collections import Counter
+
+# Each group: (id, intent, [phrasings]).
+GROUPS = [
+    (
+        "comorbidity",
+        "Patient with a comorbidity asks which drugs need caution (the P08 shape)",
+        [
+            "BN tăng huyết áp kèm xơ gan Child-Pugh B dùng thuốc nào cần lưu ý?",
+            "Bệnh nhân bị cao huyết áp và xơ gan thì cần thận trọng với thuốc nào?",
+            "Người bệnh xơ gan Child-Pugh B, huyết áp cao, nên lưu ý những thuốc gì?",
+            "Có xơ gan mà bị tăng huyết áp thì thuốc nào phải cẩn thận?",
+        ],
+    ),
+    (
+        "contraindication",
+        "Straight contraindication lookup for one named drug",
+        [
+            "Chống chỉ định của Ibuprofen là gì?",
+            "Ibuprofen chống chỉ định với ai?",
+            "Những trường hợp nào không được dùng Ibuprofen?",
+            "Ai không nên uống Ibuprofen?",
+        ],
+    ),
+    (
+        "pediatric_dose",
+        "Paediatric dose, which the service must clarify on (age/weight required)",
+        [
+            "Liều Paracetamol cho trẻ em là bao nhiêu?",
+            "Trẻ con uống Paracetamol liều thế nào?",
+            "Cho bé dùng Paracetamol bao nhiêu mg?",
+            "Paracetamol dùng cho trẻ nhỏ liều ra sao?",
+        ],
+    ),
+    (
+        "out_of_scope",
+        "Out of scope -- must refuse every time, this is the safety threshold",
+        [
+            "Thuốc Paracetamol giá bao nhiêu tiền?",
+            "Mua Paracetamol ở đâu rẻ nhất?",
+            "Paracetamol hãng nào tốt nhất hiện nay?",
+            "Giá một hộp Paracetamol là bao nhiêu?",
+        ],
+    ),
+]
+
+
+def ask(endpoint: str, question: str, conversation_id: str, timeout: float):
+    payload = {"content": question, "conversationId": conversation_id}
+    request = urllib.request.Request(
+        endpoint,
+        data=json.dumps(payload, ensure_ascii=False).encode("utf-8"),
+        headers={"Content-Type": "application/json; charset=utf-8"},
+        method="POST",
+    )
+    with urllib.request.urlopen(request, timeout=timeout) as response:
+        body = json.loads(response.read().decode("utf-8"))
+        message = body.get("message") or {}
+        return {
+            "decision": message.get("decision"),
+            "reason": message.get("reason"),
+            "drug": message.get("resolvedDrugId"),
+            "citations": len(message.get("citations") or []),
+            "otel_trace_id": response.headers.get("X-Trace-ID"),
+            "answer": (message.get("content") or "")[:160],
+        }
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--base-url", required=True)
+    parser.add_argument("--timeout", type=float, default=120.0)
+    parser.add_argument("--out", default="")
+    args = parser.parse_args()
+
+    endpoint = args.base_url.rstrip("/") + "/api/chat"
+    run_id = str(int(time.time()))
+    results, unstable = [], 0
+
+    for group_id, intent, phrasings in GROUPS:
+        print(f"\n=== {group_id} — {intent}")
+        decisions = []
+        for index, question in enumerate(phrasings, start=1):
+            # A fresh conversation per phrasing: shared history would let an
+            # earlier turn answer a later one and hide the instability.
+            conversation_id = f"para-{group_id}-{index}-{run_id}"
+            try:
+                row = ask(endpoint, question, conversation_id, args.timeout)
+            except Exception as exc:  # noqa: BLE001 - recorded, not swallowed
+                row = {"decision": "ERROR", "reason": repr(exc)[:80], "citations": 0}
+            row.update({"group": group_id, "phrasing": question})
+            results.append(row)
+            decisions.append(row["decision"])
+            print(
+                f"  [{index}] {row['decision']:<11} cit={row['citations']} "
+                f"drug={row.get('drug')} :: {question[:52]}"
+            )
+        counts = Counter(decisions)
+        stable = len(counts) == 1
+        unstable += 0 if stable else 1
+        print(f"  -> {'CONSISTENT' if stable else 'INCONSISTENT'} {dict(counts)}")
+
+    print(f"\n=== {len(GROUPS) - unstable}/{len(GROUPS)} intents answered consistently ===")
+    if args.out:
+        with open(args.out, "w", encoding="utf-8") as handle:
+            for row in results:
+                handle.write(json.dumps(row, ensure_ascii=False) + "\n")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

apps/ai-service/tests/test_understanding.py

@@ -163,6 +163,96 @@ def test_single_section_named_is_unaffected_by_the_multi_section_clarify():
     assert frame.needs_clarify is False
 
 
+def test_patient_comorbidity_question_answers_instead_of_forking_on_ambiguity():
+    """Regression for eval case P08, reproduced live 4/4 times 2026-08-20:
+    "BN tăng huyết áp kèm xơ gan Child-Pugh B dùng thuốc nào cần lưu ý?"
+    reliably clarified instead of running the patient-safety assessment path
+    `patient_ckd`/`patient_multi`/etc. already use successfully. The payload
+    below is the real raw understanding output captured for that turn — the
+    model read "kèm xơ gan..." as a fork in what the QUESTION means (caution
+    vs causation) and marked the condition ambiguous with its own
+    clarify_question, discarding the richly-parsed `patient_context` it
+    produced in the very same call."""
+    understander = LlmQueryUnderstander(_FixedLlm({
+        "turn_type": "condition_relation",
+        "drugs": [], "unknown_drugs": [], "attribute": None,
+        "population": None, "weight_kg": None, "age_text": None,
+        "indication": None,
+        "condition": {
+            "original_text": "tăng huyết áp kèm xơ gan Child-Pugh B",
+            "normalized_condition": "tăng huyết áp",
+            "subtype": "Child-Pugh B",
+            "qualifiers": ["xơ gan Child-Pugh B"],
+            "ambiguous": True,
+            "clarify_question": (
+                "Anh/chị muốn biết thuốc nào cần lưu ý khi tăng huyết áp kèm "
+                "xơ gan Child-Pugh B, hay thuốc nào gây tăng huyết áp ở bệnh "
+                "nhân này?"
+            ),
+        },
+        "condition_relation": "adverse_effect",
+        "patient_context": {
+            "comorbidities": ["xơ gan Child-Pugh B"],
+            "hepatic": {"description": "xơ gan Child-Pugh B", "child_pugh": "Child-Pugh B"},
+        },
+        "context_action": "new",
+        "needs_clarify": True,
+        "clarify_reason": (
+            "Anh/chị muốn hỏi thuốc nào GÂY tăng huyết áp ở bệnh nhân xơ gan "
+            "Child-Pugh B, hay thuốc nào DÙNG ĐỂ điều trị tăng huyết áp "
+            "nhưng cần lưu ý ở bệnh nhân này?"
+        ),
+        "quick_replies": ["Thuốc nào gây tăng huyết áp?", "Thuốc nào dùng cần lưu ý?"],
+    }), CATALOG, RESOLVER)
+
+    frame = understander.understand(
+        "BN tăng huyết áp kèm xơ gan Child-Pugh B dùng thuốc nào cần lưu ý?"
+    )
+
+    assert frame.turn_type == "condition_to_drug"
+    assert frame.needs_clarify is False
+    assert frame.condition is not None
+    assert frame.condition.ambiguous is False
+    assert frame.condition.clarify_question is None
+    # The hepatic data must survive untouched -- this is what lets
+    # `_condition_to_drug`'s `assess_patient_candidates` answer safely
+    # instead of listing indications blind to the Child-Pugh B impairment.
+    assert frame.patient_context.hepatic.child_pugh == "Child-Pugh B"
+
+
+def test_bare_broad_disease_still_clarifies_despite_the_candidate_cue():
+    """The P08 fix must not swallow a genuine "which disease" fork: a bare
+    broad condition with NO patient-comorbidity cue (no "BN", "kèm", organ
+    impairment, ...) still has to ask for a subtype, same as before."""
+    understander = LlmQueryUnderstander(_FixedLlm({
+        "turn_type": "condition_relation",
+        "drugs": [], "unknown_drugs": [], "attribute": None,
+        "population": None, "weight_kg": None, "age_text": None,
+        "indication": None,
+        "condition": {
+            "original_text": "viêm gan",
+            "normalized_condition": "viêm gan",
+            "ambiguous": True,
+            "clarify_question": "Anh/chị muốn hỏi viêm gan B hay viêm gan C?",
+        },
+        "condition_relation": "indication",
+        "needs_clarify": True,
+        "clarify_reason": "Anh/chị muốn hỏi viêm gan B hay viêm gan C?",
+    }), CATALOG, RESOLVER)
+
+    frame = understander.understand("Viêm gan dùng thuốc gì?")
+
+    # `_apply_broad_condition_cue` (unaffected by this fix, and applied
+    # AFTER it) is what actually decides this case: it re-derives `condition`
+    # straight from the raw turn text via its own deterministic broad-disease
+    # regex, independent of whatever `_apply_condition_candidate_cue` did —
+    # `frame.needs_clarify` itself is reset to False either way, same as the
+    # `ambiguous_hepatitis`/`ambiguous_cancer`/`ambiguous_infection` eval
+    # cases; `agent.py` clarifies off `frame.condition.ambiguous`, not this.
+    assert frame.condition is not None
+    assert frame.condition.ambiguous is True
+
+
 def test_multi_section_clarify_does_not_inherit_a_stale_prior_attribute():
     prior = QueryFrame(
         turn_type="drug_attribute",