Fix ai-service Dockerfile: bake in drug_entities.json, override its path

apps/ai-service/tests/test_grounded_generation.py

@@ -67,23 +67,28 @@ class _FixedRouting:
 class _Generator:
     """Returns whatever payload the test wants the model to have produced.
 
-    `_generate` now makes up to four calls through this port: the main
-    answer, a sufficiency check (skipped here — one evidence block), and up
-    to two entailment calls (a reject retries once — live probing found the
-    judge noisy on an identical claim/evidence pair). They're told apart by
-    schema, so a test that only cares about the main answer doesn't also
-    have to fake an entailment response by hand; `entailment_payload`
-    overrides it when a test wants the entailment pass to reject. Pass a
-    list of payloads to get a different answer on each successive
-    entailment call (e.g. `[reject, accept]` for the retry-recovers case).
+    `_generate` now makes up to five calls through this port: the main
+    answer (a lone `evidence_sufficient: false` retries once — the same
+    noisy-judge finding as entailment, live-confirmed 2026-08-07), a
+    sufficiency check (skipped here — one evidence block), and up to three
+    entailment calls (widened from two 2026-08-07: live probing found the
+    judge noisy on an identical claim/evidence pair, and a real adversarial
+    sample showed a single retry still discarding correct answers on the
+    unlucky reject-reject draw). They're told apart by schema, so a test
+    that only cares about one call doesn't have to fake the others; `payload`
+    and `entailment_payload` each take either a fixed value or a list for a
+    different answer on each successive call to that schema (e.g.
+    `[reject, reject, accept]` for the third-attempt-recovers case).
     """
 
     def __init__(self, payload, entailment_payload=None) -> None:
-        self._payload = payload
+        payloads = payload
+        self._payloads = list(payloads) if isinstance(payloads, list) else [payloads]
+        self._call = 0
         default = {"entailed": True, "unsupported": []}
-        payloads = entailment_payload if entailment_payload is not None else default
+        e_payloads = entailment_payload if entailment_payload is not None else default
         self._entailment_payloads = (
-            list(payloads) if isinstance(payloads, list) else [payloads]
+            list(e_payloads) if isinstance(e_payloads, list) else [e_payloads]
         )
         self._entailment_call = 0
 
@@ -93,7 +98,9 @@ class _Generator:
             payload = self._entailment_payloads[index]
             self._entailment_call += 1
         else:
-            payload = self._payload
+            index = min(self._call, len(self._payloads) - 1)
+            payload = self._payloads[index]
+            self._call += 1
         if isinstance(payload, BaseException):
             raise payload
         if isinstance(payload, str):
@@ -127,7 +134,12 @@ def test_invented_dose_is_refused_and_never_reaches_the_answer():
     # this is a real LLM chatbot, not the retired offline-extractive build).
     assert grounded.answer is None
     assert grounded.result.decision == EvidenceDecision.ABSTAIN
-    assert grounded.result.reason == "generation_unavailable"
+    # The specific check that rejected it, not a generic catch-all — found
+    # live 2026-08-07: every rejection reason used to collapse into
+    # "generation_unavailable" by the time it reached the API response,
+    # making a real provider outage indistinguishable from ordinary
+    # entailment noise without reading server metrics by hand.
+    assert grounded.result.reason == "ungrounded_number"
     assert metrics.total(GENERATION_REJECTED, reason="ungrounded_number") == 1
 
 
@@ -217,17 +229,42 @@ def test_entailment_retries_once_after_a_reject_and_a_later_accept_serves():
     assert metrics.total(GENERATION_SERVED) == 1
 
 
-def test_entailment_two_agreeing_rejects_still_discard():
+def test_entailment_recovers_on_third_attempt_after_two_rejects():
+    """The improvement 2026-08-07 widened the retry from 2 to 3 attempts
+    after a live 50-question adversarial sample found the 2-attempt policy's
+    own math (~11% false-discard rate on a genuinely valid claim, from the
+    noise probed in the docstring above) matched the observed real
+    abstention rate almost exactly. Two rejects followed by a real accept
+    must now be served, not discarded."""
+    grounded, metrics = _answer(
+        {"answer": "Metformin dùng điều trị đái tháo đường [1].",
+         "evidence_sufficient": True},
+        entailment_payload=[
+            {"entailed": False, "unsupported": [1]},
+            {"entailed": False, "unsupported": [1]},
+            {"entailed": True, "unsupported": []},
+        ],
+    )
+
+    assert grounded.generated is True
+    assert metrics.total(GENERATION_SERVED) == 1
+    assert metrics.total(GENERATION_REJECTED) == 0
+
+
+def test_entailment_three_agreeing_rejects_still_discard():
     grounded, metrics = _answer(
         {"answer": "Metformin chữa ung thư [1].", "evidence_sufficient": True},
         entailment_payload=[
             {"entailed": False, "unsupported": [1]},
             {"entailed": False, "unsupported": [1]},
+            {"entailed": False, "unsupported": [1]},
         ],
     )
 
     assert grounded.generated is False
     assert grounded.answer is None
+    # All 3 attempts are noisy-judge calls against the SAME claim/evidence —
+    # a real, reliable rejection must still discard exactly once, not 3x.
     assert metrics.total(GENERATION_REJECTED, reason="unsupported_claim") == 1
 
 
@@ -287,10 +324,46 @@ def test_every_generation_failure_abstains_instead_of_a_raw_source_dump(payload,
     assert grounded.generated is False
     assert grounded.answer is None
     assert grounded.result.decision == EvidenceDecision.ABSTAIN
-    assert grounded.result.reason == "generation_unavailable"
+    # The API/trace-visible reason must match the specific check that
+    # failed, not a generic "generation_unavailable" for every cause —
+    # otherwise a real outage and ordinary model noise are indistinguishable
+    # from the outside (the exact gap a live report 2026-08-07 named).
+    assert grounded.result.reason == reason
     assert metrics.total(GENERATION_REJECTED, reason=reason) == 1
 
 
+def test_evidence_insufficient_retries_once_and_recovers():
+    """Found live 2026-08-07 via a 50-question adversarial sample: a real
+    section that plainly contains the answer (confirmed by re-asking the
+    identical question 3/3 times successfully right after) still drew an
+    `evidence_sufficient: false` self-judgment once — the same noisy-judge
+    pattern already known for entailment, just on a different field of the
+    same call. A lone insufficient verdict must not be final."""
+    grounded, metrics = _answer([
+        {"answer": "...", "evidence_sufficient": False},
+        {"answer": "Metformin dùng điều trị đái tháo đường [1].",
+         "evidence_sufficient": True},
+    ])
+
+    assert grounded.generated is True
+    assert grounded.answer == "Metformin dùng điều trị đái tháo đường [1]."
+    assert metrics.total(GENERATION_SERVED) == 1
+    assert metrics.total(GENERATION_REJECTED) == 0
+
+
+def test_evidence_insufficient_twice_still_abstains():
+    grounded, metrics = _answer([
+        {"answer": "...", "evidence_sufficient": False},
+        {"answer": "...", "evidence_sufficient": False},
+    ])
+
+    assert grounded.generated is False
+    assert grounded.answer is None
+    # Both attempts are the same noisy self-judgment on the same evidence —
+    # a real, reliable "insufficient" must still discard exactly once.
+    assert metrics.total(GENERATION_REJECTED, reason="evidence_insufficient") == 1
+
+
 def test_no_generator_configured_still_answers():
     service = GroundedAnswerService(_FixedRouting(_result()))