Checkpoint frontend UI/UX overhaul and ingestion embed benchmark work

2026-08-06 17:21:21 +07:00
parent 1e8cbdb586
commit a4b8e1c4db
78 changed files with 6761 additions and 654 deletions
@@ -20,6 +20,35 @@ PDF = ROOT / "ingestion/data/raw/duoc-thu-quoc-gia-viet-nam-2018.pdf"
 MIGRATION = Path(__file__).resolve().parents[1] / "migrations/001_rag_retrieval_trace.sql"


+class _PlumbingEmbedder:
+    """Deterministic local vectors for the Qdrant round-trip plumbing tests.
+
+    Not a semantic model — it exists only so an integration test can upsert and
+    query real chunks without a cloud call. Production has exactly one query
+    embedder (`BedrockCohereQueryEmbedder`); the old local/section-only stubs
+    were removed, so this lives with the test that needs it.
+    """
+
+    def __init__(self, dimensions: int) -> None:
+        self._dimensions = dimensions
+
+    @property
+    def dimensions(self) -> int:
+        return self._dimensions
+
+    def embed_query(self, text: str) -> list[float]:
+        import hashlib
+        import math
+
+        vector = [0.0] * self._dimensions
+        for token in text.casefold().split():
+            digest = hashlib.sha256(token.encode("utf-8")).digest()
+            index = int.from_bytes(digest[:4], "big") % self._dimensions
+            vector[index] += 1.0 if digest[4] & 1 else -1.0
+        norm = math.sqrt(sum(value * value for value in vector))
+        return [value / norm for value in vector] if norm else vector
+
+
@lru_cache
 def _first_real_chunk() -> dict:
    with CHUNKS.open(encoding="utf-8") as handle:
@@ -42,12 +71,11 @@ def test_real_qdrant_round_trip_uses_real_chunk_and_printed_folio():
    from qdrant_client import QdrantClient
    from qdrant_client.models import Distance, PointStruct, VectorParams

-    from adapters.embedding import LocalHashQueryEmbedder
    from adapters.qdrant import QdrantRetriever

    client = QdrantClient(url="http://localhost:6333")
    collection = f"integration_{uuid.uuid4().hex}"
-    embedder = LocalHashQueryEmbedder(32)
+    embedder = _PlumbingEmbedder(32)
    record = _first_real_chunk()
    try:
        client.create_collection(
@@ -103,12 +131,144 @@ def test_real_postgres_migration_insert_and_read_back():
    assert stored.citations[0]["printed_page_start"] == 101


+class _FakeJsonLlm:
+    """Deterministic stand-in for the Bedrock Converse generator. Satisfies
+    both `JsonLlm` (query understanding) and `AnswerGenerator` (answer +
+    entailment) — both ports are just `generate(system, user, schema) ->
+    str` — and tells the three call shapes apart the same way
+    `tests/test_grounded_generation.py`'s fake does: by schema shape.
+
+    Deliberately not a real Bedrock call: this suite asserts exact
+    drug id / section / citation / grounding outcomes, and this session's
+    own live probing (`docs/progress-log.md`, F-01/F-03 entries) found real
+    generation and entailment calls genuinely non-deterministic — the wrong
+    foundation for a regression assertion. The wiring under test — real
+    `RagAgent`, real `RetrievalService`/`QdrantRetriever` against a real
+    (temporary) Qdrant collection, real `GroundedAnswerService` — is
+    identical to production; only the cloud model call is faked.
+    """
+
+    def __init__(self, frame_payload: dict, answer_payload: dict) -> None:
+        self._frame_payload = frame_payload
+        self._answer_payload = answer_payload
+
+    def generate(self, system: str, user: str, schema: dict) -> str:
+        if "turn_type" in schema:  # FRAME_SCHEMA: flat, not JSON-Schema-shaped
+            return json.dumps(self._frame_payload, ensure_ascii=False)
+        if "entailed" in schema.get("properties", {}):
+            return json.dumps({"entailed": True, "unsupported": []})
+        return json.dumps(self._answer_payload, ensure_ascii=False)
+
+
+def test_real_rag_agent_end_to_end_through_the_http_api():
+    """F-10: the *production* orchestrator (`RagAgent`), driven through the
+    real `/v1/rag/query` HTTP endpoint against a real (temporary) Qdrant
+    collection seeded with one real corpus chunk, with a real Postgres trace
+    persisted and read back. Reproduces Codex's exact 2026-08-06 finding —
+    "no current test imports RagAgent, LlmQueryUnderstander, QueryFrame, or
+    retrieve_framed" and "the evaluation runner constructs an in-memory
+    lexical retriever and the old resolver rather than executing the same
+    dependency graph as the live HTTP service" — both false as of this test.
+    """
+    from fastapi.testclient import TestClient
+    from qdrant_client import QdrantClient
+    from qdrant_client.models import Distance, PointStruct, VectorParams
+
+    from adapters.postgres import PostgresTraceRepository
+    from adapters.qdrant import QdrantParentStore, QdrantRetriever
+    from config import Settings
+    from main import create_app
+    from rag.agent import RagAgent
+    from rag.answer import GroundedAnswerService
+    from rag.routing import CatalogDrugResolver, QueryRoutingService
+    from rag.service import EvidencePolicy, RetrievalService
+    from rag.understanding import LlmQueryUnderstander
+
+    qdrant = QdrantClient(url="http://localhost:6333")
+    collection = f"integration_{uuid.uuid4().hex}"
+    embedder = _PlumbingEmbedder(32)
+    record = dict(_first_real_chunk())
+    traces = PostgresTraceRepository(
+        "postgresql://duoc_thu:duoc_thu@localhost:5432/duoc_thu"
+    )
+    traces.migrate(MIGRATION)
+    try:
+        qdrant.create_collection(
+            collection_name=collection,
+            vectors_config=VectorParams(size=32, distance=Distance.COSINE),
+        )
+        qdrant.upsert(
+            collection_name=collection,
+            points=[PointStruct(
+                id=str(uuid.uuid4()),
+                vector=embedder.embed_query(record["text"]),
+                payload=record,
+            )],
+            wait=True,
+        )
+        retrieval = RetrievalService(
+            QdrantRetriever(qdrant, collection, embedder),
+            QdrantParentStore(qdrant, collection),
+            EvidencePolicy(minimum_score=0.01),
+        )
+        # A minimal one-drug catalog, real drug_id — F-04's candidate
+        # bounding runs for real here (`RagAgent`/`LlmQueryUnderstander`
+        # are not mocked), so the query must literally name the drug for
+        # the deterministic resolver to find it as a candidate.
+        resolver = CatalogDrugResolver({record["drug_id"]: {record["drug_name"]}})
+        llm = _FakeJsonLlm(
+            frame_payload={
+                "turn_type": "drug_attribute", "drugs": [record["drug_id"]],
+                "unknown_drugs": [], "attribute": None, "population": None,
+                "weight_kg": None, "age_text": None, "indication": None,
+                "needs_clarify": False, "clarify_reason": None,
+            },
+            answer_payload={
+                "answer": f"{record['text']} [1].", "evidence_sufficient": True,
+            },
+        )
+        understander = LlmQueryUnderstander(
+            llm, {record["drug_id"]: record["drug_name"]}, resolver,
+        )
+        answers = GroundedAnswerService(
+            QueryRoutingService(retrieval, resolver), generator=llm,
+        )
+        agent = RagAgent(understander, retrieval, answers)
+
+        app = create_app(
+            settings=Settings(), answer_service=answers,
+            conversational=agent, trace_writer=traces,
+        )
+        response = TestClient(app).post("/v1/rag/query", json={
+            "query": record["drug_name"],
+            "subject_scope": "human", "intent": "fact_lookup",
+        })
+
+        assert response.status_code == 200
+        body = response.json()
+        assert body["decision"] == "answerable"
+        assert body["resolved_drug_id"] == record["drug_id"]
+        assert body["citations"][0]["chunk_id"] == record["chunk_id"]
+        assert body["citations"][0]["printed_page_start"] == (
+            record["printed_page_range"][0]
+        )
+        assert record["text"] in body["answer"]
+
+        stored = traces.get(body["trace_id"])
+        assert stored is not None
+        assert stored.decision == "answerable"
+        assert stored.resolved_drug_id == record["drug_id"]
+        assert stored.citations[0]["chunk_id"] == record["chunk_id"]
+    finally:
+        with suppress(Exception):
+            qdrant.delete_collection(collection)
+
+
 def test_api_round_trip_uses_qdrant_and_persists_postgres_trace():
    from fastapi.testclient import TestClient
    from qdrant_client import QdrantClient
    from qdrant_client.models import Distance, PointStruct, VectorParams

-    from adapters.embedding import LocalHashQueryEmbedder
    from adapters.postgres import PostgresTraceRepository
    from adapters.qdrant import QdrantParentStore, QdrantRetriever
    from config import Settings
@@ -119,7 +279,7 @@ def test_api_round_trip_uses_qdrant_and_persists_postgres_trace():

    qdrant = QdrantClient(url="http://localhost:6333")
    collection = f"integration_{uuid.uuid4().hex}"
-    embedder = LocalHashQueryEmbedder(32)
+    embedder = _PlumbingEmbedder(32)
    record = dict(_first_real_chunk())
    traces = PostgresTraceRepository(
        "postgresql://duoc_thu:duoc_thu@localhost:5432/duoc_thu"