Wire the guarded conversational RAG answer layer end-to-end

2026-08-05 14:33:13 +07:00
parent 834d9e51b0
commit ef08b4929e
127 changed files with 37921 additions and 169 deletions
@@ -0,0 +1,377 @@
+"""`QdrantVectorStore` against a real Qdrant, skipped when none is running.
+
+The rest of the load suite runs against `InMemoryVectorStore` and proves the
+loader's rules. It cannot prove the adapter: whether Qdrant accepts a uuid5
+string as a point id, whether `create_payload_index` takes a bare `"keyword"`,
+whether an upsert of an existing id replaces rather than appends. Those are
+claims about another system, and the same class of claim as the Bedrock request
+bodies that are still documentation-derived and unproven — so they get a live
+check, against a free local container rather than a paid API.
+
+Start one with `docker compose -f infra/docker/docker-compose.yml up -d qdrant`.
+Without it these tests skip; they never fail for being offline.
+
+Every test works in its own collection named after the test and deletes it
+afterwards, so a shared local Qdrant is not left holding fixtures.
+"""
+import os
+import uuid
+from pathlib import Path
+
+import pytest
+
+from ingestion.load import (
+    ChunkLoader,
+    CollectionSpec,
+    CorpusManifest,
+    CorpusMismatch,
+    manifest_collection,
+    point_id_for,
+    read_manifest,
+)
+from ingestion.load.qdrant_repo import DEFAULT_URL, SCROLL_PAGE, QdrantVectorStore
+
+DIMENSIONS = 4
+QDRANT_URL = os.environ.get("QDRANT_URL", DEFAULT_URL)
+REAL_CHUNKS = (
+    Path(__file__).resolve().parents[1] / "data" / "processed" / "chunks.jsonl"
+)
+
+pytestmark = pytest.mark.integration
+
+
+def _server_is_up() -> bool:
+    try:
+        from qdrant_client import QdrantClient
+    except ImportError:
+        return False
+    try:
+        QdrantClient(url=QDRANT_URL, timeout=3.0).get_collections()
+    except Exception:
+        return False
+    return True
+
+
+requires_qdrant = pytest.mark.skipif(
+    not _server_is_up(), reason=f"no Qdrant reachable at {QDRANT_URL}"
+)
+
+
+@pytest.fixture()
+def store():
+    return QdrantVectorStore(url=QDRANT_URL, timeout=10.0)
+
+
+@pytest.fixture()
+def collection(store):
+    name = f"test_load_{uuid.uuid4().hex[:10]}"
+    yield name
+    for target in (manifest_collection(name), name):
+        if store.collection_exists(target):
+            store.delete_collection(target)
+
+
+def chunk_record(chunk_id, **overrides):
+    record = {
+        "schema_version": 4,
+        "chunk_id": chunk_id,
+        "drug_id": "abacavir",
+        "drug_name": "ABACAVIR",
+        "section_key": "lieu_luong_va_cach_dung",
+        "section_display_name": "Liều lượng và cách dùng",
+        "text": "Người lớn: 300 mg, hai lần mỗi ngày.",
+        "source_text": "Người lớn: 300 mg, hai lần mỗi ngày.",
+        "heading_physical_page": 100,
+        "source_page_range": [100, 102],
+        "printed_page_range": [101, 103],
+        "atc_codes": ["J05AF06"],
+        "part_index": 0,
+        "part_count": 1,
+        "est_tokens": 14,
+        "oversized": False,
+        "chunk_kind": "prose",
+        "attachments": [],
+        "has_quarantined_content": False,
+    }
+    record.update(overrides)
+    return record
+
+
+def manifest(**overrides):
+    values = {
+        "corpus_sha256": "a" * 64,
+        "chunk_count": 2,
+        "model_id": "amazon.titan-embed-text-v2:0",
+        "dimensions": DIMENSIONS,
+        "input_kind": "document",
+        "provider": "titan-v2",
+    }
+    values.update(overrides)
+    return CorpusManifest(**values)
+
+
+def pairs(count):
+    return [
+        (chunk_record(f"abacavir__section__{i}"), [0.1 * (i + 1)] * DIMENSIONS)
+        for i in range(count)
+    ]
+
+
+@requires_qdrant
+def test_a_real_load_creates_the_collection_indexes_and_points(store, collection):
+    spec = CollectionSpec(name=collection, vector_size=DIMENSIONS)
+
+    report = ChunkLoader(store, spec, manifest()).load(pairs(2))
+
+    assert report.collection_created is True
+    assert report.collection_count == 2
+    assert store.collection_exists(collection)
+    assert store.count(collection) == 2
+
+
+@requires_qdrant
+def test_qdrant_accepts_the_derived_uuid5_point_id_and_returns_the_payload(
+    store, collection
+):
+    spec = CollectionSpec(name=collection, vector_size=DIMENSIONS)
+    record = chunk_record("abacavir__lieu_luong__0")
+    ChunkLoader(store, spec, manifest()).load([(record, [0.5] * DIMENSIONS)])
+
+    stored = store.retrieve(collection, point_id_for(record["chunk_id"]))
+
+    assert stored is not None
+    assert stored.id == point_id_for(record["chunk_id"])
+    assert stored.payload["chunk_id"] == record["chunk_id"]
+    assert stored.payload["source_page_range"] == [100, 102]
+    assert stored.payload["atc_codes"] == ["J05AF06"]
+    assert stored.payload["has_quarantined_content"] is False
+    assert len(stored.vector) == DIMENSIONS
+
+
+@requires_qdrant
+def test_loading_twice_against_a_real_server_does_not_duplicate(store, collection):
+    spec = CollectionSpec(name=collection, vector_size=DIMENSIONS)
+    data = pairs(2)
+
+    ChunkLoader(store, spec, manifest()).load(data)
+    second = ChunkLoader(store, spec, manifest()).load(data)
+
+    assert second.collection_created is False
+    assert store.count(collection) == 2, "a re-run duplicated points in Qdrant"
+
+
+@requires_qdrant
+def test_the_manifest_sidecar_round_trips_and_leaves_the_count_exact(
+    store, collection
+):
+    spec = CollectionSpec(name=collection, vector_size=DIMENSIONS)
+    ChunkLoader(store, spec, manifest()).load(pairs(2))
+
+    stored = read_manifest(store, collection)
+
+    assert stored is not None
+    assert stored.corpus_sha256 == "a" * 64
+    assert stored.model_id == "amazon.titan-embed-text-v2:0"
+    assert stored.dimensions == DIMENSIONS
+    assert store.count(collection) == 2
+    assert store.count(manifest_collection(collection)) == 1
+
+
+@requires_qdrant
+def test_a_second_corpus_is_refused_against_a_real_collection(store, collection):
+    spec = CollectionSpec(name=collection, vector_size=DIMENSIONS)
+    ChunkLoader(store, spec, manifest()).load(pairs(2))
+
+    with pytest.raises(CorpusMismatch, match="does not match"):
+        ChunkLoader(store, spec, manifest(corpus_sha256="b" * 64)).load(pairs(2))
+
+    assert store.count(collection) == 2
+
+
+@requires_qdrant
+def test_retrieve_returns_none_for_an_id_that_was_never_loaded(store, collection):
+    spec = CollectionSpec(name=collection, vector_size=DIMENSIONS)
+    ChunkLoader(store, spec, manifest()).load(pairs(1))
+
+    assert store.retrieve(collection, point_id_for("never__loaded__0")) is None
+
+
+@requires_qdrant
+def test_bbox_floats_lose_precision_in_qdrant_but_nothing_else_does(store, collection):
+    """Pins a measured round-trip loss so it cannot silently get worse.
+
+    Scrolling all 15,066 points of a full load on 2026-08-04 found 86 chunks
+    whose payload did not compare equal to its source record. Every one of the
+    96 differing leaf values was a float inside `attachments[].bbox`, the
+    largest delta was 5.684e-14, and **no** text, id, page number, page range,
+    token count or boolean differed at all. A PDF point is 1/72 inch, so that
+    delta cannot move a rendered crop; what would matter is the loss spreading
+    to another field, or growing. This test fails if either happens.
+    """
+    spec = CollectionSpec(name=collection, vector_size=DIMENSIONS)
+    # 17 significant digits: the real corpus carries these, and they are what
+    # does not survive a float64 -> JSON -> float64 round trip.
+    bbox = [44.45098876953125, 397.45245361328125, 278.09100341796875, 463.4044494628906]
+    record = chunk_record(
+        "cefazolin__lieu_luong_va_cach_dung__0",
+        has_quarantined_content=True,
+        attachments=[
+            {
+                "block_id": "p344_t2",
+                "kind": "table",
+                "shape": "simple_table",
+                "physical_page": 344,
+                "bbox": bbox,
+                "quarantined": True,
+                "header_row": ["Cỡ lọ", "Lượng\ndung môi"],
+            }
+        ],
+    )
+    ChunkLoader(store, spec, manifest()).load([(record, [0.3] * DIMENSIONS)])
+
+    stored = store.retrieve(collection, point_id_for(record["chunk_id"])).payload
+    attachment = stored["attachments"][0]
+
+    for value, original in zip(attachment["bbox"], bbox, strict=True):
+        assert abs(value - original) < 1e-9, "bbox drift grew beyond rounding"
+
+    assert attachment["block_id"] == "p344_t2"
+    assert attachment["physical_page"] == 344
+    assert attachment["quarantined"] is True
+    assert attachment["header_row"] == ["Cỡ lọ", "Lượng\ndung môi"]
+    for field in (
+        "chunk_id",
+        "drug_id",
+        "drug_name",
+        "section_key",
+        "text",
+        "heading_physical_page",
+        "source_page_range",
+        "atc_codes",
+        "est_tokens",
+        "chunk_kind",
+        "has_quarantined_content",
+    ):
+        assert stored[field] == record[field], f"{field} must round-trip exactly"
+
+
+@requires_qdrant
+def test_the_payload_index_actually_serves_a_filtered_query(store, collection):
+    """Creating an index proves nothing; querying through it does.
+
+    Mode A of the delivery plan never ranks by vector — it filters on
+    `drug_id` + `section_key` and returns the whole section. Until this test
+    existed the loader had only established that `create_payload_index`
+    returned without error.
+    """
+    spec = CollectionSpec(name=collection, vector_size=DIMENSIONS)
+    data = (
+        _section("metformin", "chong_chi_dinh", 5)
+        + _section("metformin", "lieu_luong_va_cach_dung", 3)
+        + _section("pantoprazol", "chong_chi_dinh", 4)
+    )
+    ChunkLoader(store, spec, manifest()).load(data)
+
+    found = store.find_by_payload(
+        collection, {"drug_id": "metformin", "section_key": "chong_chi_dinh"}
+    )
+
+    assert len(found) == 5
+    assert sorted(p.payload["part_index"] for p in found) == [0, 1, 2, 3, 4]
+    assert {p.payload["drug_id"] for p in found} == {"metformin"}
+
+
+@requires_qdrant
+def test_a_section_longer_than_one_scroll_page_comes_back_whole(store, collection):
+    """Paging must not truncate a section — that is the mode A failure mode.
+
+    Sized deliberately above `SCROLL_PAGE` (256) so a single-page implementation
+    fails here rather than in production on the one drug with a long section.
+    """
+    spec = CollectionSpec(name=collection, vector_size=DIMENSIONS)
+    parts = SCROLL_PAGE + 44
+    ChunkLoader(store, spec, manifest()).load(
+        _section("insulin", "lieu_luong_va_cach_dung", parts)
+    )
+
+    found = store.find_by_payload(
+        collection,
+        {"drug_id": "insulin", "section_key": "lieu_luong_va_cach_dung"},
+    )
+
+    assert len(found) == parts
+    assert sorted(p.payload["part_index"] for p in found) == list(range(parts))
+
+
+@requires_qdrant
+def test_a_list_valued_atc_field_matches_on_any_element_in_qdrant(store, collection):
+    spec = CollectionSpec(name=collection, vector_size=DIMENSIONS)
+    record = chunk_record("metformin__lieu_luong__0", atc_codes=["A10BA02", "A10BD20"])
+    ChunkLoader(store, spec, manifest()).load([(record, [0.4] * DIMENSIONS)])
+
+    assert len(store.find_by_payload(collection, {"atc_codes": "A10BD20"})) == 1
+    assert len(store.find_by_payload(collection, {"atc_codes": "J05AF06"})) == 0
+
+
+@requires_qdrant
+@pytest.mark.skipif(
+    not REAL_CHUNKS.exists(), reason="chunks.jsonl has not been generated"
+)
+def test_a_real_multipart_section_round_trips_through_the_filter(store, collection):
+    """Against the real artifact, not fixtures: every part, and only those."""
+    from ingestion.load import iter_chunk_records
+
+    wanted = None
+    records = []
+    for record in iter_chunk_records(REAL_CHUNKS):
+        if wanted is None and record["part_count"] >= 4:
+            wanted = (record["drug_id"], record["section_key"])
+        records.append(record)
+    assert wanted is not None, "no multi-part section in the artifact"
+
+    drug_id, section_key = wanted
+    expected = {
+        r["chunk_id"]
+        for r in records
+        if r["drug_id"] == drug_id and r["section_key"] == section_key
+    }
+    subset = [
+        r for r in records
+        if r["drug_id"] == drug_id or r["chunk_id"].startswith("abacavir")
+    ]
+
+    spec = CollectionSpec(name=collection, vector_size=DIMENSIONS)
+    ChunkLoader(store, spec, manifest()).load(
+        (r, [0.2] * DIMENSIONS) for r in subset
+    )
+
+    found = store.find_by_payload(
+        collection, {"drug_id": drug_id, "section_key": section_key}
+    )
+
+    assert {p.payload["chunk_id"] for p in found} == expected
+    assert len(expected) >= 4
+
+
+def _section(drug_id, section_key, parts):
+    return [
+        (
+            chunk_record(
+                f"{drug_id}__{section_key}__{i}",
+                drug_id=drug_id,
+                section_key=section_key,
+                part_index=i,
+                part_count=parts,
+            ),
+            [0.1] * DIMENSIONS,
+        )
+        for i in range(parts)
+    ]
+
+
+@requires_qdrant
+def test_an_unsupported_distance_is_rejected_before_the_server_is_called(store):
+    with pytest.raises(ValueError, match="unsupported distance"):
+        store.create_collection(
+            CollectionSpec(name="never_created", vector_size=4, distance="manhattan")
+        )