Wire the guarded conversational RAG answer layer end-to-end

ingestion/tests/test_chunk.py

@@ -6,11 +6,19 @@ from ingestion.chunk import (
     CHUNK_KIND_PROSE,
     SCHEMA_VERSION,
     chunk_monograph,
+    chunk_all,
     chunk_section,
     write_chunks_jsonl,
 )
 from ingestion.chunk.chunker import _is_label_row, describe_block
-from ingestion.segment.models import Heading, Monograph, SectionSpan, TableBlock
+from ingestion.segment.models import (
+    PART_PROSE,
+    Heading,
+    Monograph,
+    SectionPart,
+    SectionSpan,
+    TableBlock,
+)
 from ingestion.tables import SHAPE_FORMULA_2D, SHAPE_MULTI_HEADER, SHAPE_SIMPLE
 
 
@@ -20,6 +28,15 @@ def _section(key, display, text, page=202):
         heading=Heading(text=display, physical_page=page, y0=100.0,
                         is_monograph_title=False, section_key=key),
         text=text,
+        parts=([
+            SectionPart(
+                kind=PART_PROSE,
+                text=text,
+                physical_page=page,
+                bbox=[50.0, 120.0, 550.0, 700.0],
+                source_span_ids=[f"p{page}_s0"],
+            )
+        ] if text else []),
     )
 
 
@@ -65,13 +82,17 @@ def test_a_section_whose_table_was_lifted_says_so():
 def test_a_lifted_block_gets_its_own_retrievable_descriptor():
     section = _section("lieu_luong_va_cach_dung", "Liều lượng và cách dùng", "Prose.")
     monograph = _monograph([section], [_block()])
-    descriptors = [c for c in chunk_monograph(monograph)
+    descriptors = [c for c in chunk_monograph(
+        monograph, printed_page_map={200: 201, 202: 203, 203: 204})
                    if c.chunk_kind == CHUNK_KIND_BLOCK_DESCRIPTOR]
     assert len(descriptors) == 1
     assert "AMPICILIN VÀ SULBACTAM" in descriptors[0].text
     assert "Liều lượng và cách dùng" in descriptors[0].text
     # printed page, which is what a reader holding the book looks for
     assert "trang 203" in descriptors[0].text
+    assert descriptors[0].source_page_range == [202, 202]
+    assert descriptors[0].printed_page_range == [203, 203]
+    assert descriptors[0].attachments[0].printed_page == 203
 
 
 def test_no_cell_value_ever_reaches_the_descriptor_text():
@@ -106,15 +127,17 @@ def test_a_header_row_carrying_a_number_is_refused():
     assert descriptor.attachments[0].header_row == []
 
 
-def test_only_a_simple_table_contributes_a_header():
+def test_unverified_header_rows_are_embargoed_for_every_table_shape():
     section = _section("lieu_luong_va_cach_dung", "Liều lượng và cách dùng", "Prose.")
-    header = {"p202_t0": ["Nhóm", "Liều"]}
-    for shape, expected in ((SHAPE_SIMPLE, ["Nhóm", "Liều"]),
-                            (SHAPE_MULTI_HEADER, [])):
+    header = {"p202_t0": ["Ngoại tâm thu thất", "Thường gặp", "Không rõ tần suất"]}
+    for shape in (SHAPE_SIMPLE, SHAPE_MULTI_HEADER):
         monograph = _monograph([section], [_block(shape=shape)])
         descriptor = next(c for c in chunk_monograph(monograph, header)
                           if c.chunk_kind == CHUNK_KIND_BLOCK_DESCRIPTOR)
-        assert descriptor.attachments[0].header_row == expected
+        assert descriptor.attachments[0].header_row == []
+        assert "Cột:" not in descriptor.text
+        assert "Ngoại tâm thu thất" not in descriptor.text
+        assert "Không rõ tần suất" not in descriptor.text
 
 
 def test_a_formula_block_is_described_as_a_formula():
@@ -163,6 +186,242 @@ def test_describe_block_names_the_page_even_with_no_header():
     chunks = chunk_monograph(monograph)
     attachment = next(c for c in chunks
                       if c.chunk_kind == CHUNK_KIND_BLOCK_DESCRIPTOR).attachments[0]
-    text = describe_block(monograph, section, attachment)
+    text = describe_block(monograph, section, attachment, printed_page=203)
     assert "trang 203" in text
     assert "không trích dẫn được dưới dạng văn bản" in text
+
+
+def test_chunk_carries_only_verified_printed_page_range():
+    section = _section("chi_dinh", "Chỉ định", "Nhiễm khuẩn.")
+    monograph = _monograph([section])
+    chunk = chunk_monograph(
+        monograph,
+        printed_page_map={202: 203},
+    )[0]
+    assert chunk.source_page_range == [202, 202]
+    assert chunk.printed_page_range == [203, 203]
+
+
+def test_chunk_refuses_an_unmapped_printed_folio():
+    section = _section("chi_dinh", "Chỉ định", "Nhiễm khuẩn.")
+    monograph = _monograph([section])
+    try:
+        chunk_monograph(monograph, printed_page_map={202: None})
+    except ValueError as exc:
+        assert "printed folio missing" in str(exc)
+    else:
+        raise AssertionError("missing printed folio must fail closed")
+
+
+def test_a_chunk_spanning_two_section_parts_cites_only_those_pages():
+    text = "Nội dung trang một.\nNội dung trang hai."
+    section = SectionSpan(
+        key="chi_dinh",
+        display_name="Chỉ định",
+        heading=Heading(
+            text="Chỉ định", physical_page=201, y0=100.0,
+            is_monograph_title=False, section_key="chi_dinh",
+        ),
+        text=text,
+        parts=[
+            SectionPart(PART_PROSE, "Nội dung trang một.", 201,
+                        [50.0, 100.0, 550.0, 200.0], ["p201_s0"]),
+            SectionPart(PART_PROSE, "Nội dung trang hai.", 202,
+                        [50.0, 100.0, 550.0, 200.0], ["p202_s0"]),
+        ],
+    )
+    chunk = chunk_monograph(
+        _monograph([section]), printed_page_map={201: 202, 202: 203}
+    )[0]
+
+    assert chunk.source_page_range == [201, 202]
+    assert chunk.printed_page_range == [202, 203]
+
+
+def test_whole_corpus_chunking_refuses_to_run_without_a_printed_page_map():
+    section = _section("chi_dinh", "Chỉ định", "Nhiễm khuẩn.")
+    try:
+        list(chunk_all([_monograph([section])], printed_page_map=None))
+    except ValueError as exc:
+        assert "requires a verified printed_page_map" in str(exc)
+    else:
+        raise AssertionError("whole-corpus chunking must fail closed without folios")
+
+
+def test_the_char_ratio_estimate_is_never_used_as_a_token_count():
+    """ADR 0004 sized chunks with len(text)//4 and reported 0 over the ceiling.
+
+    Counted with the real tokenizer, 1,884 of 12,838 chunks (14.7%) were over
+    it, the largest at 1,645 tokens — twice the ceiling. Vietnamese diacritics
+    cost multiple byte-pair tokens each; measured ratio real/estimate is 1.95
+    at the median and 6.0 at worst.
+    """
+    from ingestion.chunk.tokens import count_tokens, estimate_tokens
+
+    vietnamese = "Liều thường dùng cho người lớn là 1,5 - 3 g mỗi 6 giờ."
+    assert count_tokens(vietnamese) > len(vietnamese) // 4
+    # the fallback errs small, so it can never certify an oversized chunk as safe
+    assert estimate_tokens(vietnamese) > len(vietnamese) // 4
+
+
+def test_a_long_comma_list_is_split_at_commas_not_left_oversized():
+    """VORICONAZOL's interaction list is one 'sentence' hundreds of names long.
+
+    Truncated by an embedding model it reads as "this drug is not listed" — a
+    false negative in the direction that matters. A comma is a lossless break.
+    """
+    from ingestion.chunk.chunker import CEILING_TOKENS, _atoms
+
+    drugs = ", ".join(f"thuốc {n}" for n in range(400))
+    atoms = _atoms(drugs + ".", lambda t: len(t) // 2)
+    assert len(atoms) > 1
+    assert all(len(a) // 2 <= CEILING_TOKENS for a in atoms)
+    assert "".join(atoms).replace(",", "") == (drugs + ".").replace(",", "")
+
+
+def test_the_overlap_never_exceeds_its_budget():
+    """A 251-token atom produced a 273-token overlap against a 65-token
+    setting, because the loop added whole atoms until the total passed it.
+    That was most of how a 981-token chunk came about."""
+    from ingestion.chunk.chunker import OVERLAP_TOKENS, _pack
+
+    measure = lambda t: len(t)  # noqa: E731 - one-line stub for the test
+    atoms = ["a" * 300, "b" * 300, "c" * 300]
+    parts = _pack(atoms, measure)
+    assert len(parts) > 1
+    for part in parts[1:]:
+        carried = part[:-1]
+        assert sum(measure(a) for a in carried) <= OVERLAP_TOKENS
+
+
+def test_a_continuation_repeats_the_label_governing_its_dose():
+    """A budget-only overlap used to strand population labels.
+
+    The two 30-token dose atoms fit the 65-token overlap, while the preceding
+    label did not.  The continuation was therefore independently retrievable
+    as a bare dose even though its source context was population-specific.
+    """
+    from ingestion.chunk.chunker import _pack
+
+    measure = len
+    label = "Trẻ đẻ thiếu tháng:"
+    atoms = [
+        "p" * 570,
+        label,
+        "Uống liều 2 mg/kg q12h. " + "a" * 5,
+        "Nếu không uống được: " + "b" * 8,
+        "Theo dõi đáp ứng và điều chỉnh liều. " + "c" * 70,
+    ]
+
+    parts = _pack(atoms, measure)
+
+    assert len(parts) == 2
+    assert label in parts[1]
+    assert parts[1].index(label) < next(
+        index for index, atom in enumerate(parts[1]) if "liều" in atom
+    )
+
+
+def test_a_single_long_label_is_never_emitted_without_its_dose():
+    """When the current buffer held only one long label, the old loop could
+    not carry it and emitted a label-only retrievable chunk."""
+    from ingestion.chunk.chunker import _pack
+
+    label = "Trẻ sơ sinh có tình trạng lâm sàng cần hiệu chỉnh đặc biệt " * 2 + ":"
+    dose = "Dùng liều khởi đầu " + "x" * 640
+    parts = _pack([label, dose], len)
+
+    assert all(part != [label] for part in parts)
+    assert any(label in part and dose in part for part in parts)
+
+
+def test_a_new_trailing_label_does_not_orphan_the_previous_population_dose():
+    """Real shape: a neonatal dose is followed by ``Suy thận:`` at the seam.
+
+    Carrying only the new trailing label is insufficient: any dose atoms copied
+    into the overlap must retain the older population label that governs them.
+    """
+    from ingestion.chunk.chunker import _pack
+
+    population = "Trẻ đẻ thiếu tháng và trẻ sơ sinh dưới 8 ngày tuổi:"
+    renal = "Suy thận:"
+    first_dose = "100 mg/kg/ngày, chia hai lần. "
+    dose_limit = "Liều tối đa 10 mg/kg/ngày. "
+    atoms = [
+        "p" * 520,
+        population,
+        first_dose,
+        dose_limit,
+        renal,
+        "Điều chỉnh theo độ thanh thải creatinin. " + "x" * 80,
+    ]
+
+    parts = _pack(atoms, len)
+
+    assert len(parts) == 2
+    assert parts[1][0] == population
+    assert renal in parts[1]
+    copied_doses = [atom for atom in parts[1] if atom in (first_dose, dose_limit)]
+    if copied_doses:
+        assert parts[1].index(population) < min(parts[1].index(atom) for atom in copied_doses)
+
+
+def test_an_atom_ending_in_the_next_label_keeps_the_previous_dose_context():
+    """Bisoprolol has atoms shaped ``dose for step 4 ... Step 5:``.
+
+    Ending in a colon does not make the dose at the beginning of that same atom
+    belong to the new label.
+    """
+    from ingestion.chunk.chunker import _pack, _split_trailing_label
+
+    previous = "Bước 4:"
+    compound = "7,5 mg/lần/ngày trong 4 tuần; chuyển bước 5.\nBước 5:"
+    next_dose = "10 mg/lần/ngày để duy trì. "
+    split_compound = _split_trailing_label(compound)
+    assert "".join(split_compound) == compound
+    assert split_compound == [
+        "7,5 mg/lần/ngày trong 4 tuần; chuyển bước 5.\n",
+        "Bước 5:",
+    ]
+    atoms = [
+        "p" * 540,
+        previous,
+        *split_compound,
+        next_dose,
+        "Theo dõi dung nạp và điều chỉnh. " + "x" * 80,
+    ]
+
+    parts = _pack(atoms, len)
+
+    assert len(parts) == 2
+    dose_atom = split_compound[0]
+    if dose_atom in parts[1]:
+        assert previous in parts[1]
+        assert parts[1].index(previous) < parts[1].index(dose_atom)
+    assert split_compound[1] in parts[1]
+    assert parts[1].index(split_compound[1]) < parts[1].index(next_dose)
+
+
+def test_a_population_continuation_retains_its_parent_route():
+    """PARACETAMOL: age-band labels are children of ``Đường trực tràng:``."""
+    from ingestion.chunk.chunker import _pack
+
+    route = "Đường trực tràng:"
+    population = "Trẻ em 1 - 3 tháng tuổi:"
+    dose = "30 mg/kg một liều duy nhất. "
+    next_population = "Trẻ em 3 tháng - 6 tuổi:"
+    atoms = [
+        "p" * 540,
+        route,
+        population,
+        dose,
+        next_population,
+        "30 - 40 mg/kg một liều duy nhất. " + "x" * 80,
+    ]
+
+    parts = _pack(atoms, len)
+
+    assert len(parts) == 2
+    assert route in parts[1]
+    assert next_population in parts[1]
+    assert parts[1].index(route) < parts[1].index(next_population)

ingestion/tests/test_embed_cache.py

@@ -0,0 +1,233 @@
+"""The embedding cache, exercised with a counting stub and no network.
+
+The claim these tests exist to make checkable is narrow and financial: running
+the corpus a second time must cost nothing. That is asserted by counting calls
+the *inner* provider received, not by trusting a hit counter.
+
+The other half is the inverse — the cases where a hit would be wrong. Serving a
+vector after its text was edited, across two models, or across Cohere's
+document/query subspaces would each be silent: no error, just worse recall or a
+corpus of mixed vectors. There is a test per direction.
+"""
+import json
+
+import pytest
+
+from ingestion.embed import INPUT_DOCUMENT, EmbeddingVector
+from ingestion.embed.cache import (
+    CachingEmbeddingProvider,
+    EmbeddingCache,
+    cache_key,
+)
+from ingestion.embed.ports import EmbeddingProvider
+
+DIMENSIONS = 8
+
+
+class CountingProvider(EmbeddingProvider):
+    """Deterministic vectors, and a record of every text it was asked for."""
+
+    def __init__(self, model_id="stub-model-v1", dimensions=DIMENSIONS, batch=96):
+        self._model_id = model_id
+        self._dimensions = dimensions
+        self._batch = batch
+        self.embedded_texts = []
+        self.batch_calls = 0
+
+    @property
+    def name(self):
+        return "stub"
+
+    @property
+    def model_id(self):
+        return self._model_id
+
+    @property
+    def dimensions(self):
+        return self._dimensions
+
+    @property
+    def max_batch_size(self):
+        return self._batch
+
+    def _embed_batch(self, texts, input_kind):
+        self.batch_calls += 1
+        self.embedded_texts.extend(texts)
+        return [self._vector(text, input_kind) for text in texts]
+
+    def _vector(self, text, input_kind):
+        from ingestion.embed import text_digest
+
+        seed = len(text) + (0 if input_kind == INPUT_DOCUMENT else 1000)
+        return EmbeddingVector(
+            values=[float(seed + i) for i in range(self._dimensions)],
+            text_sha256=text_digest(text),
+            provider="stub",
+            model_id=self._model_id,
+            dimensions=self._dimensions,
+            input_kind=input_kind,
+            normalized=True,
+            input_token_count=len(text.split()),
+        )
+
+
+@pytest.fixture()
+def cache_path(tmp_path):
+    return tmp_path / "embeddings.jsonl"
+
+
+def test_second_run_over_the_same_texts_makes_zero_provider_requests(cache_path):
+    texts = ["paracetamol", "chống chỉ định", "liều dùng cho trẻ em"]
+    inner = CountingProvider()
+
+    first = CachingEmbeddingProvider(inner, EmbeddingCache(cache_path))
+    cold = first.embed_documents(texts)
+    assert cold.request_count == 1
+    assert inner.embedded_texts == texts
+
+    reopened = EmbeddingCache(cache_path)
+    warm = CachingEmbeddingProvider(inner, reopened).embed_documents(texts)
+
+    assert warm.request_count == 0
+    assert inner.embedded_texts == texts, "no text reached the provider twice"
+    assert reopened.stats.hits == len(texts)
+    assert reopened.stats.misses == 0
+    assert reopened.stats.hit_rate == 1.0
+    assert [v.values for v in warm.vectors] == [v.values for v in cold.vectors]
+
+
+def test_a_repeated_text_in_one_call_is_embedded_once(cache_path):
+    inner = CountingProvider()
+    provider = CachingEmbeddingProvider(inner, EmbeddingCache(cache_path))
+
+    batch = provider.embed_documents(["Abacavir.", "Abacavir.", "Abacavir."])
+
+    assert inner.embedded_texts == ["Abacavir."]
+    assert len(batch.vectors) == 3
+    assert batch.vectors[0].values == batch.vectors[2].values
+
+
+def test_editing_the_text_is_a_miss_not_a_stale_hit(cache_path):
+    inner = CountingProvider()
+    cache = EmbeddingCache(cache_path)
+    CachingEmbeddingProvider(inner, cache).embed_documents(["liều 500 mg"])
+
+    CachingEmbeddingProvider(inner, cache).embed_documents(["liều 250 mg"])
+
+    assert inner.embedded_texts == ["liều 500 mg", "liều 250 mg"]
+
+
+def test_a_second_model_never_reuses_the_first_models_vectors(cache_path):
+    cache = EmbeddingCache(cache_path)
+    titan = CountingProvider(model_id="amazon.titan-embed-text-v2:0")
+    cohere = CountingProvider(model_id="cohere.embed-v4:0")
+
+    CachingEmbeddingProvider(titan, cache).embed_documents(["metformin"])
+    CachingEmbeddingProvider(cohere, cache).embed_documents(["metformin"])
+
+    assert titan.embedded_texts == ["metformin"]
+    assert cohere.embedded_texts == ["metformin"]
+    assert len(cache) == 2
+
+
+def test_query_and_document_kinds_are_cached_separately(cache_path):
+    inner = CountingProvider()
+    cache = EmbeddingCache(cache_path)
+    provider = CachingEmbeddingProvider(inner, cache)
+
+    as_document = provider.embed_documents(["aspirin"])
+    as_query = provider.embed_queries(["aspirin"])
+
+    assert inner.embedded_texts == ["aspirin", "aspirin"]
+    assert as_document.vectors[0].values != as_query.vectors[0].values
+    assert len(cache) == 2
+
+
+def test_index_and_values_survive_reopening_the_file(cache_path):
+    inner = CountingProvider()
+    original = CachingEmbeddingProvider(
+        inner, EmbeddingCache(cache_path)
+    ).embed_documents(["ACETAZOLAMID", "ADENOSIN"])
+
+    reopened = EmbeddingCache(cache_path)
+
+    assert len(reopened) == 2
+    restored = reopened.get(cache_key(inner.model_id, INPUT_DOCUMENT, "ADENOSIN"))
+    assert restored is not None
+    assert restored.values == original.vectors[1].values
+    assert restored.input_kind == INPUT_DOCUMENT
+    assert restored.normalized is True
+    assert restored.input_token_count == 1
+
+
+def test_putting_a_key_twice_does_not_append_a_second_record(cache_path):
+    cache = EmbeddingCache(cache_path)
+    inner = CountingProvider()
+    vector = inner.embed_documents(["digoxin"]).vectors[0]
+
+    assert cache.put(vector) is True
+    assert cache.put(vector) is False
+
+    lines = cache_path.read_text(encoding="utf-8").strip().splitlines()
+    assert len(lines) == 1
+    assert len(cache) == 1
+
+
+def test_a_cached_record_whose_length_contradicts_its_dimensions_is_rejected(
+    cache_path,
+):
+    record = {
+        "model_id": "stub-model-v1",
+        "input_kind": INPUT_DOCUMENT,
+        "text_sha256": "a" * 64,
+        "provider": "stub",
+        "dimensions": 1024,
+        "normalized": True,
+        "input_token_count": 3,
+        "values": [0.1, 0.2],
+    }
+    cache_path.write_text(json.dumps(record) + "\n", encoding="utf-8")
+    cache = EmbeddingCache(cache_path)
+
+    with pytest.raises(ValueError, match="declares 1024 dimensions"):
+        cache.get(("stub-model-v1", INPUT_DOCUMENT, "a" * 64))
+
+
+def test_a_record_missing_key_fields_is_rejected_at_index_time(cache_path):
+    cache_path.write_text(
+        json.dumps({"model_id": "stub", "values": []}) + "\n", encoding="utf-8"
+    )
+
+    with pytest.raises(ValueError, match="missing key fields"):
+        EmbeddingCache(cache_path)
+
+
+def test_the_cache_wrapper_still_refuses_empty_text_and_bad_input_kind(cache_path):
+    provider = CachingEmbeddingProvider(CountingProvider(), EmbeddingCache(cache_path))
+
+    with pytest.raises(ValueError, match="empty or whitespace-only"):
+        provider.embed_documents(["paracetamol", "   "])
+    with pytest.raises(ValueError, match="input_kind must be one of"):
+        provider.embed(["paracetamol"], "search_document")
+
+
+def test_a_missing_cache_file_starts_empty_and_is_created_on_first_put(cache_path):
+    cache = EmbeddingCache(cache_path)
+
+    assert len(cache) == 0
+    assert not cache_path.exists()
+
+    CachingEmbeddingProvider(CountingProvider(), cache).embed_documents(["insulin"])
+
+    assert cache_path.exists()
+    assert len(cache) == 1
+
+
+def test_wrapper_reports_the_inner_models_identity_not_its_own(cache_path):
+    inner = CountingProvider(model_id="cohere.embed-v4:0", dimensions=DIMENSIONS)
+    provider = CachingEmbeddingProvider(inner, EmbeddingCache(cache_path))
+
+    assert provider.model_id == "cohere.embed-v4:0"
+    assert provider.dimensions == DIMENSIONS
+    assert provider.max_batch_size == inner.max_batch_size
+    assert provider.name == "cached:stub"

ingestion/tests/test_embed_providers.py

@@ -0,0 +1,324 @@
+"""Provider adapters, exercised with no AWS account and no network.
+
+Every Bedrock call goes through a recording stub, so what is under test is the
+part that can actually be wrong offline: the request body we send, and our
+reading of the response bodies AWS documents. The one thing these tests cannot
+establish is whether AWS accepts that body — that needs the live probe, and
+the coordination handoff says so explicitly.
+"""
+import io
+import json
+import math
+
+import pytest
+
+from ingestion.embed import (
+    BGE_M3,
+    COHERE_V4,
+    TITAN_V2,
+    INPUT_DOCUMENT,
+    INPUT_QUERY,
+    Boto3BedrockInvoker,
+    EmbeddingVector,
+    build_provider,
+    provider_names,
+    text_digest,
+)
+from ingestion.embed import probe
+from ingestion.embed.bedrock_cohere import CohereEmbedV4
+from ingestion.embed.bedrock_titan import TitanTextEmbeddingsV2
+from ingestion.embed.local_bge_m3 import BgeM3Local
+
+
+class RecordingInvoker:
+    """Stands in for Bedrock; remembers every request it was handed."""
+
+    def __init__(self, responses):
+        self._responses = list(responses)
+        self.calls = []
+
+    def invoke_json(self, model_id, payload, accept="application/json"):
+        self.calls.append(
+            {"model_id": model_id, "payload": payload, "accept": accept}
+        )
+        return self._responses.pop(0)
+
+
+def _titan_response(dimensions=1024, token_count=12):
+    return {
+        "embedding": [0.01] * dimensions,
+        "inputTextTokenCount": token_count,
+        "embeddingsByType": {"float": [0.01] * dimensions},
+    }
+
+
+def _cohere_by_type_response(rows, dimensions=1024):
+    return {
+        "id": "stub-id",
+        "response_type": "embeddings_by_type",
+        "embeddings": {"float": [[0.02] * dimensions for _ in range(rows)]},
+        "texts": ["stub"] * rows,
+    }
+
+
+def _cohere_floats_response(rows, dimensions=1024):
+    return {
+        "id": "stub-id",
+        "response_type": "embeddings_floats",
+        "embeddings": [[0.02] * dimensions for _ in range(rows)],
+    }
+
+
+def test_titan_request_body_matches_the_documented_v2_shape():
+    invoker = RecordingInvoker([_titan_response()])
+    provider = TitanTextEmbeddingsV2(invoker, dimensions=1024, normalize=True)
+
+    provider.embed_documents(["paracetamol"])
+
+    payload = invoker.calls[0]["payload"]
+    assert invoker.calls[0]["model_id"] == "amazon.titan-embed-text-v2:0"
+    assert payload == {
+        "inputText": "paracetamol",
+        "dimensions": 1024,
+        "normalize": True,
+    }
+
+
+def test_titan_records_provenance_and_reported_token_count():
+    invoker = RecordingInvoker([_titan_response(token_count=7)])
+    provider = TitanTextEmbeddingsV2(invoker)
+
+    vector = provider.embed_documents(["paracetamol"]).vectors[0]
+
+    assert vector.model_id == "amazon.titan-embed-text-v2:0"
+    assert vector.provider == TITAN_V2
+    assert vector.dimensions == 1024
+    assert vector.input_kind == INPUT_DOCUMENT
+    assert vector.normalized is True
+    assert vector.input_token_count == 7
+    assert vector.text_sha256 == text_digest("paracetamol")
+
+
+def test_titan_sends_one_request_per_text():
+    invoker = RecordingInvoker([_titan_response(), _titan_response()])
+    provider = TitanTextEmbeddingsV2(invoker)
+
+    batch = provider.embed_documents(["a", "b"])
+
+    assert batch.request_count == 2
+    assert len(batch.vectors) == 2
+
+
+def test_titan_rejects_a_dimension_the_model_does_not_offer():
+    with pytest.raises(ValueError, match="supports"):
+        TitanTextEmbeddingsV2(RecordingInvoker([]), dimensions=768)
+
+
+def test_cohere_uses_search_document_for_corpus_and_search_query_for_queries():
+    invoker = RecordingInvoker(
+        [_cohere_by_type_response(1), _cohere_by_type_response(1)]
+    )
+    provider = CohereEmbedV4(invoker)
+
+    provider.embed_documents(["metformin"])
+    provider.embed_queries(["liều metformin"])
+
+    assert invoker.calls[0]["payload"]["input_type"] == "search_document"
+    assert invoker.calls[1]["payload"]["input_type"] == "search_query"
+
+
+def test_cohere_request_body_pins_dimension_float_type_and_no_truncation():
+    invoker = RecordingInvoker([_cohere_by_type_response(2)])
+    provider = CohereEmbedV4(invoker, dimensions=1024)
+
+    provider.embed_documents(["a", "b"])
+
+    payload = invoker.calls[0]["payload"]
+    assert invoker.calls[0]["model_id"] == "cohere.embed-v4:0"
+    assert payload["texts"] == ["a", "b"]
+    assert payload["embedding_types"] == ["float"]
+    # Left unset the model would return 1536, which no 1024-wide collection
+    # can accept.
+    assert payload["output_dimension"] == 1024
+    # An over-length input must fail, not arrive silently shortened.
+    assert payload["truncate"] == "NONE"
+    assert invoker.calls[0]["accept"] == "*/*"
+
+
+def test_cohere_reads_the_embeddings_by_type_response():
+    invoker = RecordingInvoker([_cohere_by_type_response(2)])
+    provider = CohereEmbedV4(invoker)
+
+    batch = provider.embed_documents(["a", "b"])
+
+    assert len(batch.vectors) == 2
+    assert all(len(v.values) == 1024 for v in batch.vectors)
+    assert batch.request_count == 1
+
+
+def test_cohere_also_reads_the_plain_embeddings_floats_response():
+    invoker = RecordingInvoker([_cohere_floats_response(2)])
+    provider = CohereEmbedV4(invoker)
+
+    batch = provider.embed_documents(["a", "b"])
+
+    assert len(batch.vectors) == 2
+    assert all(len(v.values) == 1024 for v in batch.vectors)
+
+
+def test_cohere_leaves_normalization_unknown_because_the_docs_do_not_say():
+    invoker = RecordingInvoker([_cohere_by_type_response(1)])
+
+    vector = CohereEmbedV4(invoker).embed_documents(["a"]).vectors[0]
+
+    assert vector.normalized is None
+
+
+def test_cohere_splits_at_the_documented_96_text_ceiling():
+    invoker = RecordingInvoker(
+        [_cohere_by_type_response(96), _cohere_by_type_response(4)]
+    )
+    provider = CohereEmbedV4(invoker)
+
+    batch = provider.embed_documents([f"t{i}" for i in range(100)])
+
+    assert batch.request_count == 2
+    assert len(invoker.calls[0]["payload"]["texts"]) == 96
+    assert len(invoker.calls[1]["payload"]["texts"]) == 4
+    assert len(batch.vectors) == 100
+
+
+def test_cohere_rejects_a_batch_size_above_the_documented_ceiling():
+    with pytest.raises(ValueError, match="batch_size"):
+        CohereEmbedV4(RecordingInvoker([]), batch_size=97)
+
+
+def test_a_wrong_width_vector_fails_instead_of_entering_the_corpus():
+    invoker = RecordingInvoker([_titan_response(dimensions=512)])
+    provider = TitanTextEmbeddingsV2(invoker, dimensions=1024)
+
+    with pytest.raises(ValueError, match="512 dimensions"):
+        provider.embed_documents(["a"])
+
+
+def test_a_response_missing_its_vectors_fails_loudly():
+    invoker = RecordingInvoker([{"id": "stub", "response_type": "x"}])
+
+    with pytest.raises(ValueError, match="no 'embeddings' field"):
+        CohereEmbedV4(invoker).embed_documents(["a"])
+
+
+def test_a_count_mismatch_between_texts_and_vectors_fails():
+    invoker = RecordingInvoker([_cohere_by_type_response(1)])
+
+    with pytest.raises(ValueError, match="1 vectors for 2 texts"):
+        CohereEmbedV4(invoker).embed_documents(["a", "b"])
+
+
+def test_an_unknown_input_kind_is_refused_before_any_request_is_made():
+    invoker = RecordingInvoker([])
+
+    with pytest.raises(ValueError, match="input_kind"):
+        CohereEmbedV4(invoker).embed(["a"], "search_document")
+    assert invoker.calls == []
+
+
+def test_empty_text_is_refused_before_any_request_is_made():
+    invoker = RecordingInvoker([])
+
+    with pytest.raises(ValueError, match="empty"):
+        CohereEmbedV4(invoker).embed_documents(["a", "   "])
+    assert invoker.calls == []
+
+
+def test_bge_m3_runs_through_an_injected_encoder_with_no_weights_loaded():
+    seen = []
+
+    def encoder(texts):
+        seen.append(list(texts))
+        unit = 1.0 / math.sqrt(1024)
+        return [[unit] * 1024 for _ in texts]
+
+    provider = BgeM3Local(encoder=encoder, batch_size=2)
+    batch = provider.embed_queries(["a", "b", "c"])
+
+    assert seen == [["a", "b"], ["c"]]
+    assert batch.request_count == 2
+    assert len(batch.vectors) == 3
+    assert batch.vectors[0].input_kind == INPUT_QUERY
+    assert batch.vectors[0].model_id == "BAAI/bge-m3"
+    # Injected encoder: we did not set normalize_embeddings, so we do not claim it.
+    assert batch.vectors[0].normalized is None
+
+
+def test_registry_builds_every_provider_without_touching_an_sdk():
+    assert set(provider_names()) == {TITAN_V2, COHERE_V4, BGE_M3}
+
+    titan = build_provider(TITAN_V2, invoker=RecordingInvoker([]))
+    cohere = build_provider(COHERE_V4, invoker=RecordingInvoker([]))
+    local = build_provider(BGE_M3)
+
+    assert (titan.dimensions, cohere.dimensions, local.dimensions) == (
+        1024,
+        1024,
+        1024,
+    )
+    assert titan.max_batch_size == 1
+    assert cohere.max_batch_size == 96
+
+
+def test_registry_rejects_an_unknown_provider_name():
+    with pytest.raises(ValueError, match="unknown embedding provider"):
+        build_provider("text-embedding-3-small")
+
+
+class FakeBotoClient:
+    """The shape boto3's bedrock-runtime client returns: a streaming body."""
+
+    def __init__(self, response_body):
+        self._response_body = response_body
+        self.kwargs = None
+
+    def invoke_model(self, **kwargs):
+        self.kwargs = kwargs
+        return {"body": io.BytesIO(json.dumps(self._response_body).encode())}
+
+
+def test_boto3_invoker_serialises_the_request_and_reads_the_streamed_body():
+    client = FakeBotoClient({"embedding": [0.5]})
+    invoker = Boto3BedrockInvoker(region="us-east-1", client=client)
+
+    body = invoker.invoke_json("some.model", {"inputText": "à"}, accept="*/*")
+
+    assert body == {"embedding": [0.5]}
+    assert client.kwargs["modelId"] == "some.model"
+    assert client.kwargs["contentType"] == "application/json"
+    assert client.kwargs["accept"] == "*/*"
+    # Vietnamese must survive the round trip as characters, not \\u escapes
+    # the model would then embed literally.
+    assert json.loads(client.kwargs["body"]) == {"inputText": "à"}
+
+
+def test_probe_measures_the_l2_norm_rather_than_trusting_the_docs():
+    unit = 1.0 / math.sqrt(4)
+    assert probe._l2_norm([unit] * 4) == pytest.approx(1.0)
+    assert probe._l2_norm([3.0, 4.0]) == pytest.approx(5.0)
+
+
+def test_probe_reports_a_vector_without_raising(capsys):
+    vector = EmbeddingVector(
+        values=[0.5, 0.5, 0.5, 0.5],
+        text_sha256=text_digest("x"),
+        provider=TITAN_V2,
+        model_id="amazon.titan-embed-text-v2:0",
+        dimensions=4,
+        input_kind=INPUT_DOCUMENT,
+        normalized=True,
+        input_token_count=3,
+    )
+
+    probe._report(vector, latency_ms=12.5, requests=1)
+
+    out = capsys.readouterr().out
+    assert "amazon.titan-embed-text-v2:0" in out
+    assert "measured L2 norm: 1.000000" in out

ingestion/tests/test_entities_catalog.py

@@ -0,0 +1,40 @@
+from pathlib import Path
+
+import pytest
+
+from ingestion.entities.catalog import build_entities
+
+DATA = Path(__file__).parents[1] / "data"
+PDF = DATA / "raw/duoc-thu-quoc-gia-viet-nam-2018.pdf"
+MONOGRAPHS = DATA / "processed/monographs.jsonl"
+
+needs_source = pytest.mark.skipif(
+    not PDF.exists() or not MONOGRAPHS.exists(),
+    reason="source corpus artifacts are not available",
+)
+
+
+@needs_source
+def test_verified_entity_catalog_maps_every_explicit_see_alias():
+    payload = build_entities(MONOGRAPHS, PDF)
+    stats = payload["stats"]
+    assert stats["entity_count"] == 684
+    assert stats["back_index_see_relations"] == 344
+    assert stats["back_index_aliases_mapped"] == 344
+    assert stats["back_index_aliases_unresolved"] == 0
+    assert stats["back_index_aliases_ambiguous"] == 0
+    assert stats["trade_name_sections"] == 492
+
+
+@needs_source
+def test_common_parenthesized_names_are_emitted_as_aliases():
+    payload = build_entities(MONOGRAPHS, PDF)
+    entities = {item["drug_id"]: item for item in payload["entities"]}
+    aliases = {
+        drug_id: {alias.casefold() for alias in entity["aliases"]}
+        for drug_id, entity in entities.items()
+    }
+    assert "paracetamol" in aliases["paracetamol_acetaminophen"]
+    assert "acetaminophen" in aliases["paracetamol_acetaminophen"]
+    assert "aspirin" in aliases["acid_acetylsalicylic_aspirin"]
+    assert "oresol" in aliases["thuoc_uong_bu_nuoc_va_ien_giai"]

ingestion/tests/test_extract_formulas.py

@@ -2,6 +2,7 @@ import json
 from pathlib import Path
 
 from ingestion.extract.formulas import (
+    BARLESS_FORMULA_BOTTOM_PT,
     FORMULA_BAND_HEIGHT_PT,
     FORMULA_SIDE_MARGIN_PT,
     load_formula_regions,
@@ -50,6 +51,16 @@ def test_the_barless_adenosin_formula_is_recorded_as_a_recall_limit():
     assert "UNMEASURED" in payload["recall_limit"]
 
 
+def test_barless_adenosin_band_reaches_its_printed_denominator():
+    payload = json.loads(VERIFIED.read_text(encoding="utf-8"))
+    entry = next(r for r in payload["regions"] if r.get("source_prints_no_bar"))
+    region = next(r for r in load_formula_regions() if r.physical_page == 147)
+    assert region.bbox[3] == entry["bar_bbox"][3] + BARLESS_FORMULA_BOTTOM_PT
+    # "Ví dụ:" begins immediately afterwards with its span centre at ~704.3;
+    # the formula band must stop before that prose and the following table.
+    assert region.bbox[3] < 704
+
+
 def test_outlined_text_transcriptions_cover_every_detected_run():
     payload = json.loads(TRANSCRIPTIONS.read_text(encoding="utf-8"))
     runs = payload["runs"]

ingestion/tests/test_load_qdrant.py

@@ -0,0 +1,589 @@
+"""The load stage, exercised against the in-memory store with no server.
+
+Two classes of failure are silent in a vector database and are what most of
+these tests aim at. Loading the same corpus twice can leave two copies of a
+dose, and every query still succeeds — so idempotency is asserted by point
+count, not by inspecting the upsert calls. Mixing two corpus generations or two
+models into one collection also raises nothing at query time; every search
+returns *something*, just from the wrong material. That is what the manifest
+gate exists to make loud, and there is a test per way it can be violated.
+
+The last test runs over the real `chunks.jsonl` when it is present, because a
+provenance rule that only holds for hand-written records is not evidence.
+"""
+import json
+from pathlib import Path
+
+import pytest
+
+from ingestion.load import (
+    ChunkLoader,
+    CollectionSpec,
+    CorpusManifest,
+    CorpusMismatch,
+    InMemoryVectorStore,
+    PointCountMismatch,
+    build_point,
+    corpus_sha256,
+    count_chunks,
+    iter_chunk_records,
+    manifest_collection,
+    point_id_for,
+    read_manifest,
+    validate_chunk_record,
+)
+
+DIMENSIONS = 4
+COLLECTION = "duoc_thu_chunks"
+CORPUS_SHA = "a" * 64
+OTHER_SHA = "b" * 64
+MODEL = "amazon.titan-embed-text-v2:0"
+
+REAL_CHUNKS = (
+    Path(__file__).resolve().parents[1] / "data" / "processed" / "chunks.jsonl"
+)
+
+
+def chunk_record(chunk_id="abacavir__lieu_luong__0", **overrides):
+    record = {
+        "schema_version": 4,
+        "chunk_id": chunk_id,
+        "drug_id": "abacavir",
+        "drug_name": "ABACAVIR",
+        "section_key": "lieu_luong_va_cach_dung",
+        "section_display_name": "Liều lượng và cách dùng",
+        "text": "Người lớn: 300 mg, hai lần mỗi ngày.",
+        "source_text": "Người lớn: 300 mg, hai lần mỗi ngày.",
+        "heading_physical_page": 100,
+        "source_page_range": [100, 102],
+        "printed_page_range": [101, 103],
+        "atc_codes": ["J05AF06"],
+        "part_index": 0,
+        "part_count": 1,
+        "est_tokens": 14,
+        "oversized": False,
+        "chunk_kind": "prose",
+        "attachments": [],
+        "has_quarantined_content": False,
+    }
+    record.update(overrides)
+    return record
+
+
+def vector(seed=0.1):
+    return [seed] * DIMENSIONS
+
+
+def manifest(**overrides):
+    values = {
+        "corpus_sha256": CORPUS_SHA,
+        "chunk_count": 3,
+        "model_id": MODEL,
+        "dimensions": DIMENSIONS,
+        "input_kind": "document",
+        "provider": "titan-v2",
+    }
+    values.update(overrides)
+    return CorpusManifest(**values)
+
+
+def spec(**overrides):
+    values = {"name": COLLECTION, "vector_size": DIMENSIONS}
+    values.update(overrides)
+    return CollectionSpec(**values)
+
+
+def loader(store, **overrides):
+    return ChunkLoader(
+        store,
+        overrides.pop("spec", spec()),
+        overrides.pop("manifest", manifest()),
+        **overrides,
+    )
+
+
+def pairs(count=3):
+    return [
+        (chunk_record(chunk_id=f"drug__section__{i}"), vector(0.1 * (i + 1)))
+        for i in range(count)
+    ]
+
+
+# --- A5: derived ids and idempotency -------------------------------------
+
+
+def test_point_id_is_derived_from_chunk_id_and_is_stable():
+    first = point_id_for("abacavir__lieu_luong__0")
+    second = point_id_for("abacavir__lieu_luong__0")
+
+    assert first == second
+    assert first != point_id_for("abacavir__lieu_luong__1")
+
+
+def test_point_id_refuses_an_empty_chunk_id():
+    with pytest.raises(ValueError, match="chunk_id is required"):
+        point_id_for("   ")
+
+
+def test_loading_the_same_corpus_twice_leaves_the_point_count_unchanged():
+    store = InMemoryVectorStore()
+    data = pairs(3)
+
+    first = loader(store).load(data)
+    second = loader(store).load(data)
+
+    assert first.collection_created is True
+    assert second.collection_created is False
+    assert first.collection_count == 3
+    assert second.collection_count == 3, "a re-run duplicated points"
+    assert second.points_upserted == 3
+
+
+def test_a_reloaded_chunk_overwrites_its_own_point_rather_than_adding_one():
+    store = InMemoryVectorStore()
+    record = chunk_record()
+    loader(store).load([(record, vector(0.1))])
+
+    edited = chunk_record(text="Người lớn: 600 mg, một lần mỗi ngày.")
+    loader(store).load([(edited, vector(0.9))])
+
+    assert store.count(COLLECTION) == 1
+    stored = store.retrieve(COLLECTION, point_id_for(record["chunk_id"]))
+    assert stored.payload["text"] == "Người lớn: 600 mg, một lần mỗi ngày."
+    assert stored.vector == vector(0.9)
+
+
+def test_records_are_upserted_in_batches_of_the_configured_size():
+    store = InMemoryVectorStore()
+
+    report = loader(store, batch_size=2).load(pairs(5))
+
+    assert report.batches == 3
+    assert report.points_upserted == 5
+    assert report.collection_count == 5
+
+
+def test_batch_size_must_be_positive():
+    with pytest.raises(ValueError, match="batch_size must be positive"):
+        ChunkLoader(InMemoryVectorStore(), spec(), manifest(), batch_size=0)
+
+
+# --- A4: collection shape and payload ------------------------------------
+
+
+def test_collection_is_created_with_the_declared_size_and_payload_indexes():
+    store = InMemoryVectorStore()
+
+    loader(store).load(pairs(1))
+
+    assert store.spec(COLLECTION).vector_size == DIMENSIONS
+    assert store.spec(COLLECTION).distance == "Cosine"
+    indexed = dict(store.indexed_fields(COLLECTION))
+    assert indexed["drug_id"] == "keyword"
+    assert indexed["section_key"] == "keyword"
+    assert indexed["atc_codes"] == "keyword"
+    assert indexed["chunk_kind"] == "keyword"
+    assert indexed["has_quarantined_content"] == "bool"
+
+
+def test_payload_carries_every_provenance_field_of_the_chunk_record():
+    store = InMemoryVectorStore()
+    record = chunk_record()
+
+    loader(store).load([(record, vector())])
+
+    payload = store.retrieve(COLLECTION, point_id_for(record["chunk_id"])).payload
+    assert payload == record
+
+
+def test_a_field_added_by_a_future_chunker_flows_through_untouched():
+    store = InMemoryVectorStore()
+    record = chunk_record(
+        population_tags=["Người lớn", "Suy thận"], printed_page_range=[101, 103]
+    )
+
+    loader(store).load([(record, vector())])
+
+    payload = store.retrieve(COLLECTION, point_id_for(record["chunk_id"])).payload
+    assert payload["population_tags"] == ["Người lớn", "Suy thận"]
+    assert payload["printed_page_range"] == [101, 103]
+
+
+@pytest.mark.parametrize(
+    "field",
+    [
+        "chunk_id",
+        "drug_id",
+        "section_key",
+        "source_page_range",
+        "printed_page_range",
+        "text",
+    ],
+)
+def test_a_chunk_missing_a_required_provenance_field_is_refused(field):
+    with pytest.raises(ValueError, match="missing required provenance fields"):
+        validate_chunk_record(chunk_record(**{field: None}))
+
+
+# --- failing closed on incomplete provenance ------------------------------
+#
+# Every case below passed an earlier version of this validator. The cost of
+# that is not an exception at load time — it is paying for an embedding run and
+# then discovering every answer abstains because the chunks cannot be cited.
+
+
+@pytest.mark.parametrize("field", ["source_page_range", "printed_page_range"])
+def test_an_empty_page_range_is_missing_not_present(field):
+    """`[] in (None, "")` is False, which is exactly how this slipped through."""
+    with pytest.raises(ValueError, match="missing required provenance fields"):
+        validate_chunk_record(chunk_record(**{field: []}))
+
+
+def test_an_unknown_old_or_future_schema_is_refused_fail_closed():
+    for version in (3, 5):
+        with pytest.raises(ValueError, match="supports exactly v4"):
+            validate_chunk_record(chunk_record(schema_version=version))
+
+
+def test_a_chunk_with_no_schema_version_at_all_is_refused():
+    record = chunk_record()
+    del record["schema_version"]
+    with pytest.raises(ValueError, match="declares schema_version None"):
+        validate_chunk_record(record)
+
+
+@pytest.mark.parametrize(
+    "value", [[101], [101, 102, 103], "101-103", 101, {"start": 101}]
+)
+def test_a_page_range_that_is_not_a_pair_is_refused(value):
+    with pytest.raises(ValueError, match=r"expected a \[start, end\] pair"):
+        validate_chunk_record(chunk_record(printed_page_range=value))
+
+
+def test_a_page_range_running_backwards_is_refused():
+    with pytest.raises(ValueError, match="running backwards"):
+        validate_chunk_record(chunk_record(printed_page_range=[103, 101]))
+
+
+def test_a_non_integer_page_is_refused():
+    with pytest.raises(ValueError, match="non-integer page"):
+        validate_chunk_record(chunk_record(printed_page_range=[101.5, 103]))
+
+
+def test_boolean_pages_are_not_accepted_as_python_integers():
+    with pytest.raises(ValueError, match="non-integer page"):
+        validate_chunk_record(chunk_record(printed_page_range=[False, True]))
+
+
+@pytest.mark.parametrize(
+    ("field", "value"),
+    [("source_page_range", [-1, 0]), ("printed_page_range", [0, 1])],
+)
+def test_page_ranges_reject_impossible_lower_bounds(field, value):
+    with pytest.raises(ValueError, match="pages must start"):
+        validate_chunk_record(chunk_record(**{field: value}))
+
+
+def test_page_zero_and_false_are_values_not_absences():
+    """Physical pages are 0-indexed; a falsiness test would reject real records."""
+    validate_chunk_record(
+        chunk_record(
+            heading_physical_page=0,
+            source_page_range=[0, 0],
+            printed_page_range=[1, 1],
+            has_quarantined_content=False,
+            oversized=False,
+            part_index=0,
+        )
+    )
+
+
+def test_a_wrong_sized_vector_is_refused_before_anything_is_upserted():
+    store = InMemoryVectorStore()
+
+    with pytest.raises(ValueError, match="expects 4"):
+        loader(store).load([(chunk_record(), [0.1, 0.2])])
+
+    assert store.count(COLLECTION) == 0
+
+
+def test_manifest_dimensions_must_agree_with_the_collection_spec():
+    with pytest.raises(ValueError, match="manifest declares 8 dimensions"):
+        ChunkLoader(InMemoryVectorStore(), spec(), manifest(dimensions=8))
+
+
+def test_collection_spec_refuses_a_nonpositive_vector_size():
+    with pytest.raises(ValueError, match="vector_size must be positive"):
+        CollectionSpec(name=COLLECTION, vector_size=0)
+
+
+# --- A6: the corpus binding gate -----------------------------------------
+
+
+def test_the_manifest_lives_beside_the_data_so_the_point_count_stays_exact():
+    store = InMemoryVectorStore()
+
+    loader(store).load(pairs(3))
+
+    assert store.count(COLLECTION) == 3, "the manifest must not inflate the count"
+    assert store.count(manifest_collection(COLLECTION)) == 1
+    stored = read_manifest(store, COLLECTION)
+    assert stored.corpus_sha256 == CORPUS_SHA
+    assert stored.model_id == MODEL
+    assert stored.provider == "titan-v2"
+
+
+def test_a_second_corpus_generation_is_refused_and_nothing_is_written():
+    store = InMemoryVectorStore()
+    loader(store).load(pairs(3))
+
+    with pytest.raises(CorpusMismatch, match="does not match"):
+        loader(store, manifest=manifest(corpus_sha256=OTHER_SHA)).load(pairs(2))
+
+    assert store.count(COLLECTION) == 3
+    assert read_manifest(store, COLLECTION).corpus_sha256 == CORPUS_SHA
+
+
+def test_a_second_model_is_refused_even_when_the_corpus_matches():
+    store = InMemoryVectorStore()
+    loader(store).load(pairs(1))
+
+    with pytest.raises(CorpusMismatch, match="cohere.embed-v4:0"):
+        loader(store, manifest=manifest(model_id="cohere.embed-v4:0")).load(pairs(1))
+
+
+def test_a_query_subspace_vector_is_refused_for_a_document_collection():
+    store = InMemoryVectorStore()
+    loader(store).load(pairs(1))
+
+    with pytest.raises(CorpusMismatch, match="input kind query"):
+        loader(store, manifest=manifest(input_kind="query")).load(pairs(1))
+
+
+def test_a_dimension_change_is_refused():
+    store = InMemoryVectorStore()
+    loader(store).load(pairs(1))
+
+    eight = CollectionSpec(name=COLLECTION, vector_size=8)
+    with pytest.raises(CorpusMismatch, match="8 dimensions"):
+        ChunkLoader(store, eight, manifest(dimensions=8)).load(
+            [(chunk_record(), [0.1] * 8)]
+        )
+
+
+def test_an_existing_collection_with_no_manifest_is_refused():
+    store = InMemoryVectorStore()
+    store.create_collection(spec())
+    store.upsert(COLLECTION, [build_point(chunk_record(), vector())])
+
+    with pytest.raises(CorpusMismatch, match="has no manifest"):
+        loader(store).load(pairs(1))
+
+
+def test_the_same_corpus_and_model_is_allowed_through():
+    store = InMemoryVectorStore()
+    loader(store).load(pairs(3))
+
+    report = loader(store).load(pairs(3))
+
+    assert report.collection_count == 3
+    assert report.count_matches is True
+
+
+# --- the v1 point-count gate ---------------------------------------------
+
+
+def test_assert_point_count_passes_when_every_chunk_has_exactly_one_point():
+    store = InMemoryVectorStore()
+    active = loader(store)
+    active.load(pairs(3))
+
+    assert active.assert_point_count(3) == 3
+
+
+def test_assert_point_count_raises_when_the_collection_is_short():
+    store = InMemoryVectorStore()
+    active = loader(store)
+    active.load(pairs(3))
+
+    with pytest.raises(PointCountMismatch, match="holds 3 points but the corpus has 4"):
+        active.assert_point_count(4)
+
+
+# --- mode A: exhaustive filter retrieval ---------------------------------
+
+
+def section_pairs(drug_id, section_key, parts):
+    return [
+        (
+            chunk_record(
+                chunk_id=f"{drug_id}__{section_key}__{i}",
+                drug_id=drug_id,
+                section_key=section_key,
+                part_index=i,
+                part_count=parts,
+            ),
+            vector(0.1),
+        )
+        for i in range(parts)
+    ]
+
+
+def test_a_filter_returns_every_part_of_a_section_not_a_top_k():
+    """The rule mode A exists for: two of five contraindications is worse than none."""
+    store = InMemoryVectorStore()
+    loader(store).load(
+        section_pairs("metformin", "chong_chi_dinh", 5)
+        + section_pairs("metformin", "lieu_luong_va_cach_dung", 3)
+        + section_pairs("pantoprazol", "chong_chi_dinh", 2)
+    )
+
+    found = store.find_by_payload(
+        COLLECTION, {"drug_id": "metformin", "section_key": "chong_chi_dinh"}
+    )
+
+    assert len(found) == 5
+    assert sorted(p.payload["part_index"] for p in found) == [0, 1, 2, 3, 4]
+    assert {p.payload["drug_id"] for p in found} == {"metformin"}
+
+
+def test_a_filter_never_leaks_a_neighbouring_drugs_section():
+    store = InMemoryVectorStore()
+    loader(store).load(
+        section_pairs("pantoprazol", "chong_chi_dinh", 2)
+        + section_pairs("omeprazol", "chong_chi_dinh", 2)
+    )
+
+    found = store.find_by_payload(
+        COLLECTION, {"drug_id": "pantoprazol", "section_key": "chong_chi_dinh"}
+    )
+
+    assert len(found) == 2
+    assert {p.payload["drug_id"] for p in found} == {"pantoprazol"}
+
+
+def test_a_list_valued_field_matches_on_any_element():
+    store = InMemoryVectorStore()
+    record = chunk_record(atc_codes=["A10BA02", "A10BD20"])
+    loader(store).load([(record, vector())])
+
+    assert len(store.find_by_payload(COLLECTION, {"atc_codes": "A10BD20"})) == 1
+    assert len(store.find_by_payload(COLLECTION, {"atc_codes": "J05AF06"})) == 0
+
+
+def test_a_filter_matching_nothing_returns_empty_rather_than_raising():
+    store = InMemoryVectorStore()
+    loader(store).load(pairs(2))
+
+    assert store.find_by_payload(COLLECTION, {"drug_id": "khong_ton_tai"}) == []
+
+
+def test_a_filter_with_no_condition_is_refused():
+    store = InMemoryVectorStore()
+    loader(store).load(pairs(1))
+
+    with pytest.raises(ValueError, match="at least one condition"):
+        store.find_by_payload(COLLECTION, {})
+
+
+def test_parts_reassemble_in_order_into_the_whole_section():
+    store = InMemoryVectorStore()
+    bodies = ["Phần một.", "Phần hai.", "Phần ba."]
+    records = [
+        (
+            chunk_record(
+                chunk_id=f"metformin__chong_chi_dinh__{i}",
+                drug_id="metformin",
+                section_key="chong_chi_dinh",
+                text=body,
+                part_index=i,
+                part_count=len(bodies),
+            ),
+            vector(),
+        )
+        for i, body in enumerate(bodies)
+    ]
+    loader(store).load(records)
+
+    found = store.find_by_payload(
+        COLLECTION, {"drug_id": "metformin", "section_key": "chong_chi_dinh"}
+    )
+    ordered = sorted(found, key=lambda p: p.payload["part_index"])
+
+    assert [p.payload["text"] for p in ordered] == bodies
+    assert {p.payload["part_count"] for p in found} == {3}
+
+
+# --- corpus digest and reading -------------------------------------------
+
+
+def test_corpus_sha256_changes_when_a_single_byte_changes(tmp_path):
+    path = tmp_path / "chunks.jsonl"
+    path.write_text(json.dumps(chunk_record()) + "\n", encoding="utf-8")
+    before = corpus_sha256(path)
+
+    path.write_text(
+        json.dumps(chunk_record(text="Người lớn: 301 mg.")) + "\n", encoding="utf-8"
+    )
+
+    assert corpus_sha256(path) != before
+    assert len(before) == 64
+
+
+def test_the_same_data_hashes_the_same_under_crlf_and_lf(tmp_path):
+    """A gate that cries wolf gets switched off.
+
+    A raw-byte digest made a Windows CRLF checkout and a Linux LF checkout of
+    identical data disagree, so A6 would refuse a CI load against the very
+    corpus it was built from.
+    """
+    body = json.dumps(chunk_record()) + "\n" + json.dumps(chunk_record("b")) + "\n"
+    lf = tmp_path / "lf.jsonl"
+    crlf = tmp_path / "crlf.jsonl"
+    lf.write_bytes(body.encode("utf-8"))
+    crlf.write_bytes(body.replace("\n", "\r\n").encode("utf-8"))
+
+    assert corpus_sha256(lf) == corpus_sha256(crlf)
+    assert crlf.stat().st_size > lf.stat().st_size, "the files really do differ"
+
+
+def test_blank_lines_are_skipped_and_bad_json_names_its_line(tmp_path):
+    path = tmp_path / "chunks.jsonl"
+    path.write_text(
+        json.dumps(chunk_record()) + "\n\n" + json.dumps(chunk_record("b")) + "\n",
+        encoding="utf-8",
+    )
+
+    assert count_chunks(path) == 2
+
+    broken = tmp_path / "broken.jsonl"
+    broken.write_text(json.dumps(chunk_record()) + "\n{oops\n", encoding="utf-8")
+    with pytest.raises(ValueError, match="line 2 is not valid JSON"):
+        list(iter_chunk_records(broken))
+
+
+# --- the real artifact ----------------------------------------------------
+
+
+@pytest.mark.skipif(
+    not REAL_CHUNKS.exists(), reason="chunks.jsonl has not been generated"
+)
+def test_every_real_chunk_satisfies_the_loader_provenance_contract():
+    """Whole-artifact scope: all records in `data/processed/chunks.jsonl`."""
+    seen_ids = set()
+    seen_points = set()
+    total = 0
+    for record in iter_chunk_records(REAL_CHUNKS):
+        validate_chunk_record(record)
+        point = point_id_for(record["chunk_id"])
+        assert point not in seen_points, f"point id collision on {record['chunk_id']}"
+        seen_points.add(point)
+        seen_ids.add(record["chunk_id"])
+        total += 1
+
+    assert total == len(seen_ids), "duplicate chunk_id in the artifact"
+    assert total == len(seen_points)
+    # A floor, not the exact count: the corpus is regenerated as `segment/`
+    # changes, but a truncated or half-written artifact must not pass as
+    # whole-artifact evidence. Measured 15,066 records on 2026-08-04.
+    assert total > 10_000, f"chunks.jsonl looks truncated: only {total} records"

ingestion/tests/test_load_qdrant_integration.py

@@ -0,0 +1,377 @@
+"""`QdrantVectorStore` against a real Qdrant, skipped when none is running.
+
+The rest of the load suite runs against `InMemoryVectorStore` and proves the
+loader's rules. It cannot prove the adapter: whether Qdrant accepts a uuid5
+string as a point id, whether `create_payload_index` takes a bare `"keyword"`,
+whether an upsert of an existing id replaces rather than appends. Those are
+claims about another system, and the same class of claim as the Bedrock request
+bodies that are still documentation-derived and unproven — so they get a live
+check, against a free local container rather than a paid API.
+
+Start one with `docker compose -f infra/docker/docker-compose.yml up -d qdrant`.
+Without it these tests skip; they never fail for being offline.
+
+Every test works in its own collection named after the test and deletes it
+afterwards, so a shared local Qdrant is not left holding fixtures.
+"""
+import os
+import uuid
+from pathlib import Path
+
+import pytest
+
+from ingestion.load import (
+    ChunkLoader,
+    CollectionSpec,
+    CorpusManifest,
+    CorpusMismatch,
+    manifest_collection,
+    point_id_for,
+    read_manifest,
+)
+from ingestion.load.qdrant_repo import DEFAULT_URL, SCROLL_PAGE, QdrantVectorStore
+
+DIMENSIONS = 4
+QDRANT_URL = os.environ.get("QDRANT_URL", DEFAULT_URL)
+REAL_CHUNKS = (
+    Path(__file__).resolve().parents[1] / "data" / "processed" / "chunks.jsonl"
+)
+
+pytestmark = pytest.mark.integration
+
+
+def _server_is_up() -> bool:
+    try:
+        from qdrant_client import QdrantClient
+    except ImportError:
+        return False
+    try:
+        QdrantClient(url=QDRANT_URL, timeout=3.0).get_collections()
+    except Exception:
+        return False
+    return True
+
+
+requires_qdrant = pytest.mark.skipif(
+    not _server_is_up(), reason=f"no Qdrant reachable at {QDRANT_URL}"
+)
+
+
+@pytest.fixture()
+def store():
+    return QdrantVectorStore(url=QDRANT_URL, timeout=10.0)
+
+
+@pytest.fixture()
+def collection(store):
+    name = f"test_load_{uuid.uuid4().hex[:10]}"
+    yield name
+    for target in (manifest_collection(name), name):
+        if store.collection_exists(target):
+            store.delete_collection(target)
+
+
+def chunk_record(chunk_id, **overrides):
+    record = {
+        "schema_version": 4,
+        "chunk_id": chunk_id,
+        "drug_id": "abacavir",
+        "drug_name": "ABACAVIR",
+        "section_key": "lieu_luong_va_cach_dung",
+        "section_display_name": "Liều lượng và cách dùng",
+        "text": "Người lớn: 300 mg, hai lần mỗi ngày.",
+        "source_text": "Người lớn: 300 mg, hai lần mỗi ngày.",
+        "heading_physical_page": 100,
+        "source_page_range": [100, 102],
+        "printed_page_range": [101, 103],
+        "atc_codes": ["J05AF06"],
+        "part_index": 0,
+        "part_count": 1,
+        "est_tokens": 14,
+        "oversized": False,
+        "chunk_kind": "prose",
+        "attachments": [],
+        "has_quarantined_content": False,
+    }
+    record.update(overrides)
+    return record
+
+
+def manifest(**overrides):
+    values = {
+        "corpus_sha256": "a" * 64,
+        "chunk_count": 2,
+        "model_id": "amazon.titan-embed-text-v2:0",
+        "dimensions": DIMENSIONS,
+        "input_kind": "document",
+        "provider": "titan-v2",
+    }
+    values.update(overrides)
+    return CorpusManifest(**values)
+
+
+def pairs(count):
+    return [
+        (chunk_record(f"abacavir__section__{i}"), [0.1 * (i + 1)] * DIMENSIONS)
+        for i in range(count)
+    ]
+
+
+@requires_qdrant
+def test_a_real_load_creates_the_collection_indexes_and_points(store, collection):
+    spec = CollectionSpec(name=collection, vector_size=DIMENSIONS)
+
+    report = ChunkLoader(store, spec, manifest()).load(pairs(2))
+
+    assert report.collection_created is True
+    assert report.collection_count == 2
+    assert store.collection_exists(collection)
+    assert store.count(collection) == 2
+
+
+@requires_qdrant
+def test_qdrant_accepts_the_derived_uuid5_point_id_and_returns_the_payload(
+    store, collection
+):
+    spec = CollectionSpec(name=collection, vector_size=DIMENSIONS)
+    record = chunk_record("abacavir__lieu_luong__0")
+    ChunkLoader(store, spec, manifest()).load([(record, [0.5] * DIMENSIONS)])
+
+    stored = store.retrieve(collection, point_id_for(record["chunk_id"]))
+
+    assert stored is not None
+    assert stored.id == point_id_for(record["chunk_id"])
+    assert stored.payload["chunk_id"] == record["chunk_id"]
+    assert stored.payload["source_page_range"] == [100, 102]
+    assert stored.payload["atc_codes"] == ["J05AF06"]
+    assert stored.payload["has_quarantined_content"] is False
+    assert len(stored.vector) == DIMENSIONS
+
+
+@requires_qdrant
+def test_loading_twice_against_a_real_server_does_not_duplicate(store, collection):
+    spec = CollectionSpec(name=collection, vector_size=DIMENSIONS)
+    data = pairs(2)
+
+    ChunkLoader(store, spec, manifest()).load(data)
+    second = ChunkLoader(store, spec, manifest()).load(data)
+
+    assert second.collection_created is False
+    assert store.count(collection) == 2, "a re-run duplicated points in Qdrant"
+
+
+@requires_qdrant
+def test_the_manifest_sidecar_round_trips_and_leaves_the_count_exact(
+    store, collection
+):
+    spec = CollectionSpec(name=collection, vector_size=DIMENSIONS)
+    ChunkLoader(store, spec, manifest()).load(pairs(2))
+
+    stored = read_manifest(store, collection)
+
+    assert stored is not None
+    assert stored.corpus_sha256 == "a" * 64
+    assert stored.model_id == "amazon.titan-embed-text-v2:0"
+    assert stored.dimensions == DIMENSIONS
+    assert store.count(collection) == 2
+    assert store.count(manifest_collection(collection)) == 1
+
+
+@requires_qdrant
+def test_a_second_corpus_is_refused_against_a_real_collection(store, collection):
+    spec = CollectionSpec(name=collection, vector_size=DIMENSIONS)
+    ChunkLoader(store, spec, manifest()).load(pairs(2))
+
+    with pytest.raises(CorpusMismatch, match="does not match"):
+        ChunkLoader(store, spec, manifest(corpus_sha256="b" * 64)).load(pairs(2))
+
+    assert store.count(collection) == 2
+
+
+@requires_qdrant
+def test_retrieve_returns_none_for_an_id_that_was_never_loaded(store, collection):
+    spec = CollectionSpec(name=collection, vector_size=DIMENSIONS)
+    ChunkLoader(store, spec, manifest()).load(pairs(1))
+
+    assert store.retrieve(collection, point_id_for("never__loaded__0")) is None
+
+
+@requires_qdrant
+def test_bbox_floats_lose_precision_in_qdrant_but_nothing_else_does(store, collection):
+    """Pins a measured round-trip loss so it cannot silently get worse.
+
+    Scrolling all 15,066 points of a full load on 2026-08-04 found 86 chunks
+    whose payload did not compare equal to its source record. Every one of the
+    96 differing leaf values was a float inside `attachments[].bbox`, the
+    largest delta was 5.684e-14, and **no** text, id, page number, page range,
+    token count or boolean differed at all. A PDF point is 1/72 inch, so that
+    delta cannot move a rendered crop; what would matter is the loss spreading
+    to another field, or growing. This test fails if either happens.
+    """
+    spec = CollectionSpec(name=collection, vector_size=DIMENSIONS)
+    # 17 significant digits: the real corpus carries these, and they are what
+    # does not survive a float64 -> JSON -> float64 round trip.
+    bbox = [44.45098876953125, 397.45245361328125, 278.09100341796875, 463.4044494628906]
+    record = chunk_record(
+        "cefazolin__lieu_luong_va_cach_dung__0",
+        has_quarantined_content=True,
+        attachments=[
+            {
+                "block_id": "p344_t2",
+                "kind": "table",
+                "shape": "simple_table",
+                "physical_page": 344,
+                "bbox": bbox,
+                "quarantined": True,
+                "header_row": ["Cỡ lọ", "Lượng\ndung môi"],
+            }
+        ],
+    )
+    ChunkLoader(store, spec, manifest()).load([(record, [0.3] * DIMENSIONS)])
+
+    stored = store.retrieve(collection, point_id_for(record["chunk_id"])).payload
+    attachment = stored["attachments"][0]
+
+    for value, original in zip(attachment["bbox"], bbox, strict=True):
+        assert abs(value - original) < 1e-9, "bbox drift grew beyond rounding"
+
+    assert attachment["block_id"] == "p344_t2"
+    assert attachment["physical_page"] == 344
+    assert attachment["quarantined"] is True
+    assert attachment["header_row"] == ["Cỡ lọ", "Lượng\ndung môi"]
+    for field in (
+        "chunk_id",
+        "drug_id",
+        "drug_name",
+        "section_key",
+        "text",
+        "heading_physical_page",
+        "source_page_range",
+        "atc_codes",
+        "est_tokens",
+        "chunk_kind",
+        "has_quarantined_content",
+    ):
+        assert stored[field] == record[field], f"{field} must round-trip exactly"
+
+
+@requires_qdrant
+def test_the_payload_index_actually_serves_a_filtered_query(store, collection):
+    """Creating an index proves nothing; querying through it does.
+
+    Mode A of the delivery plan never ranks by vector — it filters on
+    `drug_id` + `section_key` and returns the whole section. Until this test
+    existed the loader had only established that `create_payload_index`
+    returned without error.
+    """
+    spec = CollectionSpec(name=collection, vector_size=DIMENSIONS)
+    data = (
+        _section("metformin", "chong_chi_dinh", 5)
+        + _section("metformin", "lieu_luong_va_cach_dung", 3)
+        + _section("pantoprazol", "chong_chi_dinh", 4)
+    )
+    ChunkLoader(store, spec, manifest()).load(data)
+
+    found = store.find_by_payload(
+        collection, {"drug_id": "metformin", "section_key": "chong_chi_dinh"}
+    )
+
+    assert len(found) == 5
+    assert sorted(p.payload["part_index"] for p in found) == [0, 1, 2, 3, 4]
+    assert {p.payload["drug_id"] for p in found} == {"metformin"}
+
+
+@requires_qdrant
+def test_a_section_longer_than_one_scroll_page_comes_back_whole(store, collection):
+    """Paging must not truncate a section — that is the mode A failure mode.
+
+    Sized deliberately above `SCROLL_PAGE` (256) so a single-page implementation
+    fails here rather than in production on the one drug with a long section.
+    """
+    spec = CollectionSpec(name=collection, vector_size=DIMENSIONS)
+    parts = SCROLL_PAGE + 44
+    ChunkLoader(store, spec, manifest()).load(
+        _section("insulin", "lieu_luong_va_cach_dung", parts)
+    )
+
+    found = store.find_by_payload(
+        collection,
+        {"drug_id": "insulin", "section_key": "lieu_luong_va_cach_dung"},
+    )
+
+    assert len(found) == parts
+    assert sorted(p.payload["part_index"] for p in found) == list(range(parts))
+
+
+@requires_qdrant
+def test_a_list_valued_atc_field_matches_on_any_element_in_qdrant(store, collection):
+    spec = CollectionSpec(name=collection, vector_size=DIMENSIONS)
+    record = chunk_record("metformin__lieu_luong__0", atc_codes=["A10BA02", "A10BD20"])
+    ChunkLoader(store, spec, manifest()).load([(record, [0.4] * DIMENSIONS)])
+
+    assert len(store.find_by_payload(collection, {"atc_codes": "A10BD20"})) == 1
+    assert len(store.find_by_payload(collection, {"atc_codes": "J05AF06"})) == 0
+
+
+@requires_qdrant
+@pytest.mark.skipif(
+    not REAL_CHUNKS.exists(), reason="chunks.jsonl has not been generated"
+)
+def test_a_real_multipart_section_round_trips_through_the_filter(store, collection):
+    """Against the real artifact, not fixtures: every part, and only those."""
+    from ingestion.load import iter_chunk_records
+
+    wanted = None
+    records = []
+    for record in iter_chunk_records(REAL_CHUNKS):
+        if wanted is None and record["part_count"] >= 4:
+            wanted = (record["drug_id"], record["section_key"])
+        records.append(record)
+    assert wanted is not None, "no multi-part section in the artifact"
+
+    drug_id, section_key = wanted
+    expected = {
+        r["chunk_id"]
+        for r in records
+        if r["drug_id"] == drug_id and r["section_key"] == section_key
+    }
+    subset = [
+        r for r in records
+        if r["drug_id"] == drug_id or r["chunk_id"].startswith("abacavir")
+    ]
+
+    spec = CollectionSpec(name=collection, vector_size=DIMENSIONS)
+    ChunkLoader(store, spec, manifest()).load(
+        (r, [0.2] * DIMENSIONS) for r in subset
+    )
+
+    found = store.find_by_payload(
+        collection, {"drug_id": drug_id, "section_key": section_key}
+    )
+
+    assert {p.payload["chunk_id"] for p in found} == expected
+    assert len(expected) >= 4
+
+
+def _section(drug_id, section_key, parts):
+    return [
+        (
+            chunk_record(
+                f"{drug_id}__{section_key}__{i}",
+                drug_id=drug_id,
+                section_key=section_key,
+                part_index=i,
+                part_count=parts,
+            ),
+            [0.1] * DIMENSIONS,
+        )
+        for i in range(parts)
+    ]
+
+
+@requires_qdrant
+def test_an_unsupported_distance_is_rejected_before_the_server_is_called(store):
+    with pytest.raises(ValueError, match="unsupported distance"):
+        store.create_collection(
+            CollectionSpec(name="never_created", vector_size=4, distance="manhattan")
+        )

ingestion/tests/test_segment_assembler.py

@@ -1,3 +1,5 @@
+from dataclasses import replace
+
 import pytest
 
 from ingestion.extract.models import Span
@@ -50,6 +52,55 @@ def test_non_bold_combined_heading_value_span_confirmed_real_amitriptylin_case()
     m = list(assemble(spans))[0]
     assert m.sections["ma_atc"].text == "N06AA09."
     assert m.atc_codes == ["N06AA09"]
+    part = m.sections["ma_atc"].parts[0]
+    assert part.physical_page == 184
+    assert part.bbox != [0.0, 0.0, 0.0, 0.0]
+    assert part.source_span_ids == [spans[3].span_id]
+
+
+def test_combined_international_name_and_atc_heading_is_a_title_anchor():
+    # Confirmed real GnRH class-monograph variant on physical page 1371.
+    spans = [
+        _span("THUỐC TƯƠNG TỰ HORMON GIẢI PHÓNG", 1371, 664.0),
+        _span("GONADOTROPIN", 1371, 676.0),
+        _span("Tên chung quốc tế và mã ATC", 1371, 690.0),
+        _span("Gonadorelin: H01CA01; Triptorelin: L02AE04.", 1371, 702.0, bold=False),
+        _span("Chỉ định", 1371, 714.0),
+        _span("Kích thích phóng noãn.", 1371, 726.0, bold=False),
+    ]
+    m = list(assemble(spans))[0]
+    assert m.drug_name == "THUỐC TƯƠNG TỰ HORMON GIẢI PHÓNG GONADOTROPIN"
+    assert m.atc_codes == ["H01CA01", "L02AE04"]
+
+
+def test_plain_wrapped_section_label_is_body_not_a_heading():
+    # NADROPARIN CALCI p1016: "không phải là" / "chống chỉ định."
+    # are adjacent lines of one sentence in the same PDF block.
+    spans = [
+        _span("NADROPARIN CALCI", 1016, 60.0),
+        _span("Tên chung quốc tế", 1016, 80.0),
+        _span("Nadroparin calcium.", 1016, 92.0, bold=False),
+        _span("Thời kỳ cho con bú", 1016, 110.0),
+    ]
+    lead = replace(_span("Việc dùng thuốc không phải là", 1016, 122.0, bold=False),
+                   block=4, line=7)
+    tail = replace(_span("chống chỉ định.", 1016, 134.0, bold=False),
+                   block=4, line=8)
+    m = list(assemble(spans + [lead, tail]))[0]
+    assert m.sections["thoi_ky_cho_con_bu"].text.endswith("chống chỉ định.")
+    assert "chong_chi_dinh" not in m.sections
+
+
+def test_plain_heading_after_completed_prose_still_opens_section():
+    spans = [
+        _span("TESTDRUG", 300, 60.0),
+        _span("Tên chung quốc tế", 300, 80.0),
+        replace(_span("Testdrug.", 300, 92.0, bold=False), block=2, line=0),
+        replace(_span("Chỉ định", 300, 104.0, bold=False), block=2, line=1),
+        replace(_span("Điều trị thử nghiệm.", 300, 116.0, bold=False), block=2, line=2),
+    ]
+    m = list(assemble(spans))[0]
+    assert m.sections["chi_dinh"].text == "Điều trị thử nghiệm."
 
 
 def test_atc_stated_absent_propagates():
@@ -330,3 +381,71 @@ def test_a_bold_label_line_still_opens_its_section():
     ]
     monograph = list(assemble(spans))[0]
     assert monograph.sections["chong_chi_dinh"].text == "Suy tủy nặng."
+
+
+def test_a_section_name_printed_mid_line_is_body_not_a_heading():
+    """CISPLATIN, physical page 402 — confirmed content loss.
+
+    The book prints "Suy thận: Chống chỉ định." inside the dosing section. The
+    second half is itself a section name, so it was matched as a heading: the
+    renal-impairment contraindication vanished from the dosing text and the
+    section ended on a bare "Suy thận:". ISOPRENALIN had the same shape. A
+    real heading opens its line; this one does not.
+    """
+    spans = [
+        _span("CISPLATIN", 401, 60.0),
+        _span("Tên chung quốc tế", 401, 80.0),
+        _span("Cisplatinum.", 401, 92.0, bold=False),
+        _span("Liều lượng và cách dùng", 401, 110.0),
+        _span("Truyền tĩnh mạch mỗi 3 tuần.", 401, 122.0, bold=False),
+    ]
+    label = _span("Suy thận: ", 401, 140.0, bold=False)
+    label = replace(label, block=4, line=0, x0=35.0, x1=70.0)
+    trailing = _span("Chống chỉ định.", 401, 140.0, bold=False)
+    trailing = replace(trailing, block=4, line=0, x0=70.0, x1=140.0)
+
+    monograph = list(assemble(spans + [label, trailing]))[0]
+    dosing = monograph.sections["lieu_luong_va_cach_dung"].text
+    assert "Suy thận: Chống chỉ định." in dosing
+    assert "chong_chi_dinh" not in monograph.sections
+
+
+def test_italic_cross_reference_overlapping_its_neighbour_by_a_hairline_is_body():
+    """NEVIRAPIN, physical page 1045 — confirmed misassignment, whole-corpus.
+
+    The book prints `Xem thêm mục ` (x1=104.89) immediately before an italic
+    `Liều lượng và cách dùng` (x0=104.88): the trailing space's advance width
+    makes the neighbour end 0.01pt *after* the cross-reference starts. An
+    end-before-start test therefore read a mid-line cross-reference as a
+    heading. Same shape, same cause, in CALCI LACTAT (p296, `xem thêm mục
+    Tương tác thuốc`, 0.02pt) and CEFAZOLIN (p344, `ghi ở mục: Dạng thuốc và
+    hàm lượng.`), where 4,533 characters of adult dosing were filed under
+    dosage forms.
+    """
+    spans = [
+        _span("NEVIRAPIN", 1044, 60.0),
+        _span("Tên chung quốc tế", 1044, 80.0),
+        _span("Nevirapine.", 1044, 92.0, bold=False),
+        _span("Hướng dẫn cách xử trí ADR", 1044, 110.0),
+        _span("Điều trị các phản ứng bất lợi theo triệu chứng.", 1044, 122.0, bold=False),
+    ]
+    lead = replace(_span("Xem thêm mục ", 1044, 140.0, bold=False),
+                   block=4, line=0, x0=43.94, x1=104.89)
+    reference = replace(_span("Liều lượng và cách dùng", 1044, 140.0, bold=False),
+                        block=4, line=0, x0=104.88, x1=199.57)
+
+    monograph = list(assemble(spans + [lead, reference]))[0]
+    assert "Xem thêm mục Liều lượng và cách dùng" in monograph.sections["huong_dan_xu_tri_adr"].text
+    assert "lieu_luong_va_cach_dung" not in monograph.sections
+
+
+def test_a_section_name_opening_its_own_line_is_still_a_heading():
+    spans = [
+        _span("CISPLATIN", 401, 60.0),
+        _span("Tên chung quốc tế", 401, 80.0),
+        _span("Cisplatinum.", 401, 92.0, bold=False),
+        _span("Chống chỉ định", 401, 110.0),
+        _span("Suy tủy nặng.", 401, 122.0, bold=False),
+    ]
+    monograph = list(assemble(spans))[0]
+    assert monograph.sections["chong_chi_dinh"].text == "Suy tủy nặng."

ingestion/tests/test_segment_detector.py

@@ -1,5 +1,9 @@
 from ingestion.extract.models import Span
-from ingestion.segment.detector import detect_monograph_titles, detect_section_headings
+from ingestion.segment.detector import (
+    detect_monograph_titles,
+    detect_section_headings,
+    in_monograph_range,
+)
 
 
 def _span(text, physical_page, printed_page, y0=100.0, bold=True, size=10.0):
@@ -86,3 +90,12 @@ def test_unknown_bold_text_not_matched_as_section():
 def test_section_heading_outside_monograph_range_excluded():
     spans = [_span("Chỉ định", 5, 6, bold=True, size=9.5)]
     assert list(detect_section_headings(spans)) == []
+
+
+def test_back_index_cannot_reenter_range_via_bad_inferred_printed_page():
+    # Confirmed real failure: physical page 1655 of the back index was mapped
+    # to printed page 1496, making its "Tương tác thuốc" entry extend
+    # ZOLPIDEM's source range from page 1494 through page 1655.
+    index_span = _span("Tương tác thuốc", 1655, 1496)
+    assert in_monograph_range(index_span) is False
+    assert list(detect_section_headings([index_span])) == []

ingestion/tests/test_segment_tables.py

@@ -1,6 +1,12 @@
 from ingestion.extract.models import Span
 from ingestion.segment import assemble
-from ingestion.tables import SHAPE_GRID_2D, SHAPE_SIMPLE, TableRegion, index_by_page
+from ingestion.tables import (
+    SHAPE_FORMULA_2D,
+    SHAPE_GRID_2D,
+    SHAPE_SIMPLE,
+    TableRegion,
+    index_by_page,
+)
 
 
 def _span(text, page, y0, *, bold=False, x0=50.0, block=0, line=0, column="left"):
@@ -54,6 +60,40 @@ def test_table_spans_are_lifted_out_of_section_prose():
     assert block.quarantined is True
 
 
+def test_section_named_table_cell_does_not_change_owning_section():
+    # Confirmed in WARFARIN p1485 and IOBITRIDOL p826: a table column named
+    # "Chỉ định" belongs to the dosing table; it is not a document
+    # section heading and must not move the block into chi_dinh.
+    spans = [
+        _span("WARFARIN", 1485, 60.0, bold=True),
+        _span("Tên chung quốc tế", 1485, 80.0, bold=True),
+        _span("Warfarinum.", 1485, 92.0),
+        _span("Liều lượng và cách dùng", 1485, 200.0, bold=True),
+        _span("Chỉ định", 1485, 400.0, bold=True, block=5),
+        _span("INR 2,0 - 3,0", 1485, 412.0, block=5),
+    ]
+    region = TableRegion("p1485_t0", 1485, (40.0, 380.0, 400.0, 460.0), 2, 2, SHAPE_SIMPLE)
+    m = list(assemble(spans, table_index=index_by_page([region])))[0]
+    assert "chi_dinh" not in m.sections
+    assert len(m.tables) == 1
+    assert m.tables[0].section_key == "lieu_luong_va_cach_dung"
+
+
+def test_wide_formula_band_does_not_swallow_the_opposite_column():
+    spans = _monograph_spans([
+        _span("Công thức:", 109, 360.0),
+        _span("Cl", 109, 400.0, x0=280.0, column="left", block=5),
+        _span("Xem thêm Liều lượng và cách dùng", 109, 400.0,
+              x0=310.0, column="right", block=6),
+    ])
+    # Deliberately extends across the gutter, as verified formula bands do.
+    region = TableRegion("p109_f0", 109, (40.0, 380.0, 390.0, 430.0),
+                         2, 1, SHAPE_FORMULA_2D)
+    m = list(assemble(spans, table_index=index_by_page([region])))[0]
+    assert m.tables[0].text == "Cl"
+    assert "Xem thêm Liều lượng và cách dùng" in m.sections["dang_thuoc_va_ham_luong"].text
+
+
 def test_without_a_region_map_behaviour_is_unchanged():
     spans = _monograph_spans([
         _span("Thuốc dùng đường uống.", 109, 220.0),
@@ -85,24 +125,53 @@ def test_non_table_regions_are_never_lifted():
 
 
 def test_table_block_ids_stay_unique_when_a_section_resumes():
-    # a region flushed twice (section closes, then resumes) must not emit two
-    # blocks with the same table_id — provenance ids have to be unique
+    # Confirmed on CAPECITABIN pp. 308-309 and IMATINIB p. 795: PDF block
+    # order can place a visually later heading between cells from one physical
+    # table.  The complete region must stay atomic and owned by the section
+    # active where the table first appears.
     spans = [
         _span("CEFAMANDOL", 339, 60.0, bold=True),
         _span("Tên chung quốc tế", 339, 80.0, bold=True),
         _span("Cefamandolum.", 339, 92.0),
         _span("Liều lượng và cách dùng", 339, 200.0, bold=True),
         _span("80 - 50", 339, 400.0, block=5),
-        _span("Liều lượng và cách dùng", 339, 500.0, bold=True),
+        # Visually below the table, but emitted before its final cell by the
+        # PDF's internal block order.
+        _span("Tương tác thuốc", 339, 640.0, bold=True),
         _span("< 25 - 10", 339, 600.0, block=9),
+        _span("Không phối hợp với thuốc X.", 339, 660.0, block=10),
     ]
     region = TableRegion("p339_t0", 339, (40.0, 380.0, 400.0, 620.0), 5, 2, SHAPE_SIMPLE)
     m = list(assemble(spans, table_index=index_by_page([region])))[0]
-    # table_id is deterministic per REGION, so two parts of one table share
-    # it on purpose; table_part_id is the unique key, derived from the first
-    # source span rather than a counter (a counter would renumber whenever
-    # anything upstream shifted, hiding rather than identifying a duplicate)
+    assert len(m.tables) == 1
+    assert m.tables[0].section_key == "lieu_luong_va_cach_dung"
+    assert "80 - 50" in m.tables[0].text
+    assert "< 25 - 10" in m.tables[0].text
+    assert "Không phối hợp với thuốc X." in m.sections["tuong_tac_thuoc"].text
     assert len({t.table_part_id for t in m.tables}) == len(m.tables)
     assert {t.continuation_group for t in m.tables} == {"p339_t0"}
     assert all(t.table_part_id.startswith("p339_t0@") for t in m.tables)
     assert all(t.quarantined for t in m.tables)
+
+
+def test_explicit_dose_adjustment_caption_reassigns_late_appendix_table():
+    # CAPECITABIN p. 309: the PDF puts dose-adjustment tables after the trade
+    # names and does not repeat the ordinary dosage section heading.  Internal
+    # block order can even emit a cell before the visually preceding caption.
+    spans = [
+        _span("CAPECITABIN", 309, 40.0, bold=True),
+        _span("Tên chung quốc tế", 309, 50.0, bold=True),
+        _span("Capecitabinum.", 309, 60.0),
+        _span("Tên thương mại", 309, 70.0, bold=True),
+        _span("Xeloda.", 309, 80.0),
+        _span("Mức độ theo NCIC", 309, 120.0, block=5),
+        _span("Bảng 3. Điều chỉnh liều do độc tính.", 309, 100.0),
+        _span("Ngừng thuốc cho đến khi về mức 0.", 309, 140.0, block=5),
+    ]
+    region = TableRegion("p309_t0", 309, (40.0, 115.0, 400.0, 180.0),
+                         3, 4, SHAPE_SIMPLE)
+    m = list(assemble(spans, table_index=index_by_page([region])))[0]
+    assert len(m.tables) == 1
+    assert m.tables[0].section_key == "lieu_luong_va_cach_dung"
+    dosage = m.sections["lieu_luong_va_cach_dung"].text
+    assert dosage.count("Bảng 3. Điều chỉnh liều do độc tính.") == 1

ingestion/tests/test_segment_vocab.py

@@ -53,6 +53,10 @@ def test_real_spelling_variants_found_in_the_book_all_match():
         "Hướng dẫn cách sử trí ADR": "huong_dan_xu_tri_adr",
         "Quá liều và xử lý": "qua_lieu_va_xu_tri",
         "Lọai thuốc": "loai_thuoc",
+        "Tên chung quốc tế và mã ATC": "ten_chung_quoc_te",
+        "Dạng bào chế và hàm lượng": "dang_thuoc_va_ham_luong",
+        "Liều lượng và cách dùng giải độc tố uốn ván hấp phụ đơn giá":
+            "lieu_luong_va_cach_dung",
     }
     for text, expected_key in cases.items():
         matched = match_section(text)

ingestion/tests/test_validation_readiness.py

@@ -0,0 +1,72 @@
+from ingestion.validation.readiness import evaluate, evaluate_chunks
+
+
+def _count(monographs, gate_name):
+    return next(g.count for g in evaluate(monographs) if g.name == gate_name)
+
+
+def test_readiness_rejects_a_part_without_source_span_provenance():
+    corpus = [{
+        "drug_id": "testdrug",
+        "source_page_range": [100, 100],
+        "sections": {
+            "ma_atc": {
+                "text": "N00AA00",
+                "parts": [{"kind": "prose", "text": "N00AA00", "source_span_ids": []}],
+            },
+        },
+        "tables": [],
+    }]
+    assert _count(corpus, "section_without_provenance") == 0
+    assert _count(corpus, "part_without_source_span_ids") == 1
+
+
+def test_readiness_accepts_part_level_source_span_provenance():
+    corpus = [{
+        "drug_id": "testdrug",
+        "source_page_range": [100, 100],
+        "sections": {
+            "ma_atc": {
+                "text": "N00AA00",
+                "parts": [{
+                    "kind": "prose", "text": "N00AA00",
+                    "source_span_ids": ["p100_b1_l0_s0"],
+                }],
+            },
+        },
+        "tables": [],
+    }]
+    assert _count(corpus, "part_without_source_span_ids") == 0
+
+
+def test_readiness_rejects_duplicate_physical_region_ids():
+    corpus = [{
+        "drug_id": "testdrug",
+        "source_page_range": [100, 100],
+        "sections": {},
+        "tables": [
+            {"table_id": "p100_t0", "quarantined": True},
+            {"table_id": "p100_t0", "quarantined": True},
+        ],
+    }]
+    assert _count(corpus, "duplicate_table_id") == 1
+
+
+def test_chunk_readiness_requires_a_verified_printed_page_range():
+    chunk = {
+        "chunk_id": "drug__dose__0",
+        "drug_id": "drug",
+        "section_key": "dose",
+        "chunk_kind": "prose",
+        "text": "Dose.",
+        "est_tokens": 2,
+        "attachments": [],
+    }
+    gates = evaluate_chunks([], [chunk])
+    missing = next(g for g in gates if g.name == "chunk_without_printed_page_range")
+    assert missing.count == 1
+
+    chunk["printed_page_range"] = [101, 102]
+    gates = evaluate_chunks([], [chunk])
+    present = next(g for g in gates if g.name == "chunk_without_printed_page_range")
+    assert present.count == 0