Wire the guarded conversational RAG answer layer end-to-end

ingestion/ingestion/load/manifest.py

@@ -0,0 +1,86 @@
+"""Binding a collection to the exact corpus and model it was built from (A6).
+
+**Why a sidecar collection rather than a reserved point.** Qdrant has no
+collection-level metadata field, so the manifest has to live in a point. Putting
+that point inside the data collection would make `count()` one larger than the
+chunk count — and `qdrant_point_count != chunk_count` is a v1 acceptance gate
+(`docs/v1-delivery-plan.md` §6). A gate that needs an "except the manifest"
+footnote is a gate that will eventually be read wrong. A `<name>__manifest`
+collection keeps the data collection's count exactly equal to the number of
+chunks, and keeps the manifest out of every search result by construction
+rather than by remembering to filter it.
+
+The vector on that point is a single zero. It is never searched; the point
+exists only to carry a payload.
+"""
+from __future__ import annotations
+
+from typing import Optional
+
+from .models import CollectionSpec, CorpusManifest, VectorPoint
+from .ports import VectorStore
+
+MANIFEST_SUFFIX = "__manifest"
+MANIFEST_POINT_ID = "00000000-0000-5000-8000-000000000001"
+
+
+class CorpusMismatch(RuntimeError):
+    """Raised instead of upserting a corpus into a collection built elsewhere."""
+
+
+def manifest_collection(name: str) -> str:
+    return f"{name}{MANIFEST_SUFFIX}"
+
+
+def write_manifest(store: VectorStore, name: str, manifest: CorpusManifest) -> None:
+    sidecar = manifest_collection(name)
+    if not store.collection_exists(sidecar):
+        store.create_collection(CollectionSpec(name=sidecar, vector_size=1))
+    store.upsert(
+        sidecar,
+        [
+            VectorPoint(
+                id=MANIFEST_POINT_ID,
+                vector=[0.0],
+                payload=manifest.to_payload(),
+            )
+        ],
+    )
+
+
+def read_manifest(store: VectorStore, name: str) -> Optional[CorpusManifest]:
+    sidecar = manifest_collection(name)
+    if not store.collection_exists(sidecar):
+        return None
+    point = store.retrieve(sidecar, MANIFEST_POINT_ID)
+    if point is None:
+        return None
+    return CorpusManifest.from_payload(point.payload)
+
+
+def assert_compatible(
+    store: VectorStore, name: str, incoming: CorpusManifest
+) -> Optional[CorpusManifest]:
+    """Refuse the load unless the collection was built from the same corpus.
+
+    Returns the stored manifest, or None when the collection is new. An
+    existing data collection with no manifest is itself a refusal: it was
+    loaded by something that did not record what it loaded, so nothing can be
+    said about what is already in there.
+    """
+    stored = read_manifest(store, name)
+    if stored is None:
+        if store.collection_exists(name) and store.count(name) > 0:
+            raise CorpusMismatch(
+                f"collection {name!r} already holds {store.count(name)} points but "
+                f"has no manifest; refusing to mix an unknown corpus with "
+                f"{incoming.corpus_sha256[:12]}"
+            )
+        return None
+
+    reasons = stored.conflicts_with(incoming)
+    if reasons:
+        raise CorpusMismatch(
+            f"refusing to load into {name!r}: " + "; ".join(reasons)
+        )
+    return stored