87 lines
3.1 KiB
Python
87 lines
3.1 KiB
Python
"""Binding a collection to the exact corpus and model it was built from (A6).
|
|
|
|
**Why a sidecar collection rather than a reserved point.** Qdrant has no
|
|
collection-level metadata field, so the manifest has to live in a point. Putting
|
|
that point inside the data collection would make `count()` one larger than the
|
|
chunk count — and `qdrant_point_count != chunk_count` is a v1 acceptance gate
|
|
(`docs/v1-delivery-plan.md` §6). A gate that needs an "except the manifest"
|
|
footnote is a gate that will eventually be read wrong. A `<name>__manifest`
|
|
collection keeps the data collection's count exactly equal to the number of
|
|
chunks, and keeps the manifest out of every search result by construction
|
|
rather than by remembering to filter it.
|
|
|
|
The vector on that point is a single zero. It is never searched; the point
|
|
exists only to carry a payload.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from typing import Optional
|
|
|
|
from .models import CollectionSpec, CorpusManifest, VectorPoint
|
|
from .ports import VectorStore
|
|
|
|
MANIFEST_SUFFIX = "__manifest"
|
|
MANIFEST_POINT_ID = "00000000-0000-5000-8000-000000000001"
|
|
|
|
|
|
class CorpusMismatch(RuntimeError):
|
|
"""Raised instead of upserting a corpus into a collection built elsewhere."""
|
|
|
|
|
|
def manifest_collection(name: str) -> str:
|
|
return f"{name}{MANIFEST_SUFFIX}"
|
|
|
|
|
|
def write_manifest(store: VectorStore, name: str, manifest: CorpusManifest) -> None:
|
|
sidecar = manifest_collection(name)
|
|
if not store.collection_exists(sidecar):
|
|
store.create_collection(CollectionSpec(name=sidecar, vector_size=1))
|
|
store.upsert(
|
|
sidecar,
|
|
[
|
|
VectorPoint(
|
|
id=MANIFEST_POINT_ID,
|
|
vector=[0.0],
|
|
payload=manifest.to_payload(),
|
|
)
|
|
],
|
|
)
|
|
|
|
|
|
def read_manifest(store: VectorStore, name: str) -> Optional[CorpusManifest]:
|
|
sidecar = manifest_collection(name)
|
|
if not store.collection_exists(sidecar):
|
|
return None
|
|
point = store.retrieve(sidecar, MANIFEST_POINT_ID)
|
|
if point is None:
|
|
return None
|
|
return CorpusManifest.from_payload(point.payload)
|
|
|
|
|
|
def assert_compatible(
|
|
store: VectorStore, name: str, incoming: CorpusManifest
|
|
) -> Optional[CorpusManifest]:
|
|
"""Refuse the load unless the collection was built from the same corpus.
|
|
|
|
Returns the stored manifest, or None when the collection is new. An
|
|
existing data collection with no manifest is itself a refusal: it was
|
|
loaded by something that did not record what it loaded, so nothing can be
|
|
said about what is already in there.
|
|
"""
|
|
stored = read_manifest(store, name)
|
|
if stored is None:
|
|
if store.collection_exists(name) and store.count(name) > 0:
|
|
raise CorpusMismatch(
|
|
f"collection {name!r} already holds {store.count(name)} points but "
|
|
f"has no manifest; refusing to mix an unknown corpus with "
|
|
f"{incoming.corpus_sha256[:12]}"
|
|
)
|
|
return None
|
|
|
|
reasons = stored.conflicts_with(incoming)
|
|
if reasons:
|
|
raise CorpusMismatch(
|
|
f"refusing to load into {name!r}: " + "; ".join(reasons)
|
|
)
|
|
return stored
|