Wire token-budget packing into the overview/rerank fallback path

apps/ai-service/adapters/qdrant.py

@@ -5,6 +5,16 @@ from typing import Any, Protocol, Sequence
 from rag.models import ParentDocument, RetrievalDocument, SearchHit, SourceRef
 
 
+# Vietnamese function words dropped before lexical matching — high-frequency,
+# low-signal; matching on these alone would make `search_lexical` return
+# near-arbitrary same-drug chunks instead of ones sharing real query terms.
+_LEXICAL_STOPWORDS = frozenset({
+    "cua", "va", "la", "cho", "khi", "co", "gi", "duoc", "voi", "the", "nao",
+    "nhu", "o", "trong", "de", "hay", "mot", "nay", "day", "thi", "bi",
+    "khong", "da", "se", "neu", "nen", "phai", "sao",
+})
+
+
 class QueryEmbedder(Protocol):
     @property
     def dimensions(self) -> int: ...
@@ -93,6 +103,9 @@ def _document(payload: dict[str, Any]) -> RetrievalDocument:
             bool(payload.get("requires_visual_check"))
             or bool(payload.get("has_quarantined_content"))
         ),
+        part_index=payload.get("part_index"),
+        part_count=payload.get("part_count"),
+        context_labels=tuple(payload.get("context_labels") or ()),
     )
 
 
@@ -180,6 +193,59 @@ class QdrantRetriever:
         return [hit for _, hit in hits]
 
 
+    def search_lexical(self, query: str, drug_id: str, limit: int) -> list[SearchHit]:
+        """Keyword/BM25-style candidates across ALL of one drug's sections,
+        ranked by term overlap with `query`.
+
+        Two live callers, both in `rag.service.RetrievalService`: (1) inside
+        the deterministic `_section_hits` route, to find a NEIGHBOUR section
+        whose text lexically matches strongly enough to pool in alongside
+        the one the keyword route resolved (a "thận trọng" question can have
+        its real answer only in "chống chỉ định" — see `_pooled_neighbour_
+        hits`'s docstring); (2) available for hybrid fusion with `search`'s
+        dense results (`rag.fusion.reciprocal_rank_fusion`) in the
+        similarity-fallback path, for a free-form question naming no
+        section a paraphrase makes the exact-phrase `find_by_indication`-
+        style match miss.
+
+        Qdrant's `text` index tokenizes and matches individual query tokens
+        (OR semantics across a `should` filter — no `min_should_match` needed
+        since the caller fuses ranks, not raw hits). Common short function
+        words are dropped before matching so they don't dilute every result
+        with the same handful of low-signal hits; score is the count of
+        distinct matched tokens, a transparent stand-in for a real BM25 score
+        given no term-frequency/IDF statistics are computed here.
+        """
+        from qdrant_client.models import FieldCondition, Filter, MatchText, MatchValue
+
+        from rag.text import normalize_name
+
+        tokens = sorted(set(normalize_name(query).split()) - _LEXICAL_STOPWORDS)
+        tokens = [token for token in tokens if len(token) >= 2]
+        if not tokens:
+            return []
+
+        points, _ = self._client.scroll(
+            collection_name=self._collection_name,
+            scroll_filter=Filter(
+                must=[FieldCondition(key="drug_id", match=MatchValue(value=drug_id))],
+                should=[FieldCondition(key="text", match=MatchText(text=t)) for t in tokens],
+            ),
+            limit=max(limit * 4, 20),
+            offset=None,
+            with_payload=True,
+        )
+        scored: list[tuple[int, SearchHit]] = []
+        for point in points:
+            payload = dict(point.payload or {})
+            text_normalized = normalize_name(payload.get("text", ""))
+            matched = sum(1 for t in tokens if t in text_normalized.split())
+            if matched == 0:
+                continue
+            scored.append((matched, SearchHit(_document(payload), float(matched))))
+        scored.sort(key=lambda item: -item[0])
+        return [hit for _, hit in scored[:limit]]
+
     def find_by_drug(self, drug_id: str) -> list[SearchHit]:
         """Every prose section of one drug, in book order — the monograph view.