Wire the guarded conversational RAG answer layer end-to-end

2026-08-05 14:33:13 +07:00
parent 834d9e51b0
commit ef08b4929e
127 changed files with 37921 additions and 169 deletions
@@ -30,7 +30,7 @@ from typing import Iterator, List, Optional, Union
 from ..extract.models import Span
 from ..extract.page_map import HEADER_BAND_Y
 from ..normalize import join_spans, substitute_pua
-from ..tables.classify import QUARANTINE_SHAPES
+from ..tables.classify import QUARANTINE_SHAPES, SHAPE_FORMULA_2D
 from .atc import extract_atc_codes
 from .detector import in_monograph_range, is_monograph_title_candidate
 from .merge import merge_multiline_headings, merge_same_line_bold_fragments
@@ -51,6 +51,9 @@ from .vocab import (
 )

 _QUALIFIER_RE = re.compile(r"^\(.+\)$")
+_DOSING_TABLE_CAPTION_RE = re.compile(
+    r"(?:^|\b)Bảng\s+\d+\s*[.:]\s*Điều chỉnh liều\b", re.IGNORECASE,
+)


 def _is_page_boilerplate(span: Span) -> bool:
@@ -94,6 +97,31 @@ def _slugify(text: str) -> str:
    return re.sub(r"[^a-z0-9]+", "_", ascii_text.lower()).strip("_")


+def _starts_its_visual_line(span: Span, previous: Span | None) -> bool:
+    """True when nothing else was printed to the left of this span on its line.
+
+    A real section heading opens a line. Confirmed content loss when this was
+    not checked: CISPLATIN (physical page 402) prints `Suy thận: Chống chỉ
+    định.` inside its dosing section, and the second half is a section name.
+    Matched as a heading, `Chống chỉ định.` vanished from the dosing text and
+    the section ended on a bare `Suy thận:` — a renal-impairment
+    contraindication silently dropped. ISOPRENALIN had the same shape.
+
+    "To the left" is decided on `x0`, not on `previous.x1 <= span.x0`: NEVIRAPIN
+    (physical page 1045) prints `Xem thêm mục ` at x1=104.89 immediately before
+    an italic `Liều lượng và cách dùng` at x0=104.88 — a 0.01pt overlap from the
+    trailing space's advance width, enough to make an end-before-start test call
+    a mid-line cross-reference a heading. Comparing the left edges cannot be
+    defeated by glyph-advance rounding and still reads False for the synthetic
+    fixtures, which place every span at identical coordinates.
+    """
+    if previous is None:
+        return True
+    same_line = (previous.physical_page, previous.block, previous.line) == (
+        span.physical_page, span.block, span.line)
+    return not (same_line and previous.x0 < span.x0)
+
+
 def _is_body_line_that_reads_like_a_label(span: Span, items: List) -> bool:
    """A plain line that repeats a section name, sitting under a heading.

@@ -114,7 +142,48 @@ def _is_body_line_that_reads_like_a_label(span: Span, items: List) -> bool:
    return bool(items) and isinstance(items[-1], _SectionEvent)


-def _classify(spans: List[Span]) -> List[Union[Span, _SectionEvent, _TextEvent]]:
+def _is_mid_line_label(span: Span, previous: Span | None) -> bool:
+    """A non-bold section name printed part-way along a line is body text."""
+    return not span.bold and not _starts_its_visual_line(span, previous)
+
+
+def _continues_previous_visual_line(span: Span, previous: Span | None) -> bool:
+    """A plain label-shaped span can be the wrapped tail of body prose.
+
+    Confirmed across the corpus: sentences such as ``không phải là`` /
+    ``chống chỉ định.`` remain in the same PDF block on adjacent visual
+    lines.  Exact vocabulary matching used to consume the second line as a
+    heading.  A real plain heading observed in this book follows completed
+    prose; a non-bold adjacent continuation after an unterminated line does
+    not.  Styling and PDF block geometry make this deliberately narrower than
+    a text-only language heuristic.
+    """
+    if span.bold or previous is None:
+        return False
+    same_block = (previous.physical_page, previous.block) == (
+        span.physical_page, span.block)
+    adjacent_line = span.line == previous.line + 1
+    previous_text = previous.text.rstrip()
+    terminal = previous_text.endswith((".", "!", "?", ":", ";"))
+    return same_block and adjacent_line and bool(previous_text) and not terminal
+
+
+def _is_italic_cross_reference(span: Span) -> bool:
+    """The book italicises references to other sections; headings are never italic.
+
+    Measured over the whole monograph range (physical 99-1496): 11,916 spans
+    carrying a section name are bold, 29 are plain, and **7 are italic — none
+    of them a heading**. Two of the seven wrap onto a line of their own, where
+    the "opens its line" test cannot help: CALCI LACTAT (p296) breaks `xem thêm
+    về nhu cầu hàng ngày trong mục ` / `Dược lý và cơ chế tác dụng)`, and
+    physical page 432 breaks ordinary prose — `hoặc khi không được ` /
+    `chỉ định.` — across a line, so `chỉ định.` alone opened a spurious
+    "Chỉ định" section.
+    """
+    return span.italic
+
+
+def _classify(spans: List[Span], table_index=None) -> List[Union[Span, _SectionEvent, _TextEvent]]:
    """Pass 1: tag each span. Title candidates are left as raw Span objects
    (pass 2 groups + merges them); everything else becomes a typed event.

@@ -127,29 +196,110 @@ def _classify(spans: List[Span]) -> List[Union[Span, _SectionEvent, _TextEvent]]
    same lesson as "don't gate on font size" (ADR 0003 item 10) applied to
    boldness instead.
    """
+    # PyMuPDF's block order is not guaranteed to keep every cell of a table
+    # together.  A visually later section heading can therefore occur between
+    # cells of one physical region in the extracted stream (confirmed on
+    # CAPECITABIN pp. 308-309 and IMATINIB p. 795).  Gather each region first,
+    # then emit it atomically at its first occurrence.  Besides preserving the
+    # section active at the top of the table, this guarantees one run/block per
+    # physical region instead of duplicate fragments with conflicting owners.
+    region_spans: dict[int, List[Span]] = {}
+    region_for_span: dict[int, object] = {}
+    spans_by_page: dict[int, List[Span]] = {}
+    for span in spans:
+        spans_by_page.setdefault(span.physical_page, []).append(span)
+        region = _region_for(table_index, span)
+        if region is None:
+            continue
+        region_for_span[id(span)] = region
+        region_spans.setdefault(id(region), []).append(span)
+
+    # A table can continue after the monograph's ordinary final headings.
+    # CAPECITABIN p. 309 does exactly that: two dose-adjustment tables follow
+    # the trade-name line, with their own explicit captions but without a
+    # repeated "Liều lượng và cách dùng" heading.  Use only the narrow,
+    # unambiguous caption signal and its geometry; a generic keyword rule
+    # would be unsafe in clinical prose.
+    caption_for_region: dict[int, Span] = {}
+    region_for_caption: dict[int, object] = {}
+    for page_regions in (table_index or {}).values():
+        for region in page_regions:
+            candidates = [
+                span for span in spans_by_page.get(region.physical_page, ())
+                if 0 <= region.bbox[1] - span.y1 <= 80
+                and _DOSING_TABLE_CAPTION_RE.search(span.text.strip())
+            ]
+            if candidates:
+                caption = max(candidates, key=lambda candidate: candidate.y1)
+                caption_for_region[id(region)] = caption
+                region_for_caption[id(caption)] = region
+
    items: List[Union[Span, _SectionEvent, _TextEvent]] = []
+    emitted_regions: set[int] = set()
+    emitted_captions: set[int] = set()
+    previous: Span | None = None
    for span in spans:
        if not span.text.strip():
            continue
        if _is_page_boilerplate(span):
+            previous = span
+            continue
+        caption_region = region_for_caption.get(id(span))
+        if caption_region is not None:
+            caption_key = id(span)
+            if caption_key not in emitted_captions:
+                dose_section = match_section("Liều lượng và cách dùng")
+                assert dose_section is not None
+                items.append(_SectionEvent(dose_section, span, inline_value=span.text))
+                emitted_captions.add(caption_key)
+            previous = span
+            continue
+        # Out-of-scope index text must never create section events, and cells
+        # inside a known table are content even when a cell says "Chỉ định".
+        # Classification previously happened without either context, letting
+        # the WARFARIN/IOBITRIDOL table header change the owning section.
+        region = region_for_span.get(id(span))
+        if region is not None:
+            region_key = id(region)
+            if region_key not in emitted_regions:
+                caption = caption_for_region.get(region_key)
+                if caption is not None and id(caption) not in emitted_captions:
+                    dose_section = match_section("Liều lượng và cách dùng")
+                    assert dose_section is not None
+                    items.append(_SectionEvent(dose_section, caption,
+                                               inline_value=caption.text))
+                    emitted_captions.add(id(caption))
+                items.extend(_TextEvent(cell) for cell in region_spans[region_key]
+                             if cell.text.strip() and not _is_page_boilerplate(cell))
+                emitted_regions.add(region_key)
+            previous = span
+            continue
+        if not in_monograph_range(span):
+            items.append(_TextEvent(span))
+            previous = span
            continue
        if is_monograph_title_candidate(span):
            items.append(span)
+            previous = span
            continue
        section_def = match_section(span.text)
-        if section_def is not None and not _is_body_line_that_reads_like_a_label(
-            span, items
-        ):
-            items.append(_SectionEvent(section_def, span))
-            continue
        if section_def is not None:
-            items.append(_TextEvent(span))
+            is_body = (_is_body_line_that_reads_like_a_label(span, items)
+                       or _is_mid_line_label(span, previous)
+                       or _continues_previous_visual_line(span, previous)
+                       or _is_italic_cross_reference(span))
+            items.append(_TextEvent(span) if is_body
+                         else _SectionEvent(section_def, span))
+            previous = span
            continue
        inline = match_section_with_inline_value(span.text)
-        if inline is not None:
+        if (inline is not None and not _is_mid_line_label(span, previous)
+                and not _continues_previous_visual_line(span, previous)
+                and not _is_italic_cross_reference(span)):
            items.append(_SectionEvent(inline[0], span, inline_value=inline[1]))
        else:
            items.append(_TextEvent(span))
+        previous = span
    return items


@@ -228,6 +378,17 @@ def _region_for(table_index, span: Span):
    if not table_index:
        return None
    for region in table_index.get(span.physical_page, ()):
+        if region.shape == SHAPE_FORMULA_2D and span.column in ("left", "right"):
+            # Formula bands are intentionally widened enough to reach past
+            # the gutter (some numerator spans have misleading leading-space
+            # boxes), so geometry alone can swallow prose from the opposite
+            # column.  Keep the wide band but require its source column to
+            # agree with the span's extracted block column.  This preserves
+            # AMPICILIN's gutter-adjacent "Cl" while excluding NETILMICIN's
+            # right-column cross-reference from a left-column formula.
+            region_column = "left" if (region.bbox[0] + region.bbox[2]) / 2 < 303.5 else "right"
+            if span.column != region_column:
+                continue
        if region.contains(span.x0, span.y0, span.x1, span.y1):
            return region
    return None
@@ -259,7 +420,7 @@ def assemble(spans: List[Span], table_index=None, ledger: Optional[list] = None)
    """
    raw_chars = sum(len(s.text) for s in spans)
    spans = merge_same_line_bold_fragments(spans)
-    events = _filter_false_positive_titles(_coalesce_titles(_classify(spans)))
+    events = _filter_false_positive_titles(_coalesce_titles(_classify(spans, table_index)))

    # Span-level coverage ledger. Character counts alone cannot balance here
    # (normalization joins, substitutes and drops characters), so every span
@@ -291,7 +452,7 @@ def assemble(spans: List[Span], table_index=None, ledger: Optional[list] = None)
    current: Optional[Monograph] = None
    current_section_key: Optional[str] = None
    runs: List[tuple] = []          # ordered [(region_or_None, [spans])]
-    inline_prefix: str = ""
+    inline_prefix: Optional[tuple[str, Span]] = None
    awaiting_qualifier = False

    def append_span(span: Span, region):
@@ -338,10 +499,12 @@ def assemble(spans: List[Span], table_index=None, ledger: Optional[list] = None)
                    if region is not None else False,
                ))
        if inline_prefix:
+            inline_text, inline_span = inline_prefix
            head = SectionPart(
-                kind=PART_PROSE, text=inline_prefix,
-                physical_page=parts[0].physical_page if parts else 0,
-                bbox=parts[0].bbox if parts else [0.0, 0.0, 0.0, 0.0],
+                kind=PART_PROSE, text=inline_text,
+                physical_page=inline_span.physical_page,
+                bbox=[inline_span.x0, inline_span.y0, inline_span.x1, inline_span.y1],
+                source_span_ids=[inline_span.span_id],
            )
            parts.insert(0, head)
        return parts
@@ -387,7 +550,7 @@ def assemble(spans: List[Span], table_index=None, ledger: Optional[list] = None)
                        source_span_ids=list(part.source_span_ids),
                    ))
        runs = []
-        inline_prefix = ""
+        inline_prefix = None

    def finalize(monograph: Monograph) -> Monograph:
        # Duplicate check happens here, not at title-detection time: the
@@ -401,8 +564,14 @@ def assemble(spans: List[Span], table_index=None, ledger: Optional[list] = None)
                f"line (outlier item 18) before assuming this is a real collision"
            )
        seen_ids.add(monograph.drug_id)
-        if "ma_atc" in monograph.sections:
-            result = extract_atc_codes(monograph.sections["ma_atc"].text)
+        atc_source = monograph.sections.get("ma_atc")
+        # One real class monograph combines the two labels as "Tên chung
+        # quốc tế và mã ATC".  It remains the required title anchor, and
+        # its ATC codes are still extracted rather than silently discarded.
+        if atc_source is None:
+            atc_source = monograph.sections.get("ten_chung_quoc_te")
+        if atc_source is not None:
+            result = extract_atc_codes(atc_source.text)
            monograph.atc_codes = result.codes
            monograph.atc_stated_absent = result.stated_absent
        return monograph
@@ -440,7 +609,7 @@ def assemble(spans: List[Span], table_index=None, ledger: Optional[list] = None)
                    text="",
                )
            if event.inline_value:
-                inline_prefix = event.inline_value
+                inline_prefix = (event.inline_value, event.span)
            awaiting_qualifier = False
            mark(event.span, SPAN_STATE_HEADING)
            current.source_page_range[1] = max(current.source_page_range[1], event.span.physical_page)