Fix every real lint finding and drop degenerate splice fragments

ingestion/tests/test_validation_residual_ink.py

@@ -0,0 +1,124 @@
+from pathlib import Path
+
+import pytest
+
+from ingestion.extract import OutlinedTextRun
+from ingestion.tables import TableRegion
+from ingestion.validation import (
+    FRACTION_BAR_CANDIDATE,
+    HEADER_RULE,
+    RULE_FRAGMENT,
+    TABLE_FRAME,
+    TEXT_AS_VECTOR_OUTLINE,
+    UNCLASSIFIED,
+    PageContext,
+    ResidualRegion,
+    classify,
+    scan_page,
+)
+from ingestion.validation.residual_ink import FRACTION_BAR_CANDIDATE as BAR
+
+PDF_PATH = Path(__file__).resolve().parents[1] / "data" / "raw" / (
+    "duoc-thu-quoc-gia-viet-nam-2018.pdf"
+)
+needs_pdf = pytest.mark.skipif(not PDF_PATH.exists(), reason="source PDF not present")
+
+
+def _region(x0, y0, x1, y1, page=100, ink=500):
+    return ResidualRegion(physical_page=page, bbox=(x0, y0, x1, y1), ink_px=ink)
+
+
+def test_running_header_rule_is_named_not_left_unclassified():
+    # measured on real pages: a ~516pt wide, 0pt tall rule at y≈48-52 appears
+    # on essentially every page of the book
+    assert classify(_region(36.0, 48.5, 552.0, 48.5)) == HEADER_RULE
+
+
+def test_a_thin_bar_below_the_header_band_is_a_fraction_bar_candidate():
+    # NETILMICIN, physical page 1042: the Cockcroft-Gault fraction bar
+    assert classify(_region(97.9, 492.0, 286.5, 492.0)) == FRACTION_BAR_CANDIDATE
+
+
+def test_ink_inside_a_known_table_region_is_a_table_frame_not_a_formula():
+    table = TableRegion(
+        table_id="p202_t0", physical_page=202, bbox=(299.0, 189.6, 552.4, 300.5),
+        n_rows=4, n_cols=3, shape="simple_table",
+    )
+    region = _region(299.0, 189.6, 552.4, 300.5, page=202)
+    assert classify(region, PageContext(tables=[table])) == TABLE_FRAME
+    # ...and the same geometry with no table map degrades to "look at it",
+    # never to a silent pass
+    assert classify(region) == UNCLASSIFIED
+
+
+def test_a_wide_rule_outside_the_header_band_is_not_treated_as_a_header_rule():
+    assert classify(_region(36.0, 700.0, 552.0, 700.0)) == FRACTION_BAR_CANDIDATE
+
+
+def test_a_tall_block_of_unaccounted_ink_stays_unclassified():
+    # a figure or an image of text must never be silently absorbed by a rule
+    assert classify(_region(100.0, 300.0, 400.0, 500.0)) == UNCLASSIFIED
+
+
+def test_hairline_shorter_than_the_minimum_bar_width_is_a_rule_fragment():
+    # too short to be a fraction bar, too thin to be anything but a rule
+    assert classify(_region(100.0, 300.0, 105.0, 300.0)) == RULE_FRAGMENT
+
+
+@needs_pdf
+@pytest.mark.parametrize(
+    "page,expected_bar_width_pt",
+    [
+        (1042, 188.6),  # NETILMICIN — Cockcroft-Gault
+        (202, 118.1),   # AMPICILIN VÀ SULBACTAM — Cockcroft-Gault
+    ],
+)
+def test_confirmed_2d_formula_bars_survive_the_span_mask(page, expected_bar_width_pt):
+    """Regression fixture for the two visually confirmed corrupted formulas.
+
+    Both pages are reported as having zero tables by `pdfplumber` and zero by
+    `opendataloader-pdf`; the bar is only findable as ink. If the mask padding
+    is ever loosened again the bar disappears (at 1.0pt page 1042's bar
+    shrinks from 188.6pt to 9.1pt) — this test is what catches that.
+    """
+    import fitz
+
+    doc = fitz.open(PDF_PATH)
+    bars = [
+        r for r in scan_page(doc[page])
+        if classify(r) == BAR and r.bbox[1] > 60.0
+    ]
+    assert bars, f"no fraction-bar candidate found on physical page {page}"
+    assert max(b.width_pt for b in bars) == pytest.approx(expected_bar_width_pt, abs=1.0)
+
+
+def test_vector_outlined_text_is_named_rather_than_left_unclassified():
+    # physical page 714 prints 17 lines of Gatifloxacin prose as filled paths;
+    # no text extractor returns them, so the gate must name the defect
+    line = OutlinedTextRun(
+        physical_page=714, bbox=(35.3, 75.8, 286.7, 84.4), path_items=1638,
+    )
+    region = _region(35.5, 76.0, 120.0, 84.0, page=714)
+    context = PageContext(outlined_runs=[line])
+    assert classify(region, context) == TEXT_AS_VECTOR_OUTLINE
+    # an untranscribed line must never be mistaken for recovered content
+    assert not line.is_transcribed
+
+
+@needs_pdf
+def test_outlined_text_lines_are_found_on_exactly_the_five_known_pages():
+    """Whole-document regression: 51 outlined runs on 5 pages.
+
+    Cross-checked two ways at the time of writing — the drawing-shape scan
+    below, and independently by counting glyph-shaped leftovers in the
+    residual-ink mask, which found the same five pages.
+    """
+    import fitz
+
+    from ingestion.extract import detect_outlined_text
+
+    lines = list(detect_outlined_text(fitz.open(PDF_PATH)))
+    by_page = {}
+    for line in lines:
+        by_page[line.physical_page] = by_page.get(line.physical_page, 0) + 1
+    assert by_page == {714: 31, 736: 16, 1373: 1, 1444: 1, 1445: 2}