from pathlib import Path import pytest from ingestion.extract import OutlinedTextRun from ingestion.tables import TableRegion from ingestion.validation import ( FRACTION_BAR_CANDIDATE, HEADER_RULE, RULE_FRAGMENT, TABLE_FRAME, TEXT_AS_VECTOR_OUTLINE, UNCLASSIFIED, PageContext, ResidualRegion, classify, scan_page, ) from ingestion.validation.residual_ink import FRACTION_BAR_CANDIDATE as BAR PDF_PATH = Path(__file__).resolve().parents[1] / "data" / "raw" / ( "duoc-thu-quoc-gia-viet-nam-2018.pdf" ) needs_pdf = pytest.mark.skipif(not PDF_PATH.exists(), reason="source PDF not present") def _region(x0, y0, x1, y1, page=100, ink=500): return ResidualRegion(physical_page=page, bbox=(x0, y0, x1, y1), ink_px=ink) def test_running_header_rule_is_named_not_left_unclassified(): # measured on real pages: a ~516pt wide, 0pt tall rule at y≈48-52 appears # on essentially every page of the book assert classify(_region(36.0, 48.5, 552.0, 48.5)) == HEADER_RULE def test_a_thin_bar_below_the_header_band_is_a_fraction_bar_candidate(): # NETILMICIN, physical page 1042: the Cockcroft-Gault fraction bar assert classify(_region(97.9, 492.0, 286.5, 492.0)) == FRACTION_BAR_CANDIDATE def test_ink_inside_a_known_table_region_is_a_table_frame_not_a_formula(): table = TableRegion( table_id="p202_t0", physical_page=202, bbox=(299.0, 189.6, 552.4, 300.5), n_rows=4, n_cols=3, shape="simple_table", ) region = _region(299.0, 189.6, 552.4, 300.5, page=202) assert classify(region, PageContext(tables=[table])) == TABLE_FRAME # ...and the same geometry with no table map degrades to "look at it", # never to a silent pass assert classify(region) == UNCLASSIFIED def test_a_wide_rule_outside_the_header_band_is_not_treated_as_a_header_rule(): assert classify(_region(36.0, 700.0, 552.0, 700.0)) == FRACTION_BAR_CANDIDATE def test_a_tall_block_of_unaccounted_ink_stays_unclassified(): # a figure or an image of text must never be silently absorbed by a rule assert classify(_region(100.0, 300.0, 400.0, 500.0)) == UNCLASSIFIED def test_hairline_shorter_than_the_minimum_bar_width_is_a_rule_fragment(): # too short to be a fraction bar, too thin to be anything but a rule assert classify(_region(100.0, 300.0, 105.0, 300.0)) == RULE_FRAGMENT @needs_pdf @pytest.mark.parametrize( "page,expected_bar_width_pt", [ (1042, 188.6), # NETILMICIN — Cockcroft-Gault (202, 118.1), # AMPICILIN VÀ SULBACTAM — Cockcroft-Gault ], ) def test_confirmed_2d_formula_bars_survive_the_span_mask(page, expected_bar_width_pt): """Regression fixture for the two visually confirmed corrupted formulas. Both pages are reported as having zero tables by `pdfplumber` and zero by `opendataloader-pdf`; the bar is only findable as ink. If the mask padding is ever loosened again the bar disappears (at 1.0pt page 1042's bar shrinks from 188.6pt to 9.1pt) — this test is what catches that. """ import fitz doc = fitz.open(PDF_PATH) bars = [ r for r in scan_page(doc[page]) if classify(r) == BAR and r.bbox[1] > 60.0 ] assert bars, f"no fraction-bar candidate found on physical page {page}" assert max(b.width_pt for b in bars) == pytest.approx(expected_bar_width_pt, abs=1.0) def test_vector_outlined_text_is_named_rather_than_left_unclassified(): # physical page 714 prints 17 lines of Gatifloxacin prose as filled paths; # no text extractor returns them, so the gate must name the defect line = OutlinedTextRun( physical_page=714, bbox=(35.3, 75.8, 286.7, 84.4), path_items=1638, ) region = _region(35.5, 76.0, 120.0, 84.0, page=714) context = PageContext(outlined_runs=[line]) assert classify(region, context) == TEXT_AS_VECTOR_OUTLINE # an untranscribed line must never be mistaken for recovered content assert not line.is_transcribed @needs_pdf def test_outlined_text_lines_are_found_on_exactly_the_five_known_pages(): """Whole-document regression: 51 outlined runs on 5 pages. Cross-checked two ways at the time of writing — the drawing-shape scan below, and independently by counting glyph-shaped leftovers in the residual-ink mask, which found the same five pages. """ import fitz from ingestion.extract import detect_outlined_text lines = list(detect_outlined_text(fitz.open(PDF_PATH))) by_page = {} for line in lines: by_page[line.physical_page] = by_page.get(line.physical_page, 0) + 1 assert by_page == {714: 31, 736: 16, 1373: 1, 1444: 1, 1445: 2}