125 lines
4.6 KiB
Python
125 lines
4.6 KiB
Python
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from ingestion.extract import OutlinedTextRun
|
|
from ingestion.tables import TableRegion
|
|
from ingestion.validation import (
|
|
FRACTION_BAR_CANDIDATE,
|
|
HEADER_RULE,
|
|
RULE_FRAGMENT,
|
|
TABLE_FRAME,
|
|
TEXT_AS_VECTOR_OUTLINE,
|
|
UNCLASSIFIED,
|
|
PageContext,
|
|
ResidualRegion,
|
|
classify,
|
|
scan_page,
|
|
)
|
|
from ingestion.validation.residual_ink import FRACTION_BAR_CANDIDATE as BAR
|
|
|
|
PDF_PATH = Path(__file__).resolve().parents[1] / "data" / "raw" / (
|
|
"duoc-thu-quoc-gia-viet-nam-2018.pdf"
|
|
)
|
|
needs_pdf = pytest.mark.skipif(not PDF_PATH.exists(), reason="source PDF not present")
|
|
|
|
|
|
def _region(x0, y0, x1, y1, page=100, ink=500):
|
|
return ResidualRegion(physical_page=page, bbox=(x0, y0, x1, y1), ink_px=ink)
|
|
|
|
|
|
def test_running_header_rule_is_named_not_left_unclassified():
|
|
# measured on real pages: a ~516pt wide, 0pt tall rule at y≈48-52 appears
|
|
# on essentially every page of the book
|
|
assert classify(_region(36.0, 48.5, 552.0, 48.5)) == HEADER_RULE
|
|
|
|
|
|
def test_a_thin_bar_below_the_header_band_is_a_fraction_bar_candidate():
|
|
# NETILMICIN, physical page 1042: the Cockcroft-Gault fraction bar
|
|
assert classify(_region(97.9, 492.0, 286.5, 492.0)) == FRACTION_BAR_CANDIDATE
|
|
|
|
|
|
def test_ink_inside_a_known_table_region_is_a_table_frame_not_a_formula():
|
|
table = TableRegion(
|
|
table_id="p202_t0", physical_page=202, bbox=(299.0, 189.6, 552.4, 300.5),
|
|
n_rows=4, n_cols=3, shape="simple_table",
|
|
)
|
|
region = _region(299.0, 189.6, 552.4, 300.5, page=202)
|
|
assert classify(region, PageContext(tables=[table])) == TABLE_FRAME
|
|
# ...and the same geometry with no table map degrades to "look at it",
|
|
# never to a silent pass
|
|
assert classify(region) == UNCLASSIFIED
|
|
|
|
|
|
def test_a_wide_rule_outside_the_header_band_is_not_treated_as_a_header_rule():
|
|
assert classify(_region(36.0, 700.0, 552.0, 700.0)) == FRACTION_BAR_CANDIDATE
|
|
|
|
|
|
def test_a_tall_block_of_unaccounted_ink_stays_unclassified():
|
|
# a figure or an image of text must never be silently absorbed by a rule
|
|
assert classify(_region(100.0, 300.0, 400.0, 500.0)) == UNCLASSIFIED
|
|
|
|
|
|
def test_hairline_shorter_than_the_minimum_bar_width_is_a_rule_fragment():
|
|
# too short to be a fraction bar, too thin to be anything but a rule
|
|
assert classify(_region(100.0, 300.0, 105.0, 300.0)) == RULE_FRAGMENT
|
|
|
|
|
|
@needs_pdf
|
|
@pytest.mark.parametrize(
|
|
"page,expected_bar_width_pt",
|
|
[
|
|
(1042, 188.6), # NETILMICIN — Cockcroft-Gault
|
|
(202, 118.1), # AMPICILIN VÀ SULBACTAM — Cockcroft-Gault
|
|
],
|
|
)
|
|
def test_confirmed_2d_formula_bars_survive_the_span_mask(page, expected_bar_width_pt):
|
|
"""Regression fixture for the two visually confirmed corrupted formulas.
|
|
|
|
Both pages are reported as having zero tables by `pdfplumber` and zero by
|
|
`opendataloader-pdf`; the bar is only findable as ink. If the mask padding
|
|
is ever loosened again the bar disappears (at 1.0pt page 1042's bar
|
|
shrinks from 188.6pt to 9.1pt) — this test is what catches that.
|
|
"""
|
|
import fitz
|
|
|
|
doc = fitz.open(PDF_PATH)
|
|
bars = [
|
|
r for r in scan_page(doc[page])
|
|
if classify(r) == BAR and r.bbox[1] > 60.0
|
|
]
|
|
assert bars, f"no fraction-bar candidate found on physical page {page}"
|
|
assert max(b.width_pt for b in bars) == pytest.approx(expected_bar_width_pt, abs=1.0)
|
|
|
|
|
|
def test_vector_outlined_text_is_named_rather_than_left_unclassified():
|
|
# physical page 714 prints 17 lines of Gatifloxacin prose as filled paths;
|
|
# no text extractor returns them, so the gate must name the defect
|
|
line = OutlinedTextRun(
|
|
physical_page=714, bbox=(35.3, 75.8, 286.7, 84.4), path_items=1638,
|
|
)
|
|
region = _region(35.5, 76.0, 120.0, 84.0, page=714)
|
|
context = PageContext(outlined_runs=[line])
|
|
assert classify(region, context) == TEXT_AS_VECTOR_OUTLINE
|
|
# an untranscribed line must never be mistaken for recovered content
|
|
assert not line.is_transcribed
|
|
|
|
|
|
@needs_pdf
|
|
def test_outlined_text_lines_are_found_on_exactly_the_five_known_pages():
|
|
"""Whole-document regression: 51 outlined runs on 5 pages.
|
|
|
|
Cross-checked two ways at the time of writing — the drawing-shape scan
|
|
below, and independently by counting glyph-shaped leftovers in the
|
|
residual-ink mask, which found the same five pages.
|
|
"""
|
|
import fitz
|
|
|
|
from ingestion.extract import detect_outlined_text
|
|
|
|
lines = list(detect_outlined_text(fitz.open(PDF_PATH)))
|
|
by_page = {}
|
|
for line in lines:
|
|
by_page[line.physical_page] = by_page.get(line.physical_page, 0) + 1
|
|
assert by_page == {714: 31, 736: 16, 1373: 1, 1444: 1, 1445: 2}
|