109 lines
4.9 KiB
Python
109 lines
4.9 KiB
Python
from ingestion.extract.models import Span
|
|
from ingestion.segment import assemble
|
|
from ingestion.tables import SHAPE_GRID_2D, SHAPE_SIMPLE, TableRegion, index_by_page
|
|
|
|
|
|
def _span(text, page, y0, *, bold=False, x0=50.0, block=0, line=0, column="left"):
|
|
return Span(
|
|
physical_page=page, printed_page=page + 1, column=column,
|
|
block=block, line=line, span_index=0,
|
|
x0=x0, y0=y0, x1=x0 + len(text) * 4.5, y1=y0 + 10,
|
|
text=text, font="Tiger-Bold" if bold else "Tiger", size=9.5,
|
|
)
|
|
|
|
|
|
def _monograph_spans(extra):
|
|
return [
|
|
_span("PARACETAMOL", 109, 60.0, bold=True),
|
|
_span("Tên chung quốc tế", 109, 80.0, bold=True),
|
|
_span("Paracetamolum.", 109, 92.0),
|
|
_span("Dạng thuốc và hàm lượng", 109, 200.0, bold=True),
|
|
] + extra
|
|
|
|
|
|
def test_table_spans_are_lifted_out_of_section_prose():
|
|
# real measured case: physical page 109's dosage-form table was being
|
|
# concatenated cell by cell into the section body
|
|
# ('Viên nén' + '1' + '1 - 4' + '8 - 12' + 'Viên nang tác' ...)
|
|
spans = _monograph_spans([
|
|
_span("Thuốc dùng đường uống.", 109, 220.0),
|
|
_span("Viên nén", 109, 400.0, block=5),
|
|
_span("1", 109, 400.0, block=5, x0=200.0),
|
|
_span("1 - 4", 109, 400.0, block=5, x0=260.0),
|
|
_span("Sau khi uống hấp thu nhanh.", 109, 600.0, block=9),
|
|
])
|
|
# the region must cover the table's first column too — it starts at the
|
|
# left margin, same x as body prose
|
|
region = TableRegion("p109_t0", 109, (40.0, 380.0, 400.0, 460.0), 3, 3, SHAPE_SIMPLE)
|
|
m = list(assemble(spans, table_index=index_by_page([region])))[0]
|
|
|
|
body = m.sections["dang_thuoc_va_ham_luong"].text
|
|
assert "Viên nén" not in body
|
|
assert "1 - 4" not in body
|
|
assert "Thuốc dùng đường uống." in body
|
|
assert "Sau khi uống hấp thu nhanh." in body
|
|
|
|
assert len(m.tables) == 1
|
|
block = m.tables[0]
|
|
assert block.table_id == "p109_t0"
|
|
assert "Viên nén" in block.text and "1 - 4" in block.text
|
|
assert block.section_key == "dang_thuoc_va_ham_luong"
|
|
assert block.physical_page == 109
|
|
# every multi-column table is quarantined until a real row/column
|
|
# reconstruction exists — its linearised text is not safe to cite as prose
|
|
assert block.quarantined is True
|
|
|
|
|
|
def test_without_a_region_map_behaviour_is_unchanged():
|
|
spans = _monograph_spans([
|
|
_span("Thuốc dùng đường uống.", 109, 220.0),
|
|
_span("Viên nén", 109, 400.0, block=5),
|
|
])
|
|
m = list(assemble(spans))[0]
|
|
assert m.tables == []
|
|
assert "Viên nén" in m.sections["dang_thuoc_va_ham_luong"].text
|
|
|
|
|
|
def test_2d_grid_block_is_quarantined():
|
|
# a 2D lookup grid's flattened text is meaningless without row/column
|
|
# headers (outlier item 7) — it must be marked, not silently embedded
|
|
spans = _monograph_spans([_span("0,52", 109, 400.0, block=5, x0=200.0)])
|
|
region = TableRegion("p109_t1", 109, (150.0, 380.0, 400.0, 460.0), 6, 5, SHAPE_GRID_2D)
|
|
m = list(assemble(spans, table_index=index_by_page([region])))[0]
|
|
assert len(m.tables) == 1
|
|
assert m.tables[0].quarantined is True
|
|
|
|
|
|
def test_non_table_regions_are_never_lifted():
|
|
# the 17 full-page false positives must not swallow a whole page of prose
|
|
spans = _monograph_spans([_span("Thuốc dùng đường uống.", 109, 220.0)])
|
|
region = TableRegion("p109_t0", 109, (0.0, 0.0, 595.3, 836.2), 1, 2,
|
|
"not_a_table_full_page")
|
|
m = list(assemble(spans, table_index=index_by_page([region])))[0]
|
|
assert m.tables == []
|
|
assert "Thuốc dùng đường uống." in m.sections["dang_thuoc_va_ham_luong"].text
|
|
|
|
|
|
def test_table_block_ids_stay_unique_when_a_section_resumes():
|
|
# a region flushed twice (section closes, then resumes) must not emit two
|
|
# blocks with the same table_id — provenance ids have to be unique
|
|
spans = [
|
|
_span("CEFAMANDOL", 339, 60.0, bold=True),
|
|
_span("Tên chung quốc tế", 339, 80.0, bold=True),
|
|
_span("Cefamandolum.", 339, 92.0),
|
|
_span("Liều lượng và cách dùng", 339, 200.0, bold=True),
|
|
_span("80 - 50", 339, 400.0, block=5),
|
|
_span("Liều lượng và cách dùng", 339, 500.0, bold=True),
|
|
_span("< 25 - 10", 339, 600.0, block=9),
|
|
]
|
|
region = TableRegion("p339_t0", 339, (40.0, 380.0, 400.0, 620.0), 5, 2, SHAPE_SIMPLE)
|
|
m = list(assemble(spans, table_index=index_by_page([region])))[0]
|
|
# table_id is deterministic per REGION, so two parts of one table share
|
|
# it on purpose; table_part_id is the unique key, derived from the first
|
|
# source span rather than a counter (a counter would renumber whenever
|
|
# anything upstream shifted, hiding rather than identifying a duplicate)
|
|
assert len({t.table_part_id for t in m.tables}) == len(m.tables)
|
|
assert {t.continuation_group for t in m.tables} == {"p339_t0"}
|
|
assert all(t.table_part_id.startswith("p339_t0@") for t in m.tables)
|
|
assert all(t.quarantined for t in m.tables)
|