178 lines
8.1 KiB
Python
178 lines
8.1 KiB
Python
from ingestion.extract.models import Span
|
|
from ingestion.segment import assemble
|
|
from ingestion.tables import (
|
|
SHAPE_FORMULA_2D,
|
|
SHAPE_GRID_2D,
|
|
SHAPE_SIMPLE,
|
|
TableRegion,
|
|
index_by_page,
|
|
)
|
|
|
|
|
|
def _span(text, page, y0, *, bold=False, x0=50.0, block=0, line=0, column="left"):
|
|
return Span(
|
|
physical_page=page, printed_page=page + 1, column=column,
|
|
block=block, line=line, span_index=0,
|
|
x0=x0, y0=y0, x1=x0 + len(text) * 4.5, y1=y0 + 10,
|
|
text=text, font="Tiger-Bold" if bold else "Tiger", size=9.5,
|
|
)
|
|
|
|
|
|
def _monograph_spans(extra):
|
|
return [
|
|
_span("PARACETAMOL", 109, 60.0, bold=True),
|
|
_span("Tên chung quốc tế", 109, 80.0, bold=True),
|
|
_span("Paracetamolum.", 109, 92.0),
|
|
_span("Dạng thuốc và hàm lượng", 109, 200.0, bold=True),
|
|
] + extra
|
|
|
|
|
|
def test_table_spans_are_lifted_out_of_section_prose():
|
|
# real measured case: physical page 109's dosage-form table was being
|
|
# concatenated cell by cell into the section body
|
|
# ('Viên nén' + '1' + '1 - 4' + '8 - 12' + 'Viên nang tác' ...)
|
|
spans = _monograph_spans([
|
|
_span("Thuốc dùng đường uống.", 109, 220.0),
|
|
_span("Viên nén", 109, 400.0, block=5),
|
|
_span("1", 109, 400.0, block=5, x0=200.0),
|
|
_span("1 - 4", 109, 400.0, block=5, x0=260.0),
|
|
_span("Sau khi uống hấp thu nhanh.", 109, 600.0, block=9),
|
|
])
|
|
# the region must cover the table's first column too — it starts at the
|
|
# left margin, same x as body prose
|
|
region = TableRegion("p109_t0", 109, (40.0, 380.0, 400.0, 460.0), 3, 3, SHAPE_SIMPLE)
|
|
m = list(assemble(spans, table_index=index_by_page([region])))[0]
|
|
|
|
body = m.sections["dang_thuoc_va_ham_luong"].text
|
|
assert "Viên nén" not in body
|
|
assert "1 - 4" not in body
|
|
assert "Thuốc dùng đường uống." in body
|
|
assert "Sau khi uống hấp thu nhanh." in body
|
|
|
|
assert len(m.tables) == 1
|
|
block = m.tables[0]
|
|
assert block.table_id == "p109_t0"
|
|
assert "Viên nén" in block.text and "1 - 4" in block.text
|
|
assert block.section_key == "dang_thuoc_va_ham_luong"
|
|
assert block.physical_page == 109
|
|
# every multi-column table is quarantined until a real row/column
|
|
# reconstruction exists — its linearised text is not safe to cite as prose
|
|
assert block.quarantined is True
|
|
|
|
|
|
def test_section_named_table_cell_does_not_change_owning_section():
|
|
# Confirmed in WARFARIN p1485 and IOBITRIDOL p826: a table column named
|
|
# "Chỉ định" belongs to the dosing table; it is not a document
|
|
# section heading and must not move the block into chi_dinh.
|
|
spans = [
|
|
_span("WARFARIN", 1485, 60.0, bold=True),
|
|
_span("Tên chung quốc tế", 1485, 80.0, bold=True),
|
|
_span("Warfarinum.", 1485, 92.0),
|
|
_span("Liều lượng và cách dùng", 1485, 200.0, bold=True),
|
|
_span("Chỉ định", 1485, 400.0, bold=True, block=5),
|
|
_span("INR 2,0 - 3,0", 1485, 412.0, block=5),
|
|
]
|
|
region = TableRegion("p1485_t0", 1485, (40.0, 380.0, 400.0, 460.0), 2, 2, SHAPE_SIMPLE)
|
|
m = list(assemble(spans, table_index=index_by_page([region])))[0]
|
|
assert "chi_dinh" not in m.sections
|
|
assert len(m.tables) == 1
|
|
assert m.tables[0].section_key == "lieu_luong_va_cach_dung"
|
|
|
|
|
|
def test_wide_formula_band_does_not_swallow_the_opposite_column():
|
|
spans = _monograph_spans([
|
|
_span("Công thức:", 109, 360.0),
|
|
_span("Cl", 109, 400.0, x0=280.0, column="left", block=5),
|
|
_span("Xem thêm Liều lượng và cách dùng", 109, 400.0,
|
|
x0=310.0, column="right", block=6),
|
|
])
|
|
# Deliberately extends across the gutter, as verified formula bands do.
|
|
region = TableRegion("p109_f0", 109, (40.0, 380.0, 390.0, 430.0),
|
|
2, 1, SHAPE_FORMULA_2D)
|
|
m = list(assemble(spans, table_index=index_by_page([region])))[0]
|
|
assert m.tables[0].text == "Cl"
|
|
assert "Xem thêm Liều lượng và cách dùng" in m.sections["dang_thuoc_va_ham_luong"].text
|
|
|
|
|
|
def test_without_a_region_map_behaviour_is_unchanged():
|
|
spans = _monograph_spans([
|
|
_span("Thuốc dùng đường uống.", 109, 220.0),
|
|
_span("Viên nén", 109, 400.0, block=5),
|
|
])
|
|
m = list(assemble(spans))[0]
|
|
assert m.tables == []
|
|
assert "Viên nén" in m.sections["dang_thuoc_va_ham_luong"].text
|
|
|
|
|
|
def test_2d_grid_block_is_quarantined():
|
|
# a 2D lookup grid's flattened text is meaningless without row/column
|
|
# headers (outlier item 7) — it must be marked, not silently embedded
|
|
spans = _monograph_spans([_span("0,52", 109, 400.0, block=5, x0=200.0)])
|
|
region = TableRegion("p109_t1", 109, (150.0, 380.0, 400.0, 460.0), 6, 5, SHAPE_GRID_2D)
|
|
m = list(assemble(spans, table_index=index_by_page([region])))[0]
|
|
assert len(m.tables) == 1
|
|
assert m.tables[0].quarantined is True
|
|
|
|
|
|
def test_non_table_regions_are_never_lifted():
|
|
# the 17 full-page false positives must not swallow a whole page of prose
|
|
spans = _monograph_spans([_span("Thuốc dùng đường uống.", 109, 220.0)])
|
|
region = TableRegion("p109_t0", 109, (0.0, 0.0, 595.3, 836.2), 1, 2,
|
|
"not_a_table_full_page")
|
|
m = list(assemble(spans, table_index=index_by_page([region])))[0]
|
|
assert m.tables == []
|
|
assert "Thuốc dùng đường uống." in m.sections["dang_thuoc_va_ham_luong"].text
|
|
|
|
|
|
def test_table_block_ids_stay_unique_when_a_section_resumes():
|
|
# Confirmed on CAPECITABIN pp. 308-309 and IMATINIB p. 795: PDF block
|
|
# order can place a visually later heading between cells from one physical
|
|
# table. The complete region must stay atomic and owned by the section
|
|
# active where the table first appears.
|
|
spans = [
|
|
_span("CEFAMANDOL", 339, 60.0, bold=True),
|
|
_span("Tên chung quốc tế", 339, 80.0, bold=True),
|
|
_span("Cefamandolum.", 339, 92.0),
|
|
_span("Liều lượng và cách dùng", 339, 200.0, bold=True),
|
|
_span("80 - 50", 339, 400.0, block=5),
|
|
# Visually below the table, but emitted before its final cell by the
|
|
# PDF's internal block order.
|
|
_span("Tương tác thuốc", 339, 640.0, bold=True),
|
|
_span("< 25 - 10", 339, 600.0, block=9),
|
|
_span("Không phối hợp với thuốc X.", 339, 660.0, block=10),
|
|
]
|
|
region = TableRegion("p339_t0", 339, (40.0, 380.0, 400.0, 620.0), 5, 2, SHAPE_SIMPLE)
|
|
m = list(assemble(spans, table_index=index_by_page([region])))[0]
|
|
assert len(m.tables) == 1
|
|
assert m.tables[0].section_key == "lieu_luong_va_cach_dung"
|
|
assert "80 - 50" in m.tables[0].text
|
|
assert "< 25 - 10" in m.tables[0].text
|
|
assert "Không phối hợp với thuốc X." in m.sections["tuong_tac_thuoc"].text
|
|
assert len({t.table_part_id for t in m.tables}) == len(m.tables)
|
|
assert {t.continuation_group for t in m.tables} == {"p339_t0"}
|
|
assert all(t.table_part_id.startswith("p339_t0@") for t in m.tables)
|
|
assert all(t.quarantined for t in m.tables)
|
|
|
|
|
|
def test_explicit_dose_adjustment_caption_reassigns_late_appendix_table():
|
|
# CAPECITABIN p. 309: the PDF puts dose-adjustment tables after the trade
|
|
# names and does not repeat the ordinary dosage section heading. Internal
|
|
# block order can even emit a cell before the visually preceding caption.
|
|
spans = [
|
|
_span("CAPECITABIN", 309, 40.0, bold=True),
|
|
_span("Tên chung quốc tế", 309, 50.0, bold=True),
|
|
_span("Capecitabinum.", 309, 60.0),
|
|
_span("Tên thương mại", 309, 70.0, bold=True),
|
|
_span("Xeloda.", 309, 80.0),
|
|
_span("Mức độ theo NCIC", 309, 120.0, block=5),
|
|
_span("Bảng 3. Điều chỉnh liều do độc tính.", 309, 100.0),
|
|
_span("Ngừng thuốc cho đến khi về mức 0.", 309, 140.0, block=5),
|
|
]
|
|
region = TableRegion("p309_t0", 309, (40.0, 115.0, 400.0, 180.0),
|
|
3, 4, SHAPE_SIMPLE)
|
|
m = list(assemble(spans, table_index=index_by_page([region])))[0]
|
|
assert len(m.tables) == 1
|
|
assert m.tables[0].section_key == "lieu_luong_va_cach_dung"
|
|
dosage = m.sections["lieu_luong_va_cach_dung"].text
|
|
assert dosage.count("Bảng 3. Điều chỉnh liều do độc tính.") == 1
|