from ingestion.extract.models import Span from ingestion.segment import assemble from ingestion.tables import SHAPE_GRID_2D, SHAPE_SIMPLE, TableRegion, index_by_page def _span(text, page, y0, *, bold=False, x0=50.0, block=0, line=0, column="left"): return Span( physical_page=page, printed_page=page + 1, column=column, block=block, line=line, span_index=0, x0=x0, y0=y0, x1=x0 + len(text) * 4.5, y1=y0 + 10, text=text, font="Tiger-Bold" if bold else "Tiger", size=9.5, ) def _monograph_spans(extra): return [ _span("PARACETAMOL", 109, 60.0, bold=True), _span("Tên chung quốc tế", 109, 80.0, bold=True), _span("Paracetamolum.", 109, 92.0), _span("Dạng thuốc và hàm lượng", 109, 200.0, bold=True), ] + extra def test_table_spans_are_lifted_out_of_section_prose(): # real measured case: physical page 109's dosage-form table was being # concatenated cell by cell into the section body # ('Viên nén' + '1' + '1 - 4' + '8 - 12' + 'Viên nang tác' ...) spans = _monograph_spans([ _span("Thuốc dùng đường uống.", 109, 220.0), _span("Viên nén", 109, 400.0, block=5), _span("1", 109, 400.0, block=5, x0=200.0), _span("1 - 4", 109, 400.0, block=5, x0=260.0), _span("Sau khi uống hấp thu nhanh.", 109, 600.0, block=9), ]) # the region must cover the table's first column too — it starts at the # left margin, same x as body prose region = TableRegion("p109_t0", 109, (40.0, 380.0, 400.0, 460.0), 3, 3, SHAPE_SIMPLE) m = list(assemble(spans, table_index=index_by_page([region])))[0] body = m.sections["dang_thuoc_va_ham_luong"].text assert "Viên nén" not in body assert "1 - 4" not in body assert "Thuốc dùng đường uống." in body assert "Sau khi uống hấp thu nhanh." in body assert len(m.tables) == 1 block = m.tables[0] assert block.table_id == "p109_t0" assert "Viên nén" in block.text and "1 - 4" in block.text assert block.section_key == "dang_thuoc_va_ham_luong" assert block.physical_page == 109 # every multi-column table is quarantined until a real row/column # reconstruction exists — its linearised text is not safe to cite as prose assert block.quarantined is True def test_without_a_region_map_behaviour_is_unchanged(): spans = _monograph_spans([ _span("Thuốc dùng đường uống.", 109, 220.0), _span("Viên nén", 109, 400.0, block=5), ]) m = list(assemble(spans))[0] assert m.tables == [] assert "Viên nén" in m.sections["dang_thuoc_va_ham_luong"].text def test_2d_grid_block_is_quarantined(): # a 2D lookup grid's flattened text is meaningless without row/column # headers (outlier item 7) — it must be marked, not silently embedded spans = _monograph_spans([_span("0,52", 109, 400.0, block=5, x0=200.0)]) region = TableRegion("p109_t1", 109, (150.0, 380.0, 400.0, 460.0), 6, 5, SHAPE_GRID_2D) m = list(assemble(spans, table_index=index_by_page([region])))[0] assert len(m.tables) == 1 assert m.tables[0].quarantined is True def test_non_table_regions_are_never_lifted(): # the 17 full-page false positives must not swallow a whole page of prose spans = _monograph_spans([_span("Thuốc dùng đường uống.", 109, 220.0)]) region = TableRegion("p109_t0", 109, (0.0, 0.0, 595.3, 836.2), 1, 2, "not_a_table_full_page") m = list(assemble(spans, table_index=index_by_page([region])))[0] assert m.tables == [] assert "Thuốc dùng đường uống." in m.sections["dang_thuoc_va_ham_luong"].text def test_table_block_ids_stay_unique_when_a_section_resumes(): # a region flushed twice (section closes, then resumes) must not emit two # blocks with the same table_id — provenance ids have to be unique spans = [ _span("CEFAMANDOL", 339, 60.0, bold=True), _span("Tên chung quốc tế", 339, 80.0, bold=True), _span("Cefamandolum.", 339, 92.0), _span("Liều lượng và cách dùng", 339, 200.0, bold=True), _span("80 - 50", 339, 400.0, block=5), _span("Liều lượng và cách dùng", 339, 500.0, bold=True), _span("< 25 - 10", 339, 600.0, block=9), ] region = TableRegion("p339_t0", 339, (40.0, 380.0, 400.0, 620.0), 5, 2, SHAPE_SIMPLE) m = list(assemble(spans, table_index=index_by_page([region])))[0] # table_id is deterministic per REGION, so two parts of one table share # it on purpose; table_part_id is the unique key, derived from the first # source span rather than a counter (a counter would renumber whenever # anything upstream shifted, hiding rather than identifying a duplicate) assert len({t.table_part_id for t in m.tables}) == len(m.tables) assert {t.continuation_group for t in m.tables} == {"p339_t0"} assert all(t.table_part_id.startswith("p339_t0@") for t in m.tables) assert all(t.quarantined for t in m.tables)