Fix every real lint finding and drop degenerate splice fragments

2026-08-01 13:51:38 +07:00
parent 967b917001
commit 834d9e51b0
69 changed files with 10119 additions and 39 deletions
@@ -0,0 +1,134 @@
+from ingestion.extract.models import Span
+from ingestion.segment.merge import merge_multiline_headings, merge_same_line_bold_fragments
+
+
+def _span(text, page, y0, size=9.5, font="TimesNewRomanPS-BoldMT"):
+    return Span(
+        physical_page=page, printed_page=page + 1, column="right",
+        block=0, line=0, span_index=0,
+        x0=100.0, y0=y0, x1=200.0, y1=y0 + 12.0,
+        text=text, font=font, size=size,
+    )
+
+
+def test_confirmed_gonadotropin_wrap_merges_into_one_heading():
+    # exact bboxes from physical page 1371 (0-indexed) — see module docstring
+    candidates = [
+        _span("THUỐC TƯƠNG TỰ HORMON GIẢI PHÓNG", 1371, 664.4554443359375),
+        _span("GONADOTROPIN", 1371, 676.2354736328125),
+    ]
+    headings = list(merge_multiline_headings(candidates))
+    assert len(headings) == 1
+    assert headings[0].text == "THUỐC TƯƠNG TỰ HORMON GIẢI PHÓNG GONADOTROPIN"
+
+
+def test_unrelated_single_line_titles_on_different_pages_not_merged():
+    candidates = [
+        _span("GONADOTROPIN", 755, 200.0),
+        _span("HYDROCORTISON", 900, 300.0),
+    ]
+    headings = list(merge_multiline_headings(candidates))
+    assert len(headings) == 2
+    assert [h.text for h in headings] == ["GONADOTROPIN", "HYDROCORTISON"]
+
+
+def test_large_y_gap_on_same_page_not_merged():
+    # two genuinely separate single-line titles far apart on the same page
+    # (e.g. two short monographs stacked in one column) must not merge
+    candidates = [
+        _span("ATENOLOL", 219, 100.0),
+        _span("ATRACURIUM BESYLAT", 219, 500.0),
+    ]
+    headings = list(merge_multiline_headings(candidates))
+    assert len(headings) == 2
+
+
+def test_confirmed_aciclovir_same_line_split_merges_without_space():
+    # exact bboxes from physical page 113 (0-indexed), found by rendering the
+    # page to an image and reading it directly: "ACIC" (size 10.0) and
+    # "LOVIR" (size 9.5) are one word split into two spans on the same
+    # visual line — different font size, ~0.5pt y0 gap, near-zero x-gap.
+    # Must merge WITHOUT a space ("ACICLOVIR", not "ACIC LOVIR") — see
+    # module docstring.
+    candidates = [
+        _span("ACIC", 113, 515.1914672851562, size=10.0),
+        _span("LOVIR", 113, 515.7044677734375, size=9.5),
+    ]
+    headings = list(merge_multiline_headings(candidates))
+    assert len(headings) == 1
+    assert headings[0].text == "ACICLOVIR"
+
+
+def test_wrap_and_same_line_split_use_different_join_characters():
+    # a genuine line-wrap (large y-gap) still joins with a space even when
+    # font size differs, since size is no longer part of the merge decision
+    candidates = [
+        _span("FIRST LINE", 100, 200.0, size=10.0),
+        _span("SECOND LINE", 100, 212.0, size=9.5),
+    ]
+    headings = list(merge_multiline_headings(candidates))
+    assert len(headings) == 1
+    assert headings[0].text == "FIRST LINE SECOND LINE"
+
+
+def test_single_candidate_yields_one_heading():
+    headings = list(merge_multiline_headings([_span("ABACAVIR", 100, 60.29)]))
+    assert len(headings) == 1
+    assert headings[0].text == "ABACAVIR"
+
+
+def test_empty_input_yields_nothing():
+    assert list(merge_multiline_headings([])) == []
+
+
+def test_confirmed_ten_chung_quoc_te_diacritic_split_reassembles():
+    # exact fragments + y0 from physical page 759's "GUAIFENESIN" monograph,
+    # found via a whole-book `cli validate` run (the monograph was silently
+    # dropped because "Tên chung quốc tế" never matched the section
+    # vocabulary) and confirmed by rendering the page to an image: to a
+    # human reader the line looks completely normal, but PyMuPDF splits it
+    # into 5 spans around the diacritic characters — see module docstring.
+    fragments = [
+        _span("Tên chung qu", 759, 157.614),
+        _span("ố", 759, 157.33),
+        _span("c t", 759, 157.614),
+        _span("ế", 759, 157.33),
+        _span(": ", 759, 157.614),
+    ]
+    merged = merge_same_line_bold_fragments(fragments)
+    assert len(merged) == 1
+    assert merged[0].text == "Tên chung quốc tế: "
+
+
+def test_non_bold_spans_pass_through_unmerged():
+    fragments = [
+        _span("Guaifenesin", 759, 157.24, font="TimesNewRomanPSMT"),
+        _span(".", 759, 157.24, font="TimesNewRomanPSMT"),
+    ]
+    merged = merge_same_line_bold_fragments(fragments)
+    assert len(merged) == 2
+
+
+def test_bold_spans_on_different_lines_not_merged():
+    fragments = [_span("Chỉ định", 100, 200.0), _span("Chống chỉ định", 100, 220.0)]
+    merged = merge_same_line_bold_fragments(fragments)
+    assert len(merged) == 2
+
+
+def test_merged_span_keeps_provenance_of_first_fragment():
+    fragments = [_span("Tên chung qu", 759, 157.614), _span("ố", 759, 157.33)]
+    merged = merge_same_line_bold_fragments(fragments)
+    assert merged[0].physical_page == 759
+    assert merged[0].printed_page == 760
+    assert merged[0].x0 == fragments[0].x0
+    assert merged[0].x1 == fragments[-1].x1
+
+
+def test_single_bold_span_passes_through_unchanged():
+    fragments = [_span("ABACAVIR", 100, 60.29)]
+    merged = merge_same_line_bold_fragments(fragments)
+    assert merged == fragments
+
+
+def test_empty_input_to_same_line_merge_yields_nothing():
+    assert merge_same_line_bold_fragments([]) == []