96 lines
3.4 KiB
Python
96 lines
3.4 KiB
Python
from ingestion.extract.models import Span
|
||
from ingestion.normalize import (
|
||
PUA_SUBSTITUTIONS,
|
||
find_unmapped_pua,
|
||
group_visual_lines,
|
||
join_spans,
|
||
substitute_pua,
|
||
)
|
||
|
||
|
||
def _span(text, *, page=100, block=0, line=0, index=0, x0=50.0, x1=None, y0=100.0):
|
||
return Span(
|
||
physical_page=page, printed_page=page + 1, column="left",
|
||
block=block, line=line, span_index=index,
|
||
x0=x0, y0=y0, x1=(x0 + len(text) * 4.5) if x1 is None else x1, y1=y0 + 10,
|
||
text=text, font="Tiger", size=9.5,
|
||
)
|
||
|
||
|
||
def test_pua_map_covers_every_codepoint_confirmed_in_the_corpus():
|
||
# all 8 were located in the source PDF, rendered, and read visually —
|
||
# see docs/progress-log.md for the page each was confirmed on
|
||
assert PUA_SUBSTITUTIONS[""] == "≥"
|
||
assert PUA_SUBSTITUTIONS[""] == "≤"
|
||
assert PUA_SUBSTITUTIONS[""] == "α"
|
||
assert PUA_SUBSTITUTIONS[""] == "→"
|
||
assert PUA_SUBSTITUTIONS[""] == "®"
|
||
assert PUA_SUBSTITUTIONS[""] == "₁"
|
||
assert PUA_SUBSTITUTIONS[""] == "↓"
|
||
assert PUA_SUBSTITUTIONS[""] == "γ"
|
||
|
||
|
||
def test_comparison_operators_in_real_dosing_sentences_are_restored():
|
||
# the clinically dangerous case: without this, "liều ≤ 100 mg" reaches
|
||
# embeddings as "liều 100 mg" and the operator is lost
|
||
assert substitute_pua("trẻ em 10 tuổi") == "trẻ em ≥ 10 tuổi"
|
||
assert substitute_pua("liều 100 mg") == "liều ≤ 100 mg"
|
||
|
||
|
||
def test_unmapped_pua_is_reported_not_silently_passed_through():
|
||
assert find_unmapped_pua("liều 100 mg") == []
|
||
assert find_unmapped_pua("bất ngờ đây") == [""]
|
||
|
||
|
||
def test_subscript_span_rejoins_without_a_spurious_space():
|
||
# real corpus case: "cytochrom P450" arrived as "cytochrom P\n450\ngây"
|
||
spans = [
|
||
_span("cytochrom P", x0=50.0, x1=100.0),
|
||
_span("450", x0=100.2, x1=110.0),
|
||
_span(" gây chuyển hóa.", x0=110.1, x1=180.0),
|
||
]
|
||
assert join_spans(spans) == "cytochrom P450 gây chuyển hóa."
|
||
|
||
|
||
def test_italic_run_inside_parentheses_rejoins_on_one_line():
|
||
# real corpus case: "(\nfeline immunodeficiency virus\n)"
|
||
spans = [
|
||
_span("(", x0=50.0, x1=53.0),
|
||
_span("feline immunodeficiency virus", x0=53.1, x1=180.0),
|
||
_span(")", x0=180.1, x1=183.0),
|
||
]
|
||
assert join_spans(spans) == "(feline immunodeficiency virus)"
|
||
|
||
|
||
def test_wrap_without_sentence_end_is_joined_with_a_space():
|
||
spans = [
|
||
_span("không nhai. Nếu", line=0, y0=100.0),
|
||
_span("uống viên thuốc", line=1, y0=112.0),
|
||
]
|
||
assert join_spans(spans) == "không nhai. Nếu uống viên thuốc"
|
||
|
||
|
||
def test_sentence_end_keeps_the_line_break():
|
||
spans = [
|
||
_span("Liều người lớn: 10 mg.", line=0, y0=100.0),
|
||
_span("Trẻ em: 5 mg.", line=1, y0=112.0),
|
||
]
|
||
assert join_spans(spans) == "Liều người lớn: 10 mg.\nTrẻ em: 5 mg."
|
||
|
||
|
||
def test_wide_gap_on_one_line_still_yields_a_space():
|
||
spans = [
|
||
_span("Người bệnh", x0=50.0, x1=100.0),
|
||
_span("100 kg", x0=104.0, x1=130.0),
|
||
]
|
||
assert join_spans(spans) == "Người bệnh 100 kg"
|
||
|
||
|
||
def test_visual_lines_group_by_pymupdf_block_and_line_indices():
|
||
spans = [
|
||
_span("a", block=0, line=0), _span("b", block=0, line=0),
|
||
_span("c", block=0, line=1),
|
||
_span("d", block=1, line=0),
|
||
]
|
||
assert [len(g) for g in group_visual_lines(spans)] == [2, 1, 1]
|