179 lines
8.6 KiB
Python
179 lines
8.6 KiB
Python
"""Mandatory glyph/reading-order sanity check.
|
|
|
|
Two distinct defect shapes were confirmed by testing this module against the
|
|
real PDF (not assumed from the ADR description alone):
|
|
|
|
1. **Within-span glyph reversal** (`scan_glyph_order` / `GlyphOrderIssue`):
|
|
physical page 1373 (0-indexed) contains a span whose characters are
|
|
positioned in strictly decreasing x-origin order, producing scrambled
|
|
text (e.g. "= tịx 8 y..." instead of "y 8 xịt ="). Matches ADR 0003's
|
|
original description.
|
|
|
|
2. **Cross-span row misordering within one PyMuPDF block**
|
|
(`scan_reading_order` / `ReadingOrderIssue`) — a genuinely different,
|
|
previously undocumented shape found while testing this module end-to-end:
|
|
physical page 714 has a visual text row split into multiple PyMuPDF line
|
|
objects, within a single `block`, that are emitted out of left-to-right
|
|
order relative to each other (each individual span's own characters are
|
|
fine, but the fragments interleave incorrectly), e.g. the row "...bảo
|
|
quản nhiệt độ..." is emitted as fragments "quản ", " ộ", "đ tệih", "n " in
|
|
that (wrong) order. Concatenating characters in raw extraction order
|
|
produces garbled text; re-sorting the *same* characters within one visual
|
|
row by x-origin recovers the correct reading order exactly. This means
|
|
ADR 0003's "exactly 1 occurrence in the whole book" claim was based on a
|
|
narrower (within-span-only) check and undercounted the real defect
|
|
population — corrected here, see docs/pdf-parsing-outlier-catalog.md
|
|
item 9 update.
|
|
|
|
**Getting the row-grouping key right took three iterations, each caught by
|
|
running against the real book rather than trusting the first result (per
|
|
CLAUDE.md's no-fabrication rule) — recorded here since the failure modes
|
|
generalize to any from-scratch "reconstruct visual rows from raw
|
|
coordinates" approach:**
|
|
- v1 (group by rounded y only): 1113 "issues", almost all false positives.
|
|
- v2 (group by (`_column_for_x` tag, rounded y), using the same ±20pt
|
|
tolerance `extract/spans.py` uses for informational span tagging): dropped
|
|
to 32, but a real false-positive class remained — kerning jitter (e.g.
|
|
"mefloquin"'s 'l'/'o' origins differ by only 0.095pt, well inside normal
|
|
font kerning) was treated as a reversal with no decrease tolerance, and
|
|
the ±20pt column tolerance creates an *overlapping* accepted x-range for
|
|
"left" (24-319) and "right" (288-582) — a right-column paragraph
|
|
starting near x=299 was misclassified "left" and merged with an unrelated
|
|
left-column line sharing the same y.
|
|
- v3 (this version — group by (PyMuPDF's own `block` index, rounded y)):
|
|
the real fix. Two paragraphs from genuinely different columns (e.g. page
|
|
1104: one block starting at x=299.4, another at x=35.4, both at y=70.4)
|
|
turned out to sit in **different PyMuPDF blocks**, while page 714's 3
|
|
genuinely-misordered fragments sit in the **same block** (block 20) split
|
|
across multiple `line` entries. Block identity — PyMuPDF's own layout
|
|
analysis, already validated in ADR 0003 to respect this document's
|
|
two-column structure — is a reliable discriminator that no fixed
|
|
x-coordinate threshold can be, since real paragraph start positions vary
|
|
enough to overlap any hand-picked column boundary. A minimum-decrease
|
|
threshold (`_MIN_DECREASE_PT`, well above observed kerning jitter <0.3pt
|
|
and well below observed real defects >2pt) still guards against sub-pixel
|
|
jitter within a block/row. The header band (running page number + drug
|
|
name, two unrelated boilerplate fields sharing a y-coordinate — stripped
|
|
before chunking regardless, outlier-catalog item 13) is excluded outright.
|
|
|
|
Both checks are cheap (seconds per full-book pass) and must run over 100% of
|
|
pages, not sampled, per ADR 0003's standing rigor bar.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from collections import defaultdict
|
|
from dataclasses import dataclass
|
|
from typing import Dict, List, Sequence, Tuple
|
|
|
|
import fitz
|
|
|
|
_ROW_Y_PRECISION = 1 # decimal places; same-baseline chars share y to <0.01pt in practice
|
|
_HEADER_BAND_Y = 60.0 # page number + running drug name live here; boilerplate, stripped separately
|
|
_MIN_DECREASE_PT = 1.0 # observed kerning jitter <0.3pt; observed real defects >2pt — safely between
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class GlyphOrderIssue:
|
|
physical_page: int
|
|
span_bbox: Tuple[float, float, float, float]
|
|
original_text: str
|
|
corrected_text: str
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ReadingOrderIssue:
|
|
physical_page: int
|
|
block_index: int
|
|
row_y: float
|
|
extracted_text: str
|
|
corrected_text: str
|
|
|
|
|
|
def is_reversed_order(x_origins: Sequence[float]) -> bool:
|
|
"""True if every consecutive pair strictly decreases in x — the exact
|
|
shape of the confirmed within-span defect. A normal LTR span's
|
|
x-origins strictly increase; requiring *every* pair to decrease (not
|
|
just "not sorted") avoids false-triggering on ordinary spans.
|
|
"""
|
|
if len(x_origins) < 2:
|
|
return False
|
|
# strict=False on purpose: this is the adjacent-pair idiom, so the two
|
|
# sequences differ in length by one by construction.
|
|
return all(b < a for a, b in zip(x_origins, x_origins[1:], strict=False))
|
|
|
|
|
|
def scan_glyph_order(doc: fitz.Document) -> List[GlyphOrderIssue]:
|
|
issues: List[GlyphOrderIssue] = []
|
|
for pno in range(doc.page_count):
|
|
for block in doc[pno].get_text("rawdict").get("blocks", []):
|
|
for line in block.get("lines", []):
|
|
for span in line.get("spans", []):
|
|
chars = span.get("chars", [])
|
|
if not chars:
|
|
continue
|
|
x_origins = [c["origin"][0] for c in chars]
|
|
if is_reversed_order(x_origins):
|
|
issues.append(GlyphOrderIssue(
|
|
physical_page=pno,
|
|
span_bbox=tuple(span["bbox"]),
|
|
original_text="".join(c["c"] for c in chars),
|
|
corrected_text="".join(c["c"] for c in reversed(chars)),
|
|
))
|
|
return issues
|
|
|
|
|
|
def _has_significant_backward_jump(xs: Sequence[float], min_decrease: float) -> bool:
|
|
return any(b < a - min_decrease for a, b in zip(xs, xs[1:], strict=False))
|
|
|
|
|
|
def find_reading_order_issues(
|
|
chars_by_row: Dict[Tuple[int, float], List[Tuple[float, str]]],
|
|
min_decrease: float = _MIN_DECREASE_PT,
|
|
) -> List[ReadingOrderIssue]:
|
|
"""Pure logic, unit-testable without a real PDF: given characters already
|
|
grouped by (block_index, row_y) in raw extraction order, flag a row only
|
|
when it contains a backward x-jump larger than `min_decrease` — ordinary
|
|
font kerning produces sub-0.3pt jitter (see module docstring's v2
|
|
entry), so a plain "resorting changes the text" check without this
|
|
threshold is not reliable; it self-corrupts already-correct text.
|
|
Grouping by block index (not a hand-picked x-coordinate column
|
|
boundary) is what the caller must guarantee — see module docstring's
|
|
v1/v2/v3 history for why a coordinate-based row reconstruction alone is
|
|
not safe.
|
|
"""
|
|
issues = []
|
|
for (block_index, y), chars in chars_by_row.items():
|
|
if len(chars) < 2:
|
|
continue
|
|
xs = [x for x, _ in chars]
|
|
if not _has_significant_backward_jump(xs, min_decrease):
|
|
continue
|
|
extracted = "".join(c for _, c in chars)
|
|
corrected = "".join(c for _, c in sorted(chars, key=lambda t: t[0]))
|
|
if extracted != corrected:
|
|
issues.append(ReadingOrderIssue(
|
|
physical_page=-1, block_index=block_index, row_y=y,
|
|
extracted_text=extracted, corrected_text=corrected,
|
|
))
|
|
return issues
|
|
|
|
|
|
def scan_reading_order(doc: fitz.Document) -> List[ReadingOrderIssue]:
|
|
issues: List[ReadingOrderIssue] = []
|
|
for pno in range(doc.page_count):
|
|
rows: Dict[Tuple[int, float], List[Tuple[float, str]]] = defaultdict(list)
|
|
for block_index, block in enumerate(doc[pno].get_text("rawdict").get("blocks", [])):
|
|
for line in block.get("lines", []):
|
|
for span in line.get("spans", []):
|
|
for c in span.get("chars", []):
|
|
x, y = c["origin"]
|
|
if y < _HEADER_BAND_Y:
|
|
continue
|
|
rows[(block_index, round(y, _ROW_Y_PRECISION))].append((x, c["c"]))
|
|
for issue in find_reading_order_issues(rows):
|
|
issues.append(ReadingOrderIssue(
|
|
physical_page=pno, block_index=issue.block_index, row_y=issue.row_y,
|
|
extracted_text=issue.extracted_text, corrected_text=issue.corrected_text,
|
|
))
|
|
return issues
|