99 lines
4.0 KiB
Python
99 lines
4.0 KiB
Python
"""Monograph and section boundary detection.
|
|
|
|
Validated signal (ADR 0003): monograph titles are bold + all-caps + short
|
|
line length, scoped to printed pages 99-1496 — font **size** is explicitly
|
|
NOT part of the rule (a size>=9.8 threshold silently dropped ~15% of real
|
|
monographs). Section headings are bold spans cross-checked against the
|
|
known (open/extensible) vocabulary in `vocab.py`, no all-caps requirement
|
|
(most section headings, e.g. "Chỉ định", are not all-caps).
|
|
|
|
Known false positive, explicitly excluded rather than tuned around (outlier
|
|
item 12d): "CÁC CHUYÊN LUẬN THUỐC" and other part-divider titles sit exactly
|
|
at the printed-page-99 boundary and are bold + all-caps + short, identical
|
|
in shape to a real monograph title.
|
|
|
|
"All-caps" itself is not 100% reliable either (confirmed real, outlier item
|
|
21): the class-level monograph "CÁC CHẤT ỨC CHẾ HMG-CoA REDUCTASE" embeds
|
|
the mixed-case abbreviation "CoA" (Coenzyme A) — a strict `text.isupper()`
|
|
check silently dropped this entire monograph. `_is_mostly_upper` tolerates
|
|
a small number of lowercase letters (a strict superset of `isupper()`, so
|
|
no previously-valid case is excluded) rather than requiring zero.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from typing import Iterator, List
|
|
|
|
from ..extract.models import Span
|
|
from .merge import merge_multiline_headings
|
|
from .models import Heading
|
|
from .vocab import is_part_divider, match_section
|
|
|
|
MONOGRAPH_PRINTED_PAGE_START = 99
|
|
MONOGRAPH_PRINTED_PAGE_END = 1496
|
|
# Printed folios are metadata inferred from page headers and can be wrong in
|
|
# the back index (confirmed: physical page 1655 was inferred as printed page
|
|
# 1496). The physical bounds are source-document invariants and prevent an
|
|
# index entry that looks like a section label from extending ZOLPIDEM by 161
|
|
# pages. Keep both checks: either signal alone has known failure modes.
|
|
MONOGRAPH_PHYSICAL_PAGE_START = 99
|
|
MONOGRAPH_PHYSICAL_PAGE_END = 1496
|
|
_MIN_TITLE_LEN = 3
|
|
_MAX_TITLE_LEN = 60
|
|
_MAX_LOWERCASE_RATIO = 0.10 # HMG-CoA: 1/27 = 3.7% (real title) vs "Mã ATC:": 1/5 = 20% (real
|
|
# section label, correctly rejected) — a ratio, not an absolute count, is what separates a
|
|
# long title with one embedded mixed-case abbreviation from a short label with a normal
|
|
# lowercase diacritic (found via a real regression: an earlier absolute-count version of
|
|
# this check let "Mã ATC:" through as a false title candidate).
|
|
|
|
|
|
def _is_mostly_upper(text: str) -> bool:
|
|
letters = [c for c in text if c.isalpha()]
|
|
if not letters:
|
|
return False
|
|
lowercase_ratio = sum(1 for c in letters if c.islower()) / len(letters)
|
|
return lowercase_ratio <= _MAX_LOWERCASE_RATIO
|
|
|
|
|
|
def in_monograph_range(span: Span) -> bool:
|
|
return (
|
|
MONOGRAPH_PHYSICAL_PAGE_START <= span.physical_page <= MONOGRAPH_PHYSICAL_PAGE_END
|
|
and
|
|
span.printed_page is not None
|
|
and MONOGRAPH_PRINTED_PAGE_START <= span.printed_page <= MONOGRAPH_PRINTED_PAGE_END
|
|
)
|
|
|
|
|
|
def is_monograph_title_candidate(span: Span) -> bool:
|
|
text = span.text.strip()
|
|
if not (span.bold and _is_mostly_upper(text)):
|
|
return False
|
|
if not (_MIN_TITLE_LEN <= len(text) <= _MAX_TITLE_LEN):
|
|
return False
|
|
if not in_monograph_range(span):
|
|
return False
|
|
if is_part_divider(text):
|
|
return False
|
|
return True
|
|
|
|
|
|
def detect_monograph_titles(spans: List[Span]) -> Iterator[Heading]:
|
|
"""`spans` must be in reading order (as `extract_spans` yields them)."""
|
|
candidates = [s for s in spans if is_monograph_title_candidate(s)]
|
|
yield from merge_multiline_headings(candidates)
|
|
|
|
|
|
def detect_section_headings(spans: List[Span]) -> Iterator[Heading]:
|
|
for span in spans:
|
|
if not span.bold or not in_monograph_range(span):
|
|
continue
|
|
section_def = match_section(span.text)
|
|
if section_def is None:
|
|
continue
|
|
yield Heading(
|
|
text=section_def.display_name,
|
|
physical_page=span.physical_page,
|
|
y0=span.y0,
|
|
is_monograph_title=False,
|
|
section_key=section_def.key,
|
|
)
|