667 lines
30 KiB
Python
667 lines
30 KiB
Python
"""Assembles a raw span stream into ordered Monograph records.
|
|
|
|
Three simple passes, each independently easy to reason about — avoids a
|
|
single tangled state machine (SRP: classify, then merge titles, then build):
|
|
|
|
1. Classify each span in reading order as a title candidate, a section
|
|
heading, or body text.
|
|
2. Coalesce consecutive title-candidate spans into single merged Heading
|
|
events via `merge.merge_multiline_headings` (handles both the multi-line
|
|
wrap and same-line font-size-split cases — see merge.py).
|
|
3. Walk the resulting flat event stream once, building Monograph records.
|
|
|
|
Handles the confirmed real "qualifier line" case (outlier-catalog item 18):
|
|
a monograph title can legitimately repeat (e.g. two distinct "SALBUTAMOL"
|
|
monographs, "Dùng trong hô hấp" vs "Dùng trong sản khoa") disambiguated by a
|
|
bold, parenthesized, non-all-caps line directly beneath the title. That
|
|
qualifier is folded into `drug_id` so two legitimate entries don't collide;
|
|
a genuine duplicate `drug_id` (no qualifier, same name) raises rather than
|
|
silently overwriting, since the one apparent duplicate found during ADR
|
|
0003's investigation (GONADOTROPIN) turned out to be a detector artifact,
|
|
not real — a real second collision should be surfaced, not hidden.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
import unicodedata
|
|
from dataclasses import dataclass
|
|
from typing import Iterator, List, Optional, Union
|
|
|
|
from ..extract.models import Span
|
|
from ..extract.page_map import HEADER_BAND_Y
|
|
from ..normalize import join_spans, substitute_pua
|
|
from ..tables.classify import QUARANTINE_SHAPES, SHAPE_FORMULA_2D
|
|
from .atc import extract_atc_codes
|
|
from .detector import in_monograph_range, is_monograph_title_candidate
|
|
from .merge import merge_multiline_headings, merge_same_line_bold_fragments
|
|
from .models import (
|
|
PART_PROSE,
|
|
PART_TABLE,
|
|
Heading,
|
|
Monograph,
|
|
SectionPart,
|
|
SectionSpan,
|
|
TableBlock,
|
|
)
|
|
from .vocab import (
|
|
SectionDef,
|
|
is_part_divider,
|
|
match_section,
|
|
match_section_with_inline_value,
|
|
)
|
|
|
|
_QUALIFIER_RE = re.compile(r"^\(.+\)$")
|
|
_DOSING_TABLE_CAPTION_RE = re.compile(
|
|
r"(?:^|\b)Bảng\s+\d+\s*[.:]\s*Điều chỉnh liều\b", re.IGNORECASE,
|
|
)
|
|
|
|
|
|
def _is_page_boilerplate(span: Span) -> bool:
|
|
"""Confirmed real (outlier-catalog item 13, measured via a whole-book
|
|
`assemble()` run): the running header ("DTQGVN 2" + page number +
|
|
current monograph name, e.g. physical page 1008's "DTQGVN 2" / "1009" /
|
|
"Morphin sulfat") was falling through every classification branch below
|
|
into plain body text, since it matches no section heading and isn't a
|
|
real all-caps title — silently splicing itself into the *middle* of
|
|
whatever section happens to be open when a physical page turns (1,374
|
|
of 11,409 sections / 671 of 682 monographs affected). It's reliably
|
|
identifiable independent of its (non-vocabulary) text: always the
|
|
full-page-width block in the header band, same signal `page_map.py`
|
|
already uses to read the folio.
|
|
"""
|
|
return span.column == "full_width" and span.y0 < HEADER_BAND_Y
|
|
|
|
|
|
class DuplicateDrugIdError(ValueError):
|
|
pass
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class _SectionEvent:
|
|
section_def: SectionDef
|
|
span: Span
|
|
inline_value: Optional[str] = None
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class _TextEvent:
|
|
span: Span
|
|
|
|
|
|
_Event = Union[Heading, _SectionEvent, _TextEvent] # Heading == a title event
|
|
|
|
|
|
def _slugify(text: str) -> str:
|
|
# `đ`/`Đ` (U+0111/U+0110) are standalone Vietnamese letters, not a base
|
|
# letter plus a combining mark, so NFKD leaves them whole and the ASCII
|
|
# encode below then discards them silently -- turning `ĐIỆN GIẢI` into
|
|
# `ien_giai`. Exactly three monographs contain `Đ` and all three carried a
|
|
# damaged drug_id because of this: GIẢI ĐỘC TỐ UỐN VÁN, KHÁNG ĐỘC TỐ BẠCH
|
|
# HẦU, and THUỐC UỐNG BÙ NƯỚC VÀ ĐIỆN GIẢI. Found via the adversarial eval
|
|
# suite, where both Oresol cases failed to resolve their monograph.
|
|
#
|
|
# Casefold before replacing so one pass covers both cases: `Đ` casefolds
|
|
# to `đ`. This matches what `entities/catalog.py::normalize_name` already
|
|
# does -- that function got it right and this one did not.
|
|
folded = text.casefold().replace("đ", "d")
|
|
normalized = unicodedata.normalize("NFKD", folded)
|
|
ascii_text = normalized.encode("ascii", "ignore").decode("ascii")
|
|
return re.sub(r"[^a-z0-9]+", "_", ascii_text).strip("_")
|
|
|
|
|
|
def _starts_its_visual_line(span: Span, previous: Span | None) -> bool:
|
|
"""True when nothing else was printed to the left of this span on its line.
|
|
|
|
A real section heading opens a line. Confirmed content loss when this was
|
|
not checked: CISPLATIN (physical page 402) prints `Suy thận: Chống chỉ
|
|
định.` inside its dosing section, and the second half is a section name.
|
|
Matched as a heading, `Chống chỉ định.` vanished from the dosing text and
|
|
the section ended on a bare `Suy thận:` — a renal-impairment
|
|
contraindication silently dropped. ISOPRENALIN had the same shape.
|
|
|
|
"To the left" is decided on `x0`, not on `previous.x1 <= span.x0`: NEVIRAPIN
|
|
(physical page 1045) prints `Xem thêm mục ` at x1=104.89 immediately before
|
|
an italic `Liều lượng và cách dùng` at x0=104.88 — a 0.01pt overlap from the
|
|
trailing space's advance width, enough to make an end-before-start test call
|
|
a mid-line cross-reference a heading. Comparing the left edges cannot be
|
|
defeated by glyph-advance rounding and still reads False for the synthetic
|
|
fixtures, which place every span at identical coordinates.
|
|
"""
|
|
if previous is None:
|
|
return True
|
|
same_line = (previous.physical_page, previous.block, previous.line) == (
|
|
span.physical_page, span.block, span.line)
|
|
return not (same_line and previous.x0 < span.x0)
|
|
|
|
|
|
def _is_body_line_that_reads_like_a_label(span: Span, items: List) -> bool:
|
|
"""A plain line that repeats a section name, sitting under a heading.
|
|
|
|
Confirmed real and clinically material: FLUOROURACIL (physical page 681)
|
|
prints `Thời kỳ mang thai` / `Chống chỉ định.` and `Thời kỳ cho con bú` /
|
|
`Chống chỉ định.`, verified by rendering the page. The body line matches
|
|
the section vocabulary, so it was read as a heading — leaving both
|
|
pregnancy and lactation sections empty and dropping the statement that
|
|
fluorouracil is contraindicated in both.
|
|
|
|
The book never prints an empty section, so a *non-bold* label immediately
|
|
after a heading is that heading's body. Boldness still cannot be required
|
|
in general (outlier item 20: `Mã ATC: N06AA09.` is a plain span), which is
|
|
why this is narrowed to the directly-under-a-heading position.
|
|
"""
|
|
if span.bold:
|
|
return False
|
|
return bool(items) and isinstance(items[-1], _SectionEvent)
|
|
|
|
|
|
def _is_mid_line_label(span: Span, previous: Span | None) -> bool:
|
|
"""A non-bold section name printed part-way along a line is body text."""
|
|
return not span.bold and not _starts_its_visual_line(span, previous)
|
|
|
|
|
|
def _continues_previous_visual_line(span: Span, previous: Span | None) -> bool:
|
|
"""A plain label-shaped span can be the wrapped tail of body prose.
|
|
|
|
Confirmed across the corpus: sentences such as ``không phải là`` /
|
|
``chống chỉ định.`` remain in the same PDF block on adjacent visual
|
|
lines. Exact vocabulary matching used to consume the second line as a
|
|
heading. A real plain heading observed in this book follows completed
|
|
prose; a non-bold adjacent continuation after an unterminated line does
|
|
not. Styling and PDF block geometry make this deliberately narrower than
|
|
a text-only language heuristic.
|
|
"""
|
|
if span.bold or previous is None:
|
|
return False
|
|
same_block = (previous.physical_page, previous.block) == (
|
|
span.physical_page, span.block)
|
|
adjacent_line = span.line == previous.line + 1
|
|
previous_text = previous.text.rstrip()
|
|
terminal = previous_text.endswith((".", "!", "?", ":", ";"))
|
|
return same_block and adjacent_line and bool(previous_text) and not terminal
|
|
|
|
|
|
def _is_italic_cross_reference(span: Span) -> bool:
|
|
"""The book italicises references to other sections; headings are never italic.
|
|
|
|
Measured over the whole monograph range (physical 99-1496): 11,916 spans
|
|
carrying a section name are bold, 29 are plain, and **7 are italic — none
|
|
of them a heading**. Two of the seven wrap onto a line of their own, where
|
|
the "opens its line" test cannot help: CALCI LACTAT (p296) breaks `xem thêm
|
|
về nhu cầu hàng ngày trong mục ` / `Dược lý và cơ chế tác dụng)`, and
|
|
physical page 432 breaks ordinary prose — `hoặc khi không được ` /
|
|
`chỉ định.` — across a line, so `chỉ định.` alone opened a spurious
|
|
"Chỉ định" section.
|
|
"""
|
|
return span.italic
|
|
|
|
|
|
def _classify(spans: List[Span], table_index=None) -> List[Union[Span, _SectionEvent, _TextEvent]]:
|
|
"""Pass 1: tag each span. Title candidates are left as raw Span objects
|
|
(pass 2 groups + merges them); everything else becomes a typed event.
|
|
|
|
Section matching does NOT require `span.bold` — confirmed real (outlier
|
|
item 20): AMITRIPTYLIN's "Mã ATC: N06AA09." is a single **plain, non-bold**
|
|
span (Abacavir's equivalent is bold "Mã ATC: " + a separate plain value
|
|
span), inconsistent across the book's ~700 individually-authored
|
|
monographs (the book's own foreword notes "biên soạn bởi nhiều tác giả").
|
|
Matching by exact vocabulary text (not styling) is the reliable signal,
|
|
same lesson as "don't gate on font size" (ADR 0003 item 10) applied to
|
|
boldness instead.
|
|
"""
|
|
# PyMuPDF's block order is not guaranteed to keep every cell of a table
|
|
# together. A visually later section heading can therefore occur between
|
|
# cells of one physical region in the extracted stream (confirmed on
|
|
# CAPECITABIN pp. 308-309 and IMATINIB p. 795). Gather each region first,
|
|
# then emit it atomically at its first occurrence. Besides preserving the
|
|
# section active at the top of the table, this guarantees one run/block per
|
|
# physical region instead of duplicate fragments with conflicting owners.
|
|
region_spans: dict[int, List[Span]] = {}
|
|
region_for_span: dict[int, object] = {}
|
|
spans_by_page: dict[int, List[Span]] = {}
|
|
for span in spans:
|
|
spans_by_page.setdefault(span.physical_page, []).append(span)
|
|
region = _region_for(table_index, span)
|
|
if region is None:
|
|
continue
|
|
region_for_span[id(span)] = region
|
|
region_spans.setdefault(id(region), []).append(span)
|
|
|
|
# A table can continue after the monograph's ordinary final headings.
|
|
# CAPECITABIN p. 309 does exactly that: two dose-adjustment tables follow
|
|
# the trade-name line, with their own explicit captions but without a
|
|
# repeated "Liều lượng và cách dùng" heading. Use only the narrow,
|
|
# unambiguous caption signal and its geometry; a generic keyword rule
|
|
# would be unsafe in clinical prose.
|
|
caption_for_region: dict[int, Span] = {}
|
|
region_for_caption: dict[int, object] = {}
|
|
for page_regions in (table_index or {}).values():
|
|
for region in page_regions:
|
|
candidates = [
|
|
span for span in spans_by_page.get(region.physical_page, ())
|
|
if 0 <= region.bbox[1] - span.y1 <= 80
|
|
and _DOSING_TABLE_CAPTION_RE.search(span.text.strip())
|
|
]
|
|
if candidates:
|
|
caption = max(candidates, key=lambda candidate: candidate.y1)
|
|
caption_for_region[id(region)] = caption
|
|
region_for_caption[id(caption)] = region
|
|
|
|
items: List[Union[Span, _SectionEvent, _TextEvent]] = []
|
|
emitted_regions: set[int] = set()
|
|
emitted_captions: set[int] = set()
|
|
previous: Span | None = None
|
|
for span in spans:
|
|
if not span.text.strip():
|
|
continue
|
|
if _is_page_boilerplate(span):
|
|
previous = span
|
|
continue
|
|
caption_region = region_for_caption.get(id(span))
|
|
if caption_region is not None:
|
|
caption_key = id(span)
|
|
if caption_key not in emitted_captions:
|
|
dose_section = match_section("Liều lượng và cách dùng")
|
|
assert dose_section is not None
|
|
items.append(_SectionEvent(dose_section, span, inline_value=span.text))
|
|
emitted_captions.add(caption_key)
|
|
previous = span
|
|
continue
|
|
# Out-of-scope index text must never create section events, and cells
|
|
# inside a known table are content even when a cell says "Chỉ định".
|
|
# Classification previously happened without either context, letting
|
|
# the WARFARIN/IOBITRIDOL table header change the owning section.
|
|
region = region_for_span.get(id(span))
|
|
if region is not None:
|
|
region_key = id(region)
|
|
if region_key not in emitted_regions:
|
|
caption = caption_for_region.get(region_key)
|
|
if caption is not None and id(caption) not in emitted_captions:
|
|
dose_section = match_section("Liều lượng và cách dùng")
|
|
assert dose_section is not None
|
|
items.append(_SectionEvent(dose_section, caption,
|
|
inline_value=caption.text))
|
|
emitted_captions.add(id(caption))
|
|
items.extend(_TextEvent(cell) for cell in region_spans[region_key]
|
|
if cell.text.strip() and not _is_page_boilerplate(cell))
|
|
emitted_regions.add(region_key)
|
|
previous = span
|
|
continue
|
|
if not in_monograph_range(span):
|
|
items.append(_TextEvent(span))
|
|
previous = span
|
|
continue
|
|
if is_monograph_title_candidate(span):
|
|
items.append(span)
|
|
previous = span
|
|
continue
|
|
section_def = match_section(span.text)
|
|
if section_def is not None:
|
|
is_body = (_is_body_line_that_reads_like_a_label(span, items)
|
|
or _is_mid_line_label(span, previous)
|
|
or _continues_previous_visual_line(span, previous)
|
|
or _is_italic_cross_reference(span))
|
|
items.append(_TextEvent(span) if is_body
|
|
else _SectionEvent(section_def, span))
|
|
previous = span
|
|
continue
|
|
inline = match_section_with_inline_value(span.text)
|
|
if (inline is not None and not _is_mid_line_label(span, previous)
|
|
and not _continues_previous_visual_line(span, previous)
|
|
and not _is_italic_cross_reference(span)):
|
|
items.append(_SectionEvent(inline[0], span, inline_value=inline[1]))
|
|
else:
|
|
items.append(_TextEvent(span))
|
|
previous = span
|
|
return items
|
|
|
|
|
|
def _coalesce_titles(items: List[Union[Span, _SectionEvent, _TextEvent]]) -> List[_Event]:
|
|
"""Pass 2: merge consecutive raw title-candidate Span runs into single
|
|
Heading events, preserving the order of everything else.
|
|
"""
|
|
events: List[_Event] = []
|
|
run: List[Span] = []
|
|
|
|
def flush_run():
|
|
if run:
|
|
events.extend(merge_multiline_headings(list(run)))
|
|
run.clear()
|
|
|
|
for item in items:
|
|
if isinstance(item, Span):
|
|
run.append(item)
|
|
else:
|
|
flush_run()
|
|
events.append(item)
|
|
flush_run()
|
|
return events
|
|
|
|
|
|
def _is_qualifier_line(span: Span) -> bool:
|
|
text = span.text.strip()
|
|
return span.bold and not text.isupper() and bool(_QUALIFIER_RE.match(text))
|
|
|
|
|
|
_ANCHOR_LOOKAHEAD = 6
|
|
_ANCHOR_SECTION_KEY = "ten_chung_quoc_te"
|
|
|
|
|
|
def _has_anchor_ahead(events: List[_Event], title_index: int) -> bool:
|
|
"""Every real monograph documents "Tên chung quốc tế" as its very first
|
|
section (the book's own template, item 2 — see vocab.py docstring).
|
|
Loosening this to "any known section" was tried and reverted: it let
|
|
a real, different false positive through (outlier item 21) — individual
|
|
statin names ("SIMVASTATIN", "LOVASTATIN", ...) are bold+all-caps+short
|
|
sub-headings *inside* the class-level "CÁC CHẤT ỨC CHẾ HMG-CoA
|
|
REDUCTASE" monograph, each immediately followed by their own "Liều
|
|
lượng và cách dùng" sub-section but NOT by "Tên chung quốc tế" (that
|
|
section belongs only to the parent class monograph) — the loose
|
|
"any section" check couldn't tell this apart from a real monograph
|
|
start, but the strict "Tên chung quốc tế specifically" check correctly
|
|
rejects it, since the specific book-documented template guarantees this
|
|
exact section is always first for genuine top-level monographs.
|
|
|
|
Still correctly rejects the other confirmed false positive (outlier
|
|
item 19: "HSV"/"CMV" table column headers), which aren't followed by
|
|
ANY recognized section, let alone this specific one.
|
|
"""
|
|
for j in range(title_index + 1, min(title_index + 1 + _ANCHOR_LOOKAHEAD, len(events))):
|
|
event = events[j]
|
|
if isinstance(event, Heading) and event.is_monograph_title:
|
|
return False
|
|
if isinstance(event, _SectionEvent) and event.section_def.key == _ANCHOR_SECTION_KEY:
|
|
return True
|
|
return False
|
|
|
|
|
|
def _filter_false_positive_titles(events: List[_Event]) -> List[_Event]:
|
|
"""Pass 2.5: drop title-shaped candidates that aren't followed by any
|
|
recognized section anchor before the next title candidate.
|
|
"""
|
|
return [
|
|
event for i, event in enumerate(events)
|
|
if not (isinstance(event, Heading) and event.is_monograph_title)
|
|
or _has_anchor_ahead(events, i)
|
|
]
|
|
|
|
|
|
def _region_for(table_index, span: Span):
|
|
"""The table region a span sits in, if any."""
|
|
if not table_index:
|
|
return None
|
|
for region in table_index.get(span.physical_page, ()):
|
|
if region.shape == SHAPE_FORMULA_2D and span.column in ("left", "right"):
|
|
# Formula bands are intentionally widened enough to reach past
|
|
# the gutter (some numerator spans have misleading leading-space
|
|
# boxes), so geometry alone can swallow prose from the opposite
|
|
# column. Keep the wide band but require its source column to
|
|
# agree with the span's extracted block column. This preserves
|
|
# AMPICILIN's gutter-adjacent "Cl" while excluding NETILMICIN's
|
|
# right-column cross-reference from a left-column formula.
|
|
region_column = "left" if (region.bbox[0] + region.bbox[2]) / 2 < 303.5 else "right"
|
|
if span.column != region_column:
|
|
continue
|
|
if region.contains(span.x0, span.y0, span.x1, span.y1):
|
|
return region
|
|
return None
|
|
|
|
|
|
SPAN_STATE_TEXT = "normalized_text"
|
|
SPAN_STATE_TABLE = "table"
|
|
SPAN_STATE_QUARANTINED = "quarantined"
|
|
SPAN_STATE_BOILERPLATE = "boilerplate_excluded"
|
|
SPAN_STATE_HEADING = "heading"
|
|
SPAN_STATE_OUT_OF_SCOPE = "out_of_scope"
|
|
SPAN_STATE_UNASSIGNED = "unassigned"
|
|
# Deliberately dropped, not missed: the book's own part-divider titles
|
|
# ("CÁC CHUYÊN LUẬN THUỐC" etc.) are structure, not content. Reporting them
|
|
# as `unassigned` would make a clean acceptance target of unassigned == 0
|
|
# impossible to state honestly.
|
|
SPAN_STATE_STRUCTURAL = "structural_excluded"
|
|
|
|
|
|
def assemble(spans: List[Span], table_index=None, ledger: Optional[list] = None) -> Iterator[Monograph]:
|
|
"""Assemble monographs from spans.
|
|
|
|
`table_index` maps a physical page to the table regions on it (see
|
|
`tables.index_by_page`). When supplied, spans falling inside a region are
|
|
diverted into `Monograph.tables` instead of section prose — measured
|
|
reason: physical page 109's dosage-form table was otherwise concatenated
|
|
cell by cell into a section body. Omitting it keeps the previous
|
|
behaviour, so callers without a region map still work.
|
|
"""
|
|
raw_chars = sum(len(s.text) for s in spans)
|
|
spans = merge_same_line_bold_fragments(spans)
|
|
events = _filter_false_positive_titles(_coalesce_titles(_classify(spans, table_index)))
|
|
|
|
# Span-level coverage ledger. Character counts alone cannot balance here
|
|
# (normalization joins, substitutes and drops characters), so every span
|
|
# is given a state first and characters are aggregated from that.
|
|
states: dict = {}
|
|
if ledger is not None:
|
|
for s in spans:
|
|
if not s.text.strip():
|
|
states[id(s)] = "whitespace_only"
|
|
elif _is_page_boilerplate(s):
|
|
states[id(s)] = SPAN_STATE_BOILERPLATE
|
|
elif is_part_divider(s.text):
|
|
states[id(s)] = SPAN_STATE_STRUCTURAL
|
|
elif not in_monograph_range(s):
|
|
states[id(s)] = SPAN_STATE_OUT_OF_SCOPE
|
|
elif is_monograph_title_candidate(s):
|
|
# title spans are merged into a Heading event and lose their
|
|
# link back to the source span, so they are accounted for here
|
|
# using the same predicate the classifier uses
|
|
states[id(s)] = SPAN_STATE_HEADING
|
|
else:
|
|
states[id(s)] = SPAN_STATE_UNASSIGNED
|
|
|
|
def mark(span: Span, state: str):
|
|
if ledger is not None:
|
|
states[id(span)] = state
|
|
|
|
seen_ids: set = set()
|
|
current: Optional[Monograph] = None
|
|
current_section_key: Optional[str] = None
|
|
runs: List[tuple] = [] # ordered [(region_or_None, [spans])]
|
|
inline_prefix: Optional[tuple[str, Span]] = None
|
|
awaiting_qualifier = False
|
|
|
|
def append_span(span: Span, region):
|
|
"""Keep spans in reading order, starting a new run whenever the
|
|
prose/table context changes — this is what preserves the real
|
|
prose -> table -> prose sequence inside one section."""
|
|
key = region.table_id if region is not None else None
|
|
if runs and runs[-1][0] == key:
|
|
runs[-1][1].append(span)
|
|
else:
|
|
runs.append((key, [span], region))
|
|
|
|
def build_parts() -> List[SectionPart]:
|
|
parts: List[SectionPart] = []
|
|
for entry in runs:
|
|
key, collected = entry[0], entry[1]
|
|
region = entry[2] if len(entry) > 2 else None
|
|
if not collected:
|
|
continue
|
|
text = substitute_pua(join_spans(collected))
|
|
if not text.strip():
|
|
continue
|
|
pages = [s_.physical_page for s_ in collected]
|
|
xs0 = min(s_.x0 for s_ in collected); ys0 = min(s_.y0 for s_ in collected)
|
|
xs1 = max(s_.x1 for s_ in collected); ys1 = max(s_.y1 for s_ in collected)
|
|
ids = [s_.span_id for s_ in collected]
|
|
if key is None:
|
|
parts.append(SectionPart(
|
|
kind=PART_PROSE, text=text, physical_page=min(pages),
|
|
bbox=[xs0, ys0, xs1, ys1], source_span_ids=ids,
|
|
))
|
|
else:
|
|
parts.append(SectionPart(
|
|
kind=PART_TABLE, text=text, physical_page=min(pages),
|
|
bbox=[xs0, ys0, xs1, ys1], source_span_ids=ids,
|
|
table_id=key,
|
|
# deterministic: derived from the first source span, so the
|
|
# same PDF always produces the same id. A counter suffix
|
|
# would merely hide a duplicate rather than identify it.
|
|
table_part_id=f"{key}@{ids[0]}",
|
|
continuation_group=key,
|
|
shape=region.shape if region is not None else None,
|
|
quarantined=(region.shape in QUARANTINE_SHAPES)
|
|
if region is not None else False,
|
|
))
|
|
if inline_prefix:
|
|
inline_text, inline_span = inline_prefix
|
|
head = SectionPart(
|
|
kind=PART_PROSE, text=inline_text,
|
|
physical_page=inline_span.physical_page,
|
|
bbox=[inline_span.x0, inline_span.y0, inline_span.x1, inline_span.y1],
|
|
source_span_ids=[inline_span.span_id],
|
|
)
|
|
parts.insert(0, head)
|
|
return parts
|
|
|
|
def close_current_section():
|
|
nonlocal runs, inline_prefix
|
|
if current is not None and current_section_key is None and runs:
|
|
# spans seen after the title but before any section heading
|
|
current.preamble.extend(build_parts())
|
|
if current is not None and current_section_key is not None:
|
|
existing = current.sections[current_section_key]
|
|
addition = build_parts()
|
|
# A section heading can legitimately appear twice inside one
|
|
# monograph (measured: 33 monographs, 38 occurrences — e.g.
|
|
# CEFAMANDOL's "Liều lượng và cách dùng" resumes on physical page
|
|
# 339 after a renal-dosing table). Replacing the SectionSpan here
|
|
# silently destroyed everything captured before the repeat, so
|
|
# the parts are concatenated instead. The first heading stays the
|
|
# section's provenance anchor.
|
|
combined = list(existing.parts) + addition
|
|
current.sections[current_section_key] = SectionSpan(
|
|
key=existing.key, display_name=existing.display_name,
|
|
heading=existing.heading,
|
|
# `text` is prose only. Table parts stay in `parts` with their
|
|
# own provenance and quarantine flag, so anything reading
|
|
# `.text` (the chunker included) cannot pick up linearised
|
|
# cells by accident — the ordering is preserved in `parts`.
|
|
text="\n".join(
|
|
p_.text for p_ in combined
|
|
if p_.kind == PART_PROSE and not p_.quarantined and p_.text
|
|
).strip(),
|
|
parts=combined,
|
|
)
|
|
for part in addition:
|
|
if part.kind == PART_TABLE:
|
|
current.tables.append(TableBlock(
|
|
table_id=part.table_id, shape=part.shape or "",
|
|
physical_page=part.physical_page, bbox=list(part.bbox),
|
|
section_key=current_section_key, text=part.text,
|
|
quarantined=part.quarantined,
|
|
table_part_id=part.table_part_id,
|
|
continuation_group=part.continuation_group,
|
|
source_span_ids=list(part.source_span_ids),
|
|
))
|
|
runs = []
|
|
inline_prefix = None
|
|
|
|
def finalize(monograph: Monograph) -> Monograph:
|
|
# Duplicate check happens here, not at title-detection time: the
|
|
# qualifier line (if any) is only known a few events later, so
|
|
# checking at open-time would false-positive on the legitimate
|
|
# SALBUTAMOL case (outlier item 18) before the qualifier resolves.
|
|
if monograph.drug_id in seen_ids:
|
|
raise DuplicateDrugIdError(
|
|
f"duplicate drug_id '{monograph.drug_id}' (title '{monograph.drug_name}', "
|
|
f"physical page {monograph.source_page_range[0]}) — check for a qualifier "
|
|
f"line (outlier item 18) before assuming this is a real collision"
|
|
)
|
|
seen_ids.add(monograph.drug_id)
|
|
atc_source = monograph.sections.get("ma_atc")
|
|
# One real class monograph combines the two labels as "Tên chung
|
|
# quốc tế và mã ATC". It remains the required title anchor, and
|
|
# its ATC codes are still extracted rather than silently discarded.
|
|
if atc_source is None:
|
|
atc_source = monograph.sections.get("ten_chung_quoc_te")
|
|
if atc_source is not None:
|
|
result = extract_atc_codes(atc_source.text)
|
|
monograph.atc_codes = result.codes
|
|
monograph.atc_stated_absent = result.stated_absent
|
|
return monograph
|
|
|
|
for event in events:
|
|
if isinstance(event, Heading) and event.is_monograph_title:
|
|
close_current_section()
|
|
if current is not None:
|
|
yield finalize(current)
|
|
current = Monograph(
|
|
drug_id=_slugify(event.text), drug_name=event.text,
|
|
source_page_range=[event.physical_page, event.physical_page],
|
|
)
|
|
current_section_key = None
|
|
awaiting_qualifier = True
|
|
for src in getattr(event, "source_spans", ()) or ():
|
|
mark(src, SPAN_STATE_HEADING)
|
|
continue
|
|
|
|
if current is None:
|
|
continue # front matter / general chapters before the first monograph
|
|
|
|
if isinstance(event, _SectionEvent):
|
|
close_current_section()
|
|
current_section_key = event.section_def.key
|
|
if event.section_def.key not in current.sections:
|
|
current.sections[event.section_def.key] = SectionSpan(
|
|
key=event.section_def.key,
|
|
display_name=event.section_def.display_name,
|
|
heading=Heading(
|
|
text=event.section_def.display_name,
|
|
physical_page=event.span.physical_page, y0=event.span.y0,
|
|
is_monograph_title=False, section_key=event.section_def.key,
|
|
),
|
|
text="",
|
|
)
|
|
if event.inline_value:
|
|
inline_prefix = (event.inline_value, event.span)
|
|
awaiting_qualifier = False
|
|
mark(event.span, SPAN_STATE_HEADING)
|
|
current.source_page_range[1] = max(current.source_page_range[1], event.span.physical_page)
|
|
continue
|
|
|
|
# _TextEvent
|
|
span = event.span
|
|
if awaiting_qualifier and _is_qualifier_line(span):
|
|
text = span.text.strip()
|
|
current.drug_id = f"{current.drug_id}_{_slugify(text)}"
|
|
current.drug_name = f"{current.drug_name} {text}"
|
|
awaiting_qualifier = False
|
|
mark(span, SPAN_STATE_HEADING)
|
|
continue
|
|
awaiting_qualifier = False
|
|
|
|
if not in_monograph_range(span):
|
|
continue
|
|
current.source_page_range[1] = max(current.source_page_range[1], span.physical_page)
|
|
|
|
region = _region_for(table_index, span)
|
|
append_span(span, region)
|
|
if region is not None:
|
|
mark(span, SPAN_STATE_QUARANTINED
|
|
if region.shape in QUARANTINE_SHAPES else SPAN_STATE_TABLE)
|
|
else:
|
|
mark(span, SPAN_STATE_TEXT)
|
|
|
|
if ledger is not None:
|
|
ledger.append({"raw_chars_before_merge": raw_chars})
|
|
for s_obj in spans:
|
|
ledger.append({
|
|
"state": states[id(s_obj)],
|
|
"physical_page": s_obj.physical_page,
|
|
"bbox": [s_obj.x0, s_obj.y0, s_obj.x1, s_obj.y1],
|
|
"chars": len(s_obj.text),
|
|
"text": s_obj.text[:60],
|
|
})
|
|
|
|
close_current_section()
|
|
if current is not None:
|
|
yield finalize(current)
|