363 lines
15 KiB
Python
363 lines
15 KiB
Python
"""Named gates that must hold before the corpus is chunked.
|
||
|
||
Chunking bakes whatever it is given into embeddings, where defects stop being
|
||
inspectable. So the question this module answers is not "did the pipeline
|
||
run" but "is the text going in actually the text on the page". Each gate is
|
||
reported on its own line with its own number and its own target — a single
|
||
pass/fail would hide exactly the problems that took a whole session to find.
|
||
|
||
Every gate here is computed from the artefacts, never remembered from an
|
||
earlier run: quoting a number from before a code change is the specific
|
||
mistake this project keeps catching.
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
from dataclasses import dataclass
|
||
from pathlib import Path
|
||
from typing import Dict, Iterable, List, Sequence
|
||
|
||
import re as _re
|
||
|
||
_WHITESPACE = _re.compile(r"\s+")
|
||
# Head of a source line, enough to identify it without demanding that an
|
||
# intentionally comma-split list match in full.
|
||
SENTENCE_PROBE_CHARS = 60
|
||
|
||
PUA_RANGE = (0xE000, 0xF8FF)
|
||
REPLACEMENT_CHAR = "�"
|
||
|
||
# Strings that were confirmed by eye to be corruption, each traced to a
|
||
# dropped vector-outlined glyph (outlier-catalog item 24). They are checked
|
||
# literally: if one reappears, the repair regressed.
|
||
KNOWN_CORRUPTIONS = (
|
||
"Độ n định",
|
||
"≥ 1 tu i",
|
||
"tại ch :",
|
||
)
|
||
|
||
# Fragments of 2D formulas that must never sit in prose, where the missing
|
||
# fraction bar turns a division into a multiplication.
|
||
FORMULA_FRAGMENTS = (
|
||
"Thể trọng (kg)",
|
||
"(140 - tuổi) x cân nặng",
|
||
"x (140 - số tuổi)",
|
||
"Giá trị Clcr của bệnh nhân",
|
||
"218 x P x",
|
||
"× trọng lượng cơ thể (kg)",
|
||
"Cân nặng (kg) x liều",
|
||
)
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class Gate:
|
||
name: str
|
||
count: int
|
||
target: int = 0
|
||
detail: str = ""
|
||
|
||
@property
|
||
def passed(self) -> bool:
|
||
return self.count == self.target
|
||
|
||
|
||
def _section_texts(monograph: dict) -> Iterable[str]:
|
||
for section in (monograph.get("sections") or {}).values():
|
||
yield section.get("text") or ""
|
||
|
||
|
||
def _count_pua(text: str) -> int:
|
||
return sum(1 for ch in text if PUA_RANGE[0] <= ord(ch) <= PUA_RANGE[1])
|
||
|
||
|
||
def evaluate(monographs: Sequence[dict],
|
||
transcribed_runs: Sequence[dict] = ()) -> List[Gate]:
|
||
"""Compute every readiness gate over the whole corpus."""
|
||
pua = replacement = empty = no_provenance = part_no_provenance = 0
|
||
corruptions: Dict[str, int] = {c: 0 for c in KNOWN_CORRUPTIONS}
|
||
formula_leaks: Dict[str, int] = {f: 0 for f in FORMULA_FRAGMENTS}
|
||
unflagged_blocks = 0
|
||
ids: Dict[str, int] = {}
|
||
table_ids: Dict[str, int] = {}
|
||
no_page_range = 0
|
||
corpus = []
|
||
|
||
for monograph in monographs:
|
||
ids[monograph["drug_id"]] = ids.get(monograph["drug_id"], 0) + 1
|
||
if not monograph.get("source_page_range"):
|
||
no_page_range += 1
|
||
for section in (monograph.get("sections") or {}).values():
|
||
text = section.get("text") or ""
|
||
corpus.append(text)
|
||
if not text.strip():
|
||
empty += 1
|
||
if not section.get("parts"):
|
||
no_provenance += 1
|
||
for part in section.get("parts") or []:
|
||
if not part.get("source_span_ids"):
|
||
part_no_provenance += 1
|
||
pua += _count_pua(text)
|
||
replacement += text.count(REPLACEMENT_CHAR)
|
||
for phrase in KNOWN_CORRUPTIONS:
|
||
corruptions[phrase] += text.count(phrase)
|
||
for phrase in FORMULA_FRAGMENTS:
|
||
formula_leaks[phrase] += text.count(phrase)
|
||
for block in monograph.get("tables") or []:
|
||
table_id = block.get("table_id")
|
||
if table_id:
|
||
table_ids[table_id] = table_ids.get(table_id, 0) + 1
|
||
if not block.get("quarantined"):
|
||
unflagged_blocks += 1
|
||
|
||
joined = "\n".join(corpus)
|
||
unmerged = [
|
||
run for run in transcribed_runs
|
||
if len(run["text"].strip()) > 2 and run["text"].strip() not in joined
|
||
]
|
||
|
||
return [
|
||
Gate("outlined_run_not_merged", len(unmerged),
|
||
detail="; ".join(f"p{r['physical_page']} {r['text'][:40]!r}"
|
||
for r in unmerged[:5])),
|
||
Gate("known_corruption_string", sum(corruptions.values()),
|
||
detail=", ".join(f"{k!r}={v}" for k, v in corruptions.items() if v)),
|
||
Gate("formula_fragment_in_prose", sum(formula_leaks.values()),
|
||
detail=", ".join(f"{k!r}={v}" for k, v in formula_leaks.items() if v)),
|
||
Gate("pua_char", pua),
|
||
Gate("replacement_char_ufffd", replacement),
|
||
Gate("empty_section", empty),
|
||
Gate("section_without_provenance", no_provenance),
|
||
Gate("part_without_source_span_ids", part_no_provenance),
|
||
Gate("unflagged_quarantine_block", unflagged_blocks),
|
||
Gate("duplicate_table_id", sum(1 for n in table_ids.values() if n > 1)),
|
||
Gate("duplicate_drug_id", sum(1 for n in ids.values() if n > 1)),
|
||
Gate("monograph_without_page_range", no_page_range),
|
||
]
|
||
|
||
|
||
def corpus_size(monographs: Sequence[dict]) -> Dict[str, int]:
|
||
"""Informational, not a gate: how much text chunking would consume."""
|
||
sections = [t for m in monographs for t in _section_texts(m)]
|
||
return {
|
||
"monographs": len(monographs),
|
||
"sections": len(sections),
|
||
"section_chars": sum(len(t) for t in sections),
|
||
"quarantined_blocks": sum(len(m.get("tables") or []) for m in monographs),
|
||
}
|
||
|
||
|
||
def read_monographs(path: Path) -> List[dict]:
|
||
with path.open(encoding="utf-8") as handle:
|
||
return [json.loads(line) for line in handle if line.strip()]
|
||
|
||
|
||
def evaluate_chunks(monographs: Sequence[dict],
|
||
chunks: Sequence[dict]) -> List[Gate]:
|
||
"""ADR 0006 gates: a chunk must never hide that a block was lifted.
|
||
|
||
The failure being guarded against is silent, not visible: a chunk of
|
||
AMPICILIN VÀ SULBACTAM's dosing section is grammatical, complete-looking
|
||
prose with the renal-dosing table absent and nothing marking the absence.
|
||
Measured: 127 of 167 lifted blocks came out of `liều lượng và cách dùng`.
|
||
"""
|
||
blocks_by_section: Dict[tuple, list] = {}
|
||
block_ids: Dict[str, str] = {}
|
||
block_texts: Dict[str, str] = {}
|
||
sections_by_key: Dict[tuple, dict] = {}
|
||
for monograph in monographs:
|
||
for section_key, section in (monograph.get("sections") or {}).items():
|
||
sections_by_key[(monograph["drug_id"], section_key)] = section
|
||
for block in monograph.get("tables") or []:
|
||
key = (monograph["drug_id"], block.get("section_key"))
|
||
blocks_by_section.setdefault(key, []).append(block)
|
||
block_ids[block["table_id"]] = monograph["drug_id"]
|
||
if block.get("text"):
|
||
block_texts[block["table_id"]] = block["text"]
|
||
|
||
referenced: Dict[tuple, set] = {}
|
||
unknown_id = missing_provenance = leaked = 0
|
||
descriptors = 0
|
||
descriptor_without_attachment = 0
|
||
attachment_header_row_present = 0
|
||
descriptor_with_unverified_columns = 0
|
||
unsupported_schema = 0
|
||
prose_without_source_text = 0
|
||
source_text_not_unique = 0
|
||
physical_range_not_exact = 0
|
||
descriptor_range_not_attachment_page = 0
|
||
attachment_without_printed_page = 0
|
||
context_label_missing_from_text = 0
|
||
invalid_printed_page_range = 0
|
||
|
||
for chunk in chunks:
|
||
if chunk.get("schema_version") != 4:
|
||
unsupported_schema += 1
|
||
printed_range = chunk.get("printed_page_range")
|
||
if (
|
||
not isinstance(printed_range, list)
|
||
or len(printed_range) != 2
|
||
or not all(isinstance(page, int) for page in printed_range)
|
||
or printed_range[0] > printed_range[1]
|
||
):
|
||
invalid_printed_page_range += 1
|
||
attachments = chunk.get("attachments") or []
|
||
if chunk.get("chunk_kind") == "block_descriptor":
|
||
descriptors += 1
|
||
if not attachments:
|
||
descriptor_without_attachment += 1
|
||
if "Cột:" in (chunk.get("text") or ""):
|
||
descriptor_with_unverified_columns += 1
|
||
if attachments:
|
||
page = attachments[0].get("physical_page")
|
||
if chunk.get("source_page_range") != [page, page]:
|
||
descriptor_range_not_attachment_page += 1
|
||
else:
|
||
source_body = chunk.get("source_text") or ""
|
||
if not source_body:
|
||
prose_without_source_text += 1
|
||
section = sections_by_key.get((chunk["drug_id"], chunk["section_key"]))
|
||
section_text = (section or {}).get("text", "").strip()
|
||
if not source_body or section_text.count(source_body) != 1:
|
||
source_text_not_unique += 1
|
||
else:
|
||
chunk_start = section_text.index(source_body)
|
||
chunk_end = chunk_start + len(source_body)
|
||
cursor = 0
|
||
pages = []
|
||
for part in (section or {}).get("parts") or []:
|
||
if (part.get("kind") != "prose" or part.get("quarantined")
|
||
or not part.get("text")):
|
||
continue
|
||
part_start = section_text.find(part["text"], cursor)
|
||
if part_start < 0:
|
||
pages = []
|
||
break
|
||
part_end = part_start + len(part["text"])
|
||
cursor = part_end
|
||
if part_start < chunk_end and part_end > chunk_start:
|
||
pages.append(part["physical_page"])
|
||
expected = [min(pages), max(pages)] if pages else None
|
||
if chunk.get("source_page_range") != expected:
|
||
physical_range_not_exact += 1
|
||
body = chunk.get("text") or ""
|
||
if any(label not in body for label in chunk.get("context_labels") or []):
|
||
context_label_missing_from_text += 1
|
||
key = (chunk["drug_id"], chunk["section_key"])
|
||
for attachment in attachments:
|
||
if attachment.get("header_row"):
|
||
attachment_header_row_present += 1
|
||
if attachment.get("printed_page") is None:
|
||
attachment_without_printed_page += 1
|
||
referenced.setdefault(key, set()).add(attachment["block_id"])
|
||
if block_ids.get(attachment["block_id"]) != chunk["drug_id"]:
|
||
unknown_id += 1
|
||
if attachment.get("physical_page") is None or not attachment.get("bbox"):
|
||
missing_provenance += 1
|
||
body = chunk.get("text") or ""
|
||
for attachment in attachments:
|
||
source = block_texts.get(attachment["block_id"], "")
|
||
probe = source.strip()[:60]
|
||
if len(probe) > 20 and probe in body:
|
||
leaked += 1
|
||
|
||
over_ceiling = [c for c in chunks if c.get("oversized")]
|
||
uncovered = _sections_not_covered(monographs, chunks)
|
||
|
||
unreferenced = 0
|
||
for key, blocks in blocks_by_section.items():
|
||
seen = referenced.get(key, set())
|
||
unreferenced += sum(1 for b in blocks if b["table_id"] not in seen)
|
||
|
||
total_blocks = sum(len(v) for v in blocks_by_section.values())
|
||
return [
|
||
Gate("chunk_over_token_ceiling", len(over_ceiling),
|
||
detail="; ".join(c["chunk_id"] for c in over_ceiling[:3])),
|
||
Gate("chunk_without_printed_page_range", invalid_printed_page_range),
|
||
Gate("chunk_schema_version_not_supported", unsupported_schema),
|
||
Gate("prose_without_source_text", prose_without_source_text),
|
||
Gate("chunk_source_text_not_unique", source_text_not_unique),
|
||
Gate("chunk_physical_range_not_exact", physical_range_not_exact),
|
||
Gate("descriptor_range_not_attachment_page",
|
||
descriptor_range_not_attachment_page),
|
||
Gate("attachment_without_printed_page", attachment_without_printed_page),
|
||
Gate("context_label_missing_from_text", context_label_missing_from_text),
|
||
Gate("section_not_reassemblable_from_chunks", len(uncovered),
|
||
detail="; ".join(uncovered[:3])),
|
||
Gate("section_block_without_chunk_reference", unreferenced),
|
||
Gate("attachment_block_id_unknown", unknown_id),
|
||
Gate("attachment_without_page_or_bbox", missing_provenance),
|
||
Gate("block_text_leaked_into_chunk_text", leaked),
|
||
Gate("attachment_header_row_present", attachment_header_row_present),
|
||
Gate("descriptor_with_unverified_columns", descriptor_with_unverified_columns),
|
||
Gate("descriptor_chunk_without_attachment", descriptor_without_attachment),
|
||
Gate("descriptor_count_vs_block_count", descriptors, target=total_blocks,
|
||
detail=f"{descriptors} descriptors for {total_blocks} blocks"),
|
||
]
|
||
|
||
|
||
def read_chunks(path: Path) -> List[dict]:
|
||
with path.open(encoding="utf-8") as handle:
|
||
return [json.loads(line) for line in handle if line.strip()]
|
||
|
||
|
||
def _reassemble(parts: Sequence[str]) -> str:
|
||
"""Glue chunk parts back together, removing the deliberate overlap."""
|
||
if not parts:
|
||
return ""
|
||
text = parts[0]
|
||
for part in parts[1:]:
|
||
overlap = 0
|
||
for size in range(min(len(text), len(part)), 0, -1):
|
||
if text.endswith(part[:size]):
|
||
overlap = size
|
||
break
|
||
text += part[overlap:]
|
||
return text
|
||
|
||
|
||
def _sections_not_covered(monographs: Sequence[dict],
|
||
chunks: Sequence[dict]) -> List[str]:
|
||
"""Sections that cannot be rebuilt exactly from their own chunks.
|
||
|
||
Stronger than asking whether each line still appears somewhere: it proves
|
||
the chunks are a faithful partition of the section, so nothing was dropped
|
||
*and* nothing was reordered or duplicated beyond the intended overlap.
|
||
|
||
Whitespace is removed from both sides rather than normalised, because
|
||
each split seam legitimately loses one space: a part is built with
|
||
`"".join(...).strip()`, and the space that sat between two sentences falls
|
||
on the boundary. Measured on ABACAVIR: exactly two single spaces across a
|
||
4,232-character section, nothing else. The gate exists to prove no
|
||
character of *content* is lost, reordered, or duplicated beyond the
|
||
intended overlap; it is not a formatting check.
|
||
|
||
Two weaker versions were tried first and both were instrument bugs, not
|
||
data bugs. Joining chunk texts with a newline meant a paragraph split
|
||
across parts could never match, reporting 734 sections missing when the
|
||
first one it named was present. Probing a 60-character head then failed on
|
||
NAPROXEN alone, because that probe straddled an overlap seam where the
|
||
repeated text legitimately appears twice.
|
||
"""
|
||
by_section: Dict[tuple, List[dict]] = {}
|
||
for chunk in chunks:
|
||
if chunk.get("chunk_kind") == "block_descriptor":
|
||
continue
|
||
key = (chunk["drug_id"], chunk["section_key"])
|
||
by_section.setdefault(key, []).append(chunk)
|
||
|
||
broken = []
|
||
for monograph in monographs:
|
||
for key, section in (monograph.get("sections") or {}).items():
|
||
source = _WHITESPACE.sub("", section.get("text") or "")
|
||
if not source:
|
||
continue
|
||
parts = sorted(by_section.get((monograph["drug_id"], key), []),
|
||
key=lambda c: c.get("part_index", 0))
|
||
rebuilt = _WHITESPACE.sub(
|
||
"", _reassemble([
|
||
c.get("source_text") or c.get("text") or "" for c in parts
|
||
]))
|
||
if rebuilt != source:
|
||
broken.append(f"{monograph['drug_id']}/{key}")
|
||
return broken
|