Stop the slugifier from deleting the letter D-stroke

This commit is contained in:
2026-08-19 13:07:13 +07:00
parent 24c55d1627
commit 4490a1abf0
14 changed files with 739 additions and 198 deletions
+14 -2
View File
@@ -92,9 +92,21 @@ _Event = Union[Heading, _SectionEvent, _TextEvent] # Heading == a title event
def _slugify(text: str) -> str:
normalized = unicodedata.normalize("NFKD", text)
# `đ`/`Đ` (U+0111/U+0110) are standalone Vietnamese letters, not a base
# letter plus a combining mark, so NFKD leaves them whole and the ASCII
# encode below then discards them silently -- turning `ĐIỆN GIẢI` into
# `ien_giai`. Exactly three monographs contain `Đ` and all three carried a
# damaged drug_id because of this: GIẢI ĐỘC TỐ UỐN VÁN, KHÁNG ĐỘC TỐ BẠCH
# HẦU, and THUỐC UỐNG BÙ NƯỚC VÀ ĐIỆN GIẢI. Found via the adversarial eval
# suite, where both Oresol cases failed to resolve their monograph.
#
# Casefold before replacing so one pass covers both cases: `Đ` casefolds
# to `đ`. This matches what `entities/catalog.py::normalize_name` already
# does -- that function got it right and this one did not.
folded = text.casefold().replace("đ", "d")
normalized = unicodedata.normalize("NFKD", folded)
ascii_text = normalized.encode("ascii", "ignore").decode("ascii")
return re.sub(r"[^a-z0-9]+", "_", ascii_text.lower()).strip("_")
return re.sub(r"[^a-z0-9]+", "_", ascii_text).strip("_")
def _starts_its_visual_line(span: Span, previous: Span | None) -> bool: