Stop the slugifier from deleting the letter D-stroke
This commit is contained in:
@@ -92,9 +92,21 @@ _Event = Union[Heading, _SectionEvent, _TextEvent] # Heading == a title event
|
||||
|
||||
|
||||
def _slugify(text: str) -> str:
|
||||
normalized = unicodedata.normalize("NFKD", text)
|
||||
# `đ`/`Đ` (U+0111/U+0110) are standalone Vietnamese letters, not a base
|
||||
# letter plus a combining mark, so NFKD leaves them whole and the ASCII
|
||||
# encode below then discards them silently -- turning `ĐIỆN GIẢI` into
|
||||
# `ien_giai`. Exactly three monographs contain `Đ` and all three carried a
|
||||
# damaged drug_id because of this: GIẢI ĐỘC TỐ UỐN VÁN, KHÁNG ĐỘC TỐ BẠCH
|
||||
# HẦU, and THUỐC UỐNG BÙ NƯỚC VÀ ĐIỆN GIẢI. Found via the adversarial eval
|
||||
# suite, where both Oresol cases failed to resolve their monograph.
|
||||
#
|
||||
# Casefold before replacing so one pass covers both cases: `Đ` casefolds
|
||||
# to `đ`. This matches what `entities/catalog.py::normalize_name` already
|
||||
# does -- that function got it right and this one did not.
|
||||
folded = text.casefold().replace("đ", "d")
|
||||
normalized = unicodedata.normalize("NFKD", folded)
|
||||
ascii_text = normalized.encode("ascii", "ignore").decode("ascii")
|
||||
return re.sub(r"[^a-z0-9]+", "_", ascii_text.lower()).strip("_")
|
||||
return re.sub(r"[^a-z0-9]+", "_", ascii_text).strip("_")
|
||||
|
||||
|
||||
def _starts_its_visual_line(span: Span, previous: Span | None) -> bool:
|
||||
|
||||
Reference in New Issue
Block a user