24 lines
915 B
Python
24 lines
915 B
Python
"""Vietnamese text normalisation shared by drug and section resolution.
|
|
|
|
Lives here rather than in `routing.py` because `sections.py` needs it too, and
|
|
importing it from `routing` made `service -> sections -> routing -> service` a
|
|
cycle. It is a text utility with no knowledge of drugs or sections.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
import unicodedata
|
|
|
|
WORD_RE = re.compile(r"\w+", re.UNICODE)
|
|
|
|
|
|
def normalize_name(text: str) -> str:
|
|
"""Casefold, strip diacritics, collapse to space-separated word tokens.
|
|
|
|
`đ` is replaced before decomposition because it is a distinct letter rather
|
|
than a base letter plus a combining mark, so NFKD leaves it intact.
|
|
"""
|
|
decomposed = unicodedata.normalize("NFKD", text.casefold()).replace("đ", "d")
|
|
without_marks = "".join(char for char in decomposed if not unicodedata.combining(char))
|
|
return " ".join(WORD_RE.findall(without_marks))
|