Wire the guarded conversational RAG answer layer end-to-end
This commit is contained in:
@@ -0,0 +1,23 @@
|
||||
"""Vietnamese text normalisation shared by drug and section resolution.
|
||||
|
||||
Lives here rather than in `routing.py` because `sections.py` needs it too, and
|
||||
importing it from `routing` made `service -> sections -> routing -> service` a
|
||||
cycle. It is a text utility with no knowledge of drugs or sections.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import unicodedata
|
||||
|
||||
WORD_RE = re.compile(r"\w+", re.UNICODE)
|
||||
|
||||
|
||||
def normalize_name(text: str) -> str:
|
||||
"""Casefold, strip diacritics, collapse to space-separated word tokens.
|
||||
|
||||
`đ` is replaced before decomposition because it is a distinct letter rather
|
||||
than a base letter plus a combining mark, so NFKD leaves it intact.
|
||||
"""
|
||||
decomposed = unicodedata.normalize("NFKD", text.casefold()).replace("đ", "d")
|
||||
without_marks = "".join(char for char in decomposed if not unicodedata.combining(char))
|
||||
return " ".join(WORD_RE.findall(without_marks))
|
||||
Reference in New Issue
Block a user