# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/1b.indic_normalizer.ipynb.

# %% auto 0
__all__ = ['NormalizerI', 'BaseNormalizer', 'DevanagariNormalizer', 'HindiNormalizer', 'PunjabiNormalizer', 'TeluguNormalizer',
           'GujaratiNormalizer', 'OdiaNormalizer', 'BengaliNormalizer', 'TamilNormalizer', 'KannadaNormalizer',
           'MalayalamNormalizer']

# %% ../nbs/1b.indic_normalizer.ipynb 3
import re
from indic_numtowords import num2words


from . import langinfo

# %% ../nbs/1b.indic_normalizer.ipynb 4
#  Copyright (c) 2013-present, Anoop Kunchukuttan
#  All rights reserved.
#
#  This source code is licensed under the MIT license found in the
#  LICENSE file in the root directory of this source tree.
#

# Program for normalization of text written in Unicode. This is mainly geared towards Indic scripts
#
# @author Anoop Kunchukuttan
#


class NormalizerI(object):
    """
    The normalizer classes do the following:
    * Some characters have multiple Unicode codepoints. The normalizer chooses a single standard representation
    * Some control characters are deleted
    * While typing using the Latin keyboard, certain typical mistakes occur which are corrected by the module
    Base class for normalizer. Performs some common normalization, which includes:
    * Byte order mark, word joiner, etc. removal
    * ZERO_WIDTH_NON_JOINER and ZERO_WIDTH_JOINER removal
    * ZERO_WIDTH_SPACE and NO_BREAK_SPACE replaced by spaces
    Script specific normalizers should derive from this class and override the normalize() method.
    They can call the super class 'normalize() method to avail of the common normalization
    """

    BYTE_ORDER_MARK = "\ufeff"
    BYTE_ORDER_MARK_2 = "\ufffe"
    WORD_JOINER = "\u2060"
    SOFT_HYPHEN = "\u00ad"

    ZERO_WIDTH_SPACE = "\u200b"
    NO_BREAK_SPACE = "\u00a0"

    ZERO_WIDTH_NON_JOINER = "\u200c"
    ZERO_WIDTH_JOINER = "\u200d"

    def _normalize_punctuations(self, text):
        """
        Normalize punctuations.
        Applied many of the punctuation normalizations that are part of MosesNormalizer
        from sacremoses
        """
        text = text.replace(NormalizerI.BYTE_ORDER_MARK, "")
        text = text.replace("„", r'"')
        text = text.replace("“", r'"')
        text = text.replace("”", r'"')
        text = text.replace("–", r"-")
        text = text.replace("—", r" - ")
        text = text.replace("´", r"'")
        text = text.replace("‘", r"'")
        text = text.replace("‚", r"'")
        text = text.replace("’", r"'")
        text = text.replace("''", r'"')
        text = text.replace("´´", r'"')
        text = text.replace("…", r"...")

        return text

    def normalize(self, text):
        pass

# %% ../nbs/1b.indic_normalizer.ipynb 5
class BaseNormalizer(NormalizerI):
    """Common class used in most of indic languages inherit from this code."""

    def __init__(
        self,
        lang,
        remove_nuktas=False,
        nasals_mode="do_nothing",
        do_normalize_chandras=False,
        do_normalize_vowel_ending=False,
    ):
        self.lang = lang
        self.remove_nuktas = remove_nuktas
        self.nasals_mode = nasals_mode
        self.do_normalize_chandras = do_normalize_chandras
        self.do_normalize_vowel_ending = do_normalize_vowel_ending

        self._init_normalize_chandras()
        self._init_normalize_nasals()
        self._init_normalize_vowel_ending()
        # self._init_visarga_correction()

    def _init_normalize_vowel_ending(self):
        if self.lang in langinfo.IE_LANGUAGES:
            self.fn_vowel_ending = self._normalize_word_vowel_ending_ie
        elif self.lang in langinfo.DRAVIDIAN_LANGUAGES:
            self.fn_vowel_ending = self._normalize_word_vowel_ending_dravidian
        else:
            self.fn_vowel_ending = lambda x: x

    def _init_normalize_chandras(self):
        substitution_offsets = [
            [0x0D, 0x0F],  # chandra e, independent
            [0x11, 0x13],  # chandra o, independent
            [0x45, 0x47],  # chandra e , 0xde],pendent
            [0x49, 0x4B],  # chandra o , 0xde],pendent
            # [0x72 , 0x0f], # mr: chandra e, independent
            [0x00, 0x02],  # chandrabindu
            [0x01, 0x02],  # chandrabindu
        ]

        self.chandra_substitutions = [
            (
                langinfo.offset_to_char(x[0], self.lang),
                langinfo.offset_to_char(x[1], self.lang),
            )
            for x in substitution_offsets
        ]

    def _normalize_chandras(self, text):
        for match, repl in self.chandra_substitutions:
            text = text.replace(match, repl)
        return text

    def _init_to_anusvaara_strict(self):
        """
        `r1_nasal=re.compile(r'\\u0919\\u094D([\\u0915-\\u0918])')`
        """

        pat_signatures = [
            [0x19, 0x15, 0x18],
            [0x1E, 0x1A, 0x1D],
            [0x23, 0x1F, 0x22],
            [0x28, 0x24, 0x27],
            [0x29, 0x24, 0x27],
            [0x2E, 0x2A, 0x2D],
        ]

        halant_offset = 0x4D
        anusvaara_offset = 0x02

        pats = []

        for pat_signature in pat_signatures:
            pat = re.compile(
                r"{nasal}{halant}([{start_r}-{end_r}])".format(
                    nasal=langinfo.offset_to_char(pat_signature[0], self.lang),
                    halant=langinfo.offset_to_char(halant_offset, self.lang),
                    start_r=langinfo.offset_to_char(pat_signature[1], self.lang),
                    end_r=langinfo.offset_to_char(pat_signature[2], self.lang),
                )
            )
            pats.append(pat)

        repl_string = "{anusvaara}\\1".format(
            anusvaara=langinfo.offset_to_char(anusvaara_offset, self.lang)
        )

        self.pats_repls = (pats, repl_string)

    def _to_anusvaara_strict(self, text):
        pats, repl_string = self.pats_repls
        for pat in pats:
            text = pat.sub(repl_string, text)

        return text

    def _init_to_anusvaara_relaxed(self):
        """
        `r1_nasal=re.compile(r'\\u0919\\u094D([\\u0915-\\u0918])')`
        """

        nasals_list = [0x19, 0x1E, 0x23, 0x28, 0x29, 0x2E]
        nasals_list_str = ",".join(
            [langinfo.offset_to_char(x, self.lang) for x in nasals_list]
        )

        halant_offset = 0x4D
        anusvaara_offset = 0x02

        pat = re.compile(
            r"[{nasals_list_str}]{halant}".format(
                nasals_list_str=nasals_list_str,
                halant=langinfo.offset_to_char(halant_offset, self.lang),
            )
        )

        repl_string = "{anusvaara}".format(
            anusvaara=langinfo.offset_to_char(anusvaara_offset, self.lang)
        )

        self.pats_repls = (pat, repl_string)

    def _to_anusvaara_relaxed(self, text):
        pat, repl_string = self.pats_repls
        return pat.sub(repl_string, text)

    def _init_to_nasal_consonants(self):
        """
        `r1_nasal=re.compile(r'\\u0919\\u094D([\\u0915-\\u0918])')`
        """

        pat_signatures = [
            [0x19, 0x15, 0x18],
            [0x1E, 0x1A, 0x1D],
            [0x23, 0x1F, 0x22],
            [0x28, 0x24, 0x27],
            [0x29, 0x24, 0x27],
            [0x2E, 0x2A, 0x2D],
        ]

        halant_offset = 0x4D
        anusvaara_offset = 0x02

        pats = []
        repl_strings = []

        for pat_signature in pat_signatures:
            pat = re.compile(
                r"{anusvaara}([{start_r}-{end_r}])".format(
                    anusvaara=langinfo.offset_to_char(anusvaara_offset, self.lang),
                    start_r=langinfo.offset_to_char(pat_signature[1], self.lang),
                    end_r=langinfo.offset_to_char(pat_signature[2], self.lang),
                )
            )
            pats.append(pat)
            repl_string = "{nasal}{halant}\\1".format(
                nasal=langinfo.offset_to_char(pat_signature[0], self.lang),
                halant=langinfo.offset_to_char(halant_offset, self.lang),
            )
            repl_strings.append(repl_string)

        self.pats_repls = list(zip(pats, repl_strings))

    def _to_nasal_consonants(self, text):
        for pat, repl in self.pats_repls:
            text = pat.sub(repl, text)

        return text

    def _init_normalize_nasals(self):
        if self.nasals_mode == "to_anusvaara_strict":
            self._init_to_anusvaara_strict()
        elif self.nasals_mode == "to_anusvaara_relaxed":
            self._init_to_anusvaara_relaxed()
        elif self.nasals_mode == "to_nasal_consonants":
            self._init_to_nasal_consonants()

    def _normalize_nasals(self, text):
        if self.nasals_mode == "to_anusvaara_strict":
            return self._to_anusvaara_strict(text)
        elif self.nasals_mode == "to_anusvaara_relaxed":
            return self._to_anusvaara_relaxed(text)
        elif self.nasals_mode == "to_nasal_consonants":
            return self._to_nasal_consonants(text)
        else:
            return text

    def _normalize_word_vowel_ending_dravidian(self, word):
        """
        for Dravidian
        - consonant ending: add 'a' ki maatra
        - halant ending: no change
        - 'a' ki maatra: no change
        """
        if len(word) > 0 and langinfo.is_consonant(word[-1], self.lang):
            return word + langinfo.offset_to_char(0x3E, self.lang)
        else:
            return word

    def _normalize_word_vowel_ending_ie(self, word):
        """
        for IE
        - consonant ending: add halant
        - halant ending: no change
        - 'a' ki maatra: no change
        """
        if len(word) > 0 and langinfo.is_consonant(word[-1], self.lang):
            return word + langinfo.offset_to_char(langinfo.HALANTA_OFFSET, self.lang)
        else:
            return word

    def _normalize_vowel_ending(self, text):
        return " ".join([self.fn_vowel_ending(w) for w in text.split(" ")])

    def normalize(self, text):
        """
        Method to be implemented for normalization for each script
        """
        text = text.replace(NormalizerI.BYTE_ORDER_MARK, "")
        text = text.replace(NormalizerI.BYTE_ORDER_MARK_2, "")
        text = text.replace(NormalizerI.WORD_JOINER, "")
        text = text.replace(NormalizerI.SOFT_HYPHEN, "")

        text = text.replace(NormalizerI.ZERO_WIDTH_SPACE, " ")  # ??
        text = text.replace(NormalizerI.NO_BREAK_SPACE, " ")

        text = text.replace(NormalizerI.ZERO_WIDTH_NON_JOINER, "")
        text = text.replace(NormalizerI.ZERO_WIDTH_JOINER, "")

        text = self._normalize_punctuations(text)

        if self.do_normalize_chandras:
            text = self._normalize_chandras(text)
        text = self._normalize_nasals(text)
        if self.do_normalize_vowel_ending:
            text = self._normalize_vowel_ending(text)

        return text

    def get_char_stats(self, text):
        print(len(re.findall(NormalizerI.BYTE_ORDER_MARK, text)))
        print(len(re.findall(NormalizerI.BYTE_ORDER_MARK_2, text)))
        print(len(re.findall(NormalizerI.WORD_JOINER, text)))
        print(len(re.findall(NormalizerI.SOFT_HYPHEN, text)))

        print(len(re.findall(NormalizerI.ZERO_WIDTH_SPACE, text)))
        print(len(re.findall(NormalizerI.NO_BREAK_SPACE, text)))

        print(len(re.findall(NormalizerI.ZERO_WIDTH_NON_JOINER, text)))
        print(len(re.findall(NormalizerI.ZERO_WIDTH_JOINER, text)))

        # for mobj in re.finditer(NormalizerI.ZERO_WIDTH_NON_JOINER,text):
        #    print text[mobj.start()-10:mobj.end()+10].replace('\n', ' ').replace(NormalizerI.ZERO_WIDTH_NON_JOINER,'').encode('utf-8')
        # print hex(ord(text[mobj.end():mobj.end()+1]))

    def correct_visarga(self, text, visarga_char, char_range):
        text = re.sub(r"([\u0900-\u097f]):", "\\1\u0903", text)

# %% ../nbs/1b.indic_normalizer.ipynb 6
class DevanagariNormalizer(BaseNormalizer):
    """
    Normalizer for the Devanagari script. In addition to basic normalization by the super class,
    * Replaces the composite characters containing nuktas by their decomposed form
    * replace pipe character '|' by poorna virama character
    * replace colon ':' by visarga if the colon follows a charcter in this script

    """

    NUKTA = "\u093c"

    def __init__(
        self,
        lang="hi",
        remove_nuktas=False,
        nasals_mode="do_nothing",
        do_normalize_chandras=False,
        do_normalize_vowel_ending=False,
    ):
        super(DevanagariNormalizer, self).__init__(
            lang,
            remove_nuktas,
            nasals_mode,
            do_normalize_chandras,
            do_normalize_vowel_ending,
        )

    def __call__(self, text):
        # common normalization for Indic scripts
        text = super(DevanagariNormalizer, self).normalize(text)

        # chandra a replacement for Marathi
        text = text.replace("\u0972", "\u090f")

        # decomposing Nukta based composite characters
        text = text.replace("\u0929", "\u0928" + DevanagariNormalizer.NUKTA)
        text = text.replace("\u0931", "\u0930" + DevanagariNormalizer.NUKTA)
        text = text.replace("\u0934", "\u0933" + DevanagariNormalizer.NUKTA)
        text = text.replace("\u0958", "\u0915" + DevanagariNormalizer.NUKTA)
        text = text.replace("\u0959", "\u0916" + DevanagariNormalizer.NUKTA)
        text = text.replace("\u095a", "\u0917" + DevanagariNormalizer.NUKTA)
        text = text.replace("\u095b", "\u091c" + DevanagariNormalizer.NUKTA)
        text = text.replace("\u095c", "\u0921" + DevanagariNormalizer.NUKTA)
        text = text.replace("\u095d", "\u0922" + DevanagariNormalizer.NUKTA)
        text = text.replace("\u095e", "\u092b" + DevanagariNormalizer.NUKTA)
        text = text.replace("\u095f", "\u092f" + DevanagariNormalizer.NUKTA)

        if self.remove_nuktas:
            text = text.replace(DevanagariNormalizer.NUKTA, "")

        # replace pipe character for poorna virama
        text = text.replace("\u007c", "\u0964")

        # correct visarga
        text = re.sub(r"([\u0900-\u097f]):", "\\1\u0903", text)

        has_digits = any(char.isdigit() for char in text)
        if has_digits:
            # Extract parts containing digits
            digit_parts = re.findall(r"\d+", text)
            for part in digit_parts:
                text = text.replace(part, num2words(part, lang="hi"))

        return text

    def get_char_stats(self, text):
        super(DevanagariNormalizer, self).get_char_stats(text)

        print((len(re.findall("\u0929", text))))
        print((len(re.findall("\u0931", text))))
        print((len(re.findall("\u0934", text))))
        print((len(re.findall("\u0958", text))))
        print((len(re.findall("\u0959", text))))
        print((len(re.findall("\u095a", text))))
        print((len(re.findall("\u095b", text))))
        print((len(re.findall("\u095c", text))))
        print((len(re.findall("\u095d", text))))
        print((len(re.findall("\u095e", text))))
        print((len(re.findall("\u095f", text))))

# %% ../nbs/1b.indic_normalizer.ipynb 10
class HindiNormalizer(BaseNormalizer):
    """Fork of Devanagiri normalizer. With additional changes for Hindi and tts_mode."""

    NUKTA = "\u093c"

    def __init__(
        self,
        lang="hi",
        remove_nuktas=False,
        nasals_mode="do_nothing",
        do_normalize_chandras=False,
        do_normalize_vowel_ending=False,
        tts_mode=False,
    ):
        super(HindiNormalizer, self).__init__(
            lang,
            remove_nuktas,
            nasals_mode,
            do_normalize_chandras,
            do_normalize_vowel_ending,
        )
        self.tts_mode = tts_mode

    def __call__(self, text):
        # common normalization for Indic scripts
        text = super(HindiNormalizer, self).normalize(text)

        # chandra a replacement for Marathi
        text = text.replace("\u0972", "\u090f")

        # decomposing Nukta based composite characters
        text = text.replace("\u0929", "\u0928" + HindiNormalizer.NUKTA)
        text = text.replace("\u0931", "\u0930" + HindiNormalizer.NUKTA)
        text = text.replace("\u0934", "\u0933" + HindiNormalizer.NUKTA)
        text = text.replace("\u0958", "\u0915" + HindiNormalizer.NUKTA)
        text = text.replace("\u0959", "\u0916" + HindiNormalizer.NUKTA)
        text = text.replace("\u095a", "\u0917" + HindiNormalizer.NUKTA)
        text = text.replace("\u095b", "\u091c" + HindiNormalizer.NUKTA)
        text = text.replace("\u095c", "\u0921" + HindiNormalizer.NUKTA)
        text = text.replace("\u095d", "\u0922" + HindiNormalizer.NUKTA)
        text = text.replace("\u095e", "\u092b" + HindiNormalizer.NUKTA)
        text = text.replace("\u095f", "\u092f" + HindiNormalizer.NUKTA)

        if self.remove_nuktas:
            text = text.replace(HindiNormalizer.NUKTA, "")

        # replace pipe character for poorna virama
        text = text.replace("\u007c", "\u0964")
        # correct visarga
        text = re.sub(r"([\u0900-\u097f]):", "\\1\u0903", text)

        if self.tts_mode:
            # Handle currencies
            text = re.sub(r"INR\s+(\d+)", r"रुपये \1", text)
            text = re.sub(r"Rs\.\s+(\d+)", r"रुपये \1", text)
            text = re.sub(r"₹\s*(\d+)", r"रुपये \1", text)
            text = re.sub(r"USD\s+(\d+)", r"डॉलर \1", text)
            text = re.sub(r"\$\s*(\d+)", r"डॉलर \1", text)
            text = re.sub(r"KRW\s+(\d+)", r"कोरियाई वॉन \1", text)
            text = re.sub(r"₩\s*(\d+)", r"कोरियाई वॉन \1", text)

            # Handle decimal numbers
            def replace_decimal(match):
                whole, frac = match.group(1), match.group(2)
                # Convert both parts to Hindi words
                whole_words = num2words(whole, lang="hi")
                frac_words = " ".join(num2words(digit, lang="hi") for digit in frac)
                return f"{whole_words} पॉइंट {frac_words}"

            text = re.sub(r"(\d+)\.(\d+)", replace_decimal, text)

            # Normalize full URLs
            def normalize_url(match):
                url = match.group(0)
                scheme = ""
                if url.startswith("https://"):
                    scheme = "एच टी टी पी एस कोलन स्लैश स्लैश "
                    url = url[len("https://") :]
                elif url.startswith("http://"):
                    scheme = "एच टी टी पी कोलन स्लैश स्लैश "
                    url = url[len("http://") :]

                parts = url.split("/", 1)
                domain = parts[0]
                path = parts[1] if len(parts) > 1 else ""

                domain_parts = domain.split(".")
                spoken_domain = " डॉट ".join(
                    " ".join(part.upper()) for part in domain_parts
                )

                spoken_path = ""
                if path:
                    path_parts = path.split("/")
                    spoken_path = " स्लैश ".join(
                        " ".join(p.upper()) for p in path_parts if p
                    )
                    spoken_path = " स्लैश " + spoken_path if spoken_path else ""

                return f"{scheme}{spoken_domain}{spoken_path}".strip()

            text = re.sub(r"https?://[^\s]+", normalize_url, text)

            # Normalize bare domains like www.amazon.in, openai.com
            def normalize_bare_url(match):
                domain_parts = match.group(0).split(".")
                return " डॉट ".join(" ".join(part.upper()) for part in domain_parts)

            bare_url_tld_pattern = r"\b(?:www\.)?[\w-]+\.(?:com|in|org|net|edu|gov|ai|co|io|info|biz|nic\.in|ac\.in|gov\.in)\b"
            text = re.sub(
                bare_url_tld_pattern, normalize_bare_url, text, flags=re.IGNORECASE
            )

            # Email normalization
            text = re.sub(
                r"\b([\w\.-]+)@([\w\.-]+)\.(\w+)\b",
                lambda m: f"{m.group(1)} एट {m.group(2)} डॉट {m.group(3)}",
                text,
            )
            # Replace standalone special characters
            symbol_map = {
                "&": " or ",
                "@": " at ",
                "%": " percent ",
                "+": " plus ",
                "-": " minus ",
                "=": " equals ",
                "*": " star ",
                "#": " hash ",
            }
            for sym, word in symbol_map.items():
                text = text.replace(sym, word)

            def expand_acronyms(match):
                return " ".join(match.group(0))

            text = re.sub(r"\b[A-Z]{2,}\b", expand_acronyms, text)
            text = text.lower()

        has_digits = any(char.isdigit() for char in text)
        if has_digits:
            # digit_parts = re.findall(r"\d+", text)
            digit_parts = re.findall(r"\d+\.\d+|\d+", text)
            for x in digit_parts:
                text = text.replace(x, num2words(x, lang="hi"))
        return text

# %% ../nbs/1b.indic_normalizer.ipynb 17
class PunjabiNormalizer(BaseNormalizer):
    """
    Normalizer for the Gurmukhi script. In addition to basic normalization by the super class,
    * Replaces the composite characters containing nuktas by their decomposed form
    * Replace the reserved character for poorna virama (if used) with the recommended generic Indic scripts poorna virama
    * replace pipe character '|' by poorna virama character
    * replace colon ':' by visarga if the colon follows a charcter in this script
    """

    NUKTA = "\u0a3c"

    VOWEL_NORM_MAPS = {
        ## http://www.unicode.org/versions/Unicode12.1.0/ch12.pdf
        ## Table 12-16
        "\u0a05\u0a3e": "\u0a06",
        "\u0a72\u0a3f": "\u0a07",
        "\u0a72\u0a40": "\u0a08",
        "\u0a73\u0a41": "\u0a09",
        "\u0a73\u0a42": "\u0a0a",
        "\u0a72\u0a47": "\u0a0f",
        "\u0a05\u0a48": "\u0a10",
        "\u0a73\u0a4b": "\u0a13",
        "\u0a05\u0a4c": "\u0a14",
    }

    def __init__(
        self,
        lang="pa",
        remove_nuktas=False,
        nasals_mode="do_nothing",
        do_normalize_chandras=False,
        do_normalize_vowel_ending=False,
        do_canonicalize_addak=False,
        do_canonicalize_tippi=False,
        do_replace_vowel_bases=False,
        tts_mode=False,
    ):
        super(PunjabiNormalizer, self).__init__(
            lang,
            remove_nuktas,
            nasals_mode,
            do_normalize_chandras,
            do_normalize_vowel_ending,
        )
        self.do_canonicalize_addak = do_canonicalize_addak
        self.do_canonicalize_tippi = do_canonicalize_tippi
        self.do_replace_vowel_bases = do_replace_vowel_bases
        self.tts_mode = tts_mode

    def _normalize_vowels(self, text):
        """ """

        ## standard vowel replacements as per suggestions in
        ## http://www.unicode.org/versions/Unicode12.1.0/ch12.pdf
        ## Table 12-16

        for k, v in PunjabiNormalizer.VOWEL_NORM_MAPS.items():
            text = text.replace(k, v)

        ## the above mappings should account for majority of the variantions,
        ## Rest are handled via this generic rule which looks at the diacritic
        ## following the 2 special characters
        ## TBD: don't see evidence for this in Wikipedia corpus

        ## If these special characters occur without any diacritic, replace them with closet
        ## equivalent vowels
        if self.do_replace_vowel_bases:
            text = text.replace("\u0a72", "\u0a07")
            text = text.replace("\u0a73", "\u0a09")

        return text

    def __call__(self, text):
        # Addak
        if self.do_canonicalize_addak:
            ## replace addak+consonant with consonat+halant+consonant
            text = re.sub(r"\u0a71(.)", "\1\u0a4d\1", text)

        # Tippi
        if self.do_canonicalize_tippi:
            text = text.replace("\u0a70", "\u0a02")

        # Vowels: Gurumuki has multiple ways of representing independent vowels due
        # to the characters 'iri' and 'ura'.
        text = self._normalize_vowels(text)

        # common normalization for Indic scripts
        text = super(PunjabiNormalizer, self).normalize(text)

        # decomposing Nukta based composite characters
        text = text.replace("\u0a33", "\u0a32" + PunjabiNormalizer.NUKTA)
        text = text.replace("\u0a36", "\u0a38" + PunjabiNormalizer.NUKTA)
        text = text.replace("\u0a59", "\u0a16" + PunjabiNormalizer.NUKTA)
        text = text.replace("\u0a5a", "\u0a17" + PunjabiNormalizer.NUKTA)
        text = text.replace("\u0a5b", "\u0a1c" + PunjabiNormalizer.NUKTA)
        text = text.replace("\u0a5e", "\u0a2b" + PunjabiNormalizer.NUKTA)

        if self.remove_nuktas:
            text = text.replace(PunjabiNormalizer.NUKTA, "")

        # replace the poorna virama codes specific to script
        # with generic Indic script codes
        text = text.replace("\u0a64", "\u0964")
        text = text.replace("\u0a65", "\u0965")

        ## replace pipe character for poorna virama
        text = text.replace("\u007c", "\u0964")

        # correct visarge
        text = re.sub(r"([\u0a00-\u0a7f]):", "\1\u0a03", text)

        if self.tts_mode:
            # Handle currencies
            text = re.sub(r"INR\s+(\d+)", r"ਰੁਪਏ \1", text)
            text = re.sub(r"Rs\.\s+(\d+)", r"ਰੁਪਏ \1", text)
            text = re.sub(r"₹\s*(\d+)", r"ਰੁਪਏ \1", text)
            text = re.sub(r"USD\s+(\d+)", r"ਡਾਲਰ \1", text)
            text = re.sub(r"$\s*(\d+)", r"ਡਾਲਰ \1", text)
            text = re.sub(r"KRW\s+(\d+)", r"ਕੋਰੀਆਈ ਵੌਨ \1", text)
            text = re.sub(r"₩\s*(\d+)", r"ਕੋਰੀਆਈ ਵੌਨ \1", text)

            # Handle decimal numbers
            def replace_decimal(match):
                whole, frac = match.group(1), match.group(2)
                return f"{whole} ਪੌਇੰਟ {frac}"

            text = re.sub(r"(\d+)\.(\d+)", replace_decimal, text)

            # Normalize full URLs
            def normalize_url(match):
                url = match.group(0)
                scheme = ""
                if url.startswith("https://"):
                    scheme = "ਐਚ ਟੀ ਟੀ ਪੀ ਐੱਸ ਕੋਲਨ ਸਲੈਸ਼ ਸਲੈਸ਼ "
                    url = url[len("https://") :]
                elif url.startswith("http://"):
                    scheme = "ਐਚ ਟੀ ਟੀ ਪੀ ਕੋਲਨ ਸਲੈਸ਼ ਸਲੈਸ਼ "
                    url = url[len("http://") :]

                parts = url.split("/", 1)
                domain = parts[0]
                path = parts[1] if len(parts) > 1 else ""

                domain_parts = domain.split(".")
                spoken_domain = " ਡਾਟ ".join(
                    " ".join(part.upper()) for part in domain_parts
                )

                spoken_path = ""
                if path:
                    path_parts = path.split("/")
                    spoken_path = " ਸਲੈਸ਼ ".join(
                        " ".join(p.upper()) for p in path_parts if p
                    )
                    spoken_path = " ਸਲੈਸ਼ " + spoken_path if spoken_path else ""

                return f"{scheme}{spoken_domain}{spoken_path}".strip()

            text = re.sub(r"https?://[^\s]+", normalize_url, text)

            # Normalize bare domains (e.g., www.amazon.in)
            def normalize_bare_url(match):
                domain_parts = match.group(0).split(".")
                return " ਡਾਟ ".join(" ".join(part.upper()) for part in domain_parts)

            bare_url_tld_pattern = r"\b(?:www\.)?[\w-]+\.(?:com|in|org|net|edu|gov|ai|co|io|info|biz|nic\.in|ac\.in|gov\.in)\b"
            text = re.sub(
                bare_url_tld_pattern, normalize_bare_url, text, flags=re.IGNORECASE
            )

            # Email normalization
            text = re.sub(
                r"\b([\w\.-]+)@([\w\.-]+)\.(\w+)\b",
                lambda m: f"{m.group(1)} ਐਟ {m.group(2)} ਡਾਟ {m.group(3)}",
                text,
            )

            # Replace standalone special characters
            symbol_map = {
                "&": " or ",
                "@": " at ",
                "%": " percent ",
                "+": " plus ",
                "-": " minus ",
                "=": " equals ",
                "*": " star ",
                "#": " hash ",
            }
            for sym, word in symbol_map.items():
                text = text.replace(sym, word)

            def expand_acronyms(match):
                return " ".join(match.group(0))

            text = re.sub(r"\b[A-Z]{2,}\b", expand_acronyms, text)

        has_digits = any(char.isdigit() for char in text)
        if has_digits:
            # digit_parts = re.findall(r"\d+", text)
            digit_parts = re.findall(r"\d+\.\d+|\d+", text)
            for x in digit_parts:
                text = text.replace(x, num2words(x, lang="pa"))
        return text

# %% ../nbs/1b.indic_normalizer.ipynb 20
class TeluguNormalizer(BaseNormalizer):
    """
    Normalizer for the Teluguscript. In addition to basic normalization by the super class,
    * Replace the reserved character for poorna virama (if used) with the recommended generic Indic scripts poorna virama
    * canonicalize two-part dependent vowel signs
    * replace colon ':' by visarga if the colon follows a charcter in this script
    """

    def __init__(
        self,
        lang="te",
        remove_nuktas=False,
        nasals_mode="do_nothing",
        do_normalize_chandras=False,
        do_normalize_vowel_ending=False,
        tts_mode=False,
    ):
        super(TeluguNormalizer, self).__init__(
            lang,
            remove_nuktas,
            nasals_mode,
            do_normalize_chandras,
            do_normalize_vowel_ending,
        )
        self.tts_mode = tts_mode

    def __call__(self, text: str):
        # common normalization for Indic scripts
        text = super(TeluguNormalizer, self).normalize(text)

        # replace the poorna virama codes specific to script
        # with generic Indic script codes
        text = text.replace("\u0c64", "\u0964")
        text = text.replace("\u0c65", "\u0965")

        # dependent vowels
        text = text.replace("\u0c46\u0c56", "\u0c48")

        # correct visarge
        text = re.sub(r"([\u0c00-\u0c7f]):", "\1\u0c03", text)

        if self.tts_mode:
            # Handle currencies
            text = re.sub(r"INR\s+(\d+)", r"రూపాయలు \1", text)
            text = re.sub(r"Rs\.\s+(\d+)", r"రూపాయలు \1", text)
            text = re.sub(r"₹\s*(\d+)", r"రూపాయలు \1", text)
            text = re.sub(r"USD\s+(\d+)", r"డాలర్లు \1", text)
            text = re.sub(r"$\s*(\d+)", r"డాలర్లు \1", text)
            text = re.sub(r"KRW\s+(\d+)", r"కొరియన్ వాన్ \1", text)
            text = re.sub(r"₩\s*(\d+)", r"కొరియన్ వాన్ \1", text)

            # Handle decimal numbers
            def replace_decimal(match):
                whole, frac = match.group(1), match.group(2)
                return f"{whole} పాయింట్ {frac}"

            text = re.sub(r"(\d+)\.(\d+)", replace_decimal, text)

            # Normalize full URLs
            def normalize_url(match):
                url = match.group(0)
                scheme = ""
                if url.startswith("https://"):
                    scheme = "హెచ్ టి టి పి ఎస్ కొలన్ స్లాష్ స్లాష్ "
                    url = url[len("https://") :]
                elif url.startswith("http://"):
                    scheme = "హెచ్ టి టి పి కొలన్ స్లాష్ స్లాష్ "
                    url = url[len("http://") :]

                parts = url.split("/", 1)
                domain = parts[0]
                path = parts[1] if len(parts) > 1 else ""

                domain_parts = domain.split(".")
                spoken_domain = " డాట్ ".join(
                    " ".join(part.upper()) for part in domain_parts
                )

                spoken_path = ""
                if path:
                    path_parts = path.split("/")
                    spoken_path = " స్లాష్ ".join(
                        " ".join(p.upper()) for p in path_parts if p
                    )
                    spoken_path = " స్లాష్ " + spoken_path if spoken_path else ""

                return f"{scheme}{spoken_domain}{spoken_path}".strip()

            text = re.sub(r"https?://[^\s]+", normalize_url, text)

            # Normalize bare domain names
            def normalize_bare_url(match):
                domain_parts = match.group(0).split(".")
                return " డాట్ ".join(" ".join(part.upper()) for part in domain_parts)

            bare_url_tld_pattern = r"\b(?:www\.)?[\w-]+\.(?:com|in|org|net|edu|gov|ai|co|io|info|biz|nic\.in|ac\.in|gov\.in)\b"
            text = re.sub(
                bare_url_tld_pattern, normalize_bare_url, text, flags=re.IGNORECASE
            )

            # Email normalization
            text = re.sub(
                r"\b([\w\.-]+)@([\w\.-]+)\.(\w+)\b",
                lambda m: f"{m.group(1)} అట్ {m.group(2)} డాట్ {m.group(3)}",
                text,
            )
            # Replace standalone special characters
            symbol_map = {
                "&": " or ",
                "@": " at ",
                "%": " percent ",
                "+": " plus ",
                "-": " minus ",
                "=": " equals ",
                "*": " star ",
                "#": " hash ",
            }
            for sym, word in symbol_map.items():
                text = text.replace(sym, word)

            def expand_acronyms(match):
                return " ".join(match.group(0))

            text = re.sub(r"\b[A-Z]{2,}\b", expand_acronyms, text)

        has_digits = any(char.isdigit() for char in text)
        if has_digits:
            digit_parts = re.findall(r"\d+", text)
            for part in digit_parts:
                text = text.replace(part, num2words(part, lang="te"))

        return text.lower()

# %% ../nbs/1b.indic_normalizer.ipynb 23
class GujaratiNormalizer(BaseNormalizer):
    """
    Normalizer for the Gujarati script. In addition to basic normalization by the super class,
    * Replace the reserved character for poorna virama (if used) with the recommended generic Indic scripts poorna virama
    * replace colon ':' by visarga if the colon follows a charcter in this script
    """

    NUKTA = "\u0abc"

    def __init__(
        self,
        lang="gu",
        remove_nuktas=False,
        nasals_mode="do_nothing",
        do_normalize_chandras=False,
        do_normalize_vowel_ending=False,
        tts_mode=False,
    ):
        super(GujaratiNormalizer, self).__init__(
            lang,
            remove_nuktas,
            nasals_mode,
            do_normalize_chandras,
            do_normalize_vowel_ending,
        )
        self.tts_mode = tts_mode

    def __call__(self, text: str):
        # common normalization for Indic scripts
        text = super(GujaratiNormalizer, self).normalize(text)

        # decomposing Nukta based composite characters
        if self.remove_nuktas:
            text = text.replace(GujaratiNormalizer.NUKTA, "")

        # replace the poorna virama codes specific to script
        # with generic Indic script codes
        text = text.replace("\u0ae4", "\u0964")
        text = text.replace("\u0ae5", "\u0965")

        # correct visarge
        text = re.sub(r"([\u0a80-\u0aff]):", "\\1\u0a83", text)

        if self.tts_mode:
            # Handle currencies
            text = re.sub(r"INR\s+(\d+)", r"રૂપિયા \1", text)
            text = re.sub(r"Rs\.\s+(\d+)", r"રૂપિયા \1", text)
            text = re.sub(r"₹\s*(\d+)", r"રૂપિયા \1", text)
            text = re.sub(r"USD\s+(\d+)", r"ડોલર \1", text)
            text = re.sub(r"\$\s*(\d+)", r"ડોલર \1", text)
            text = re.sub(r"KRW\s+(\d+)", r"કોરિયન વોન \1", text)
            text = re.sub(r"₩\s*(\d+)", r"કોરિયન વોન \1", text)

            # Handle decimal numbers
            def replace_decimal(match):
                whole, frac = match.group(1), match.group(2)
                return f"{whole} પોઈન્ટ {frac}"

            text = re.sub(r"(\d+)\.(\d+)", replace_decimal, text)

            # Normalize full URLs
            def normalize_url(match):
                url = match.group(0)
                scheme = ""
                if url.startswith("https://"):
                    scheme = "એચ ટી ટી પીએસ કોલન સ્લેશ સ્લેશ "
                    url = url[len("https://") :]
                elif url.startswith("http://"):
                    scheme = "એચ ટી ટી પી કોલન સ્લેશ સ્લેશ "
                    url = url[len("http://") :]

                parts = url.split("/", 1)
                domain = parts[0]
                path = parts[1] if len(parts) > 1 else ""

                domain_parts = domain.split(".")
                spoken_domain = " ડોટ ".join(
                    " ".join(part.upper()) for part in domain_parts
                )

                spoken_path = ""
                if path:
                    path_parts = path.split("/")
                    spoken_path = " સ્લેશ ".join(
                        " ".join(p.upper()) for p in path_parts if p
                    )
                    spoken_path = " સ્લેશ " + spoken_path if spoken_path else ""

                return f"{scheme}{spoken_domain}{spoken_path}".strip()

            text = re.sub(r"https?://[^\s]+", normalize_url, text)

            # Normalize bare domain names
            def normalize_bare_url(match):
                domain_parts = match.group(0).split(".")
                return " ડોટ ".join(" ".join(part.upper()) for part in domain_parts)

            bare_url_tld_pattern = r"\b(?:www\.)?[\w-]+\.(?:com|in|org|net|edu|gov|ai|co|io|info|biz|nic\.in|ac\.in|gov\.in)\b"
            text = re.sub(
                bare_url_tld_pattern, normalize_bare_url, text, flags=re.IGNORECASE
            )

            # Normalize email addresses
            text = re.sub(
                r"\b([\w\.-]+)@([\w\.-]+)\.(\w+)\b",
                lambda m: f"{m.group(1)} એટ {m.group(2)} ડોટ {m.group(3)}",
                text,
            )

            symbol_map = {
                "&": " or ",
                "@": " at ",
                "%": " percent ",
                "+": " plus ",
                "-": " minus ",
                "=": " equals ",
                "*": " star ",
                "#": " hash ",
            }
            for sym, word in symbol_map.items():
                text = text.replace(sym, word)

            def expand_acronyms(match):
                return " ".join(match.group(0))

            text = re.sub(r"\b[A-Z]{2,}\b", expand_acronyms, text)

        has_digits = any(char.isdigit() for char in text)
        if has_digits:
            digit_parts = re.findall(r"\d+", text)
            for part in digit_parts:
                text = text.replace(part, num2words(part, lang="gu"))

        return text.lower()

# %% ../nbs/1b.indic_normalizer.ipynb 25
class OdiaNormalizer(BaseNormalizer):
    """
    Normalizer for the Oriya script. In addition to basic normalization by the super class,
    * Replaces the composite characters containing nuktas by their decomposed form
    * Replace the reserved character for poorna virama (if used) with the recommended generic Indic scripts poorna virama
    * Canonicalize two part dependent vowels
    * Replace 'va' with 'ba'
    * replace pipe character '|' by poorna virama character
    * replace colon ':' by visarga if the colon follows a charcter in this script
    """

    NUKTA = "\u0b3c"

    VOWEL_NORM_MAPS = {
        ## See Table 12-22 in http://www.unicode.org/versions/Unicode12.1.0/ch12.pdf
        "\u0b05\u0b3e": "\u0b06",
        "\u0b0f\u0b57": "\u0b10",
        "\u0b13\u0b57": "\u0b14",
    }

    def __init__(
        self,
        lang="or",
        remove_nuktas=False,
        nasals_mode="do_nothing",
        do_normalize_chandras=False,
        do_normalize_vowel_ending=False,
        do_remap_wa=False,
        tts_mode=False,
    ):
        super(OdiaNormalizer, self).__init__(
            lang,
            remove_nuktas,
            nasals_mode,
            do_normalize_chandras,
            do_normalize_vowel_ending,
        )
        self.do_remap_wa = do_remap_wa
        self.tts_mode = tts_mode

    def __call__(self, text: str):
        # common normalization for Indic scripts
        text = super(OdiaNormalizer, self).normalize(text)

        ## standard vowel replacements as per suggestions in Unicode documents
        for k, v in OdiaNormalizer.VOWEL_NORM_MAPS.items():
            text = text.replace(k, v)

        # decomposing Nukta based composite characters
        text = text.replace("\u0b5c", "\u0b21" + OdiaNormalizer.NUKTA)
        text = text.replace("\u0b5d", "\u0b22" + OdiaNormalizer.NUKTA)

        if self.remove_nuktas:
            text = text.replace(OdiaNormalizer.NUKTA, "")

        # replace the poorna virama codes specific to script
        # with generic Indic script codes
        text = text.replace("\u0b64", "\u0964")
        text = text.replace("\u0b65", "\u0965")

        # replace pipe character for poorna virama
        text = text.replace("\u0b7c", "\u0964")

        # replace wa with ba
        if self.do_remap_wa:
            text = text.replace("\u0b71", "\u0b2c")

        # replace va with ba
        # NOTE: documentation (chapter on Indic scripts) and codepoint chart seem contradictory
        # (this applied to wa to ba rule also above)
        text = text.replace("\u0b35", "\u0b2c")

        # AI dependent vowel sign
        text = text.replace("\u0b47\u0b56", "\u0b58")

        # two part dependent vowels
        text = text.replace("\u0b47\u0b3e", "\u0b4b")
        text = text.replace("\u0b47\u0b57", "\u0b4c")

        # additional consonant - not clear how to handle this
        # ignore

        # correct visarge
        text = re.sub(r"([\u0b00-\u0b7f]):", "\\1\u0b03", text)

        if self.tts_mode:
            # Handle currencies
            text = re.sub(r"INR\s+(\d+)", r"ଟଙ୍କା \1", text)
            text = re.sub(r"Rs\.\s+(\d+)", r"ଟଙ୍କା \1", text)
            text = re.sub(r"₹\s*(\d+)", r"ଟଙ୍କା \1", text)
            text = re.sub(r"USD\s+(\d+)", r"ଡଲାର \1", text)
            text = re.sub(r"\$\s*(\d+)", r"ଡଲାର \1", text)
            text = re.sub(r"KRW\s+(\d+)", r"କୋରିଆନ୍ ୱନ୍ \1", text)
            text = re.sub(r"₩\s*(\d+)", r"କୋରିଆନ୍ ୱନ୍ \1", text)

            # Handle decimal numbers
            def replace_decimal(match):
                whole, frac = match.group(1), match.group(2)
                return f"{whole} ପଏଣ୍ଟ {frac}"

            text = re.sub(r"(\d+)\.(\d+)", replace_decimal, text)

            def normalize_url(match):
                url = match.group(0)
                scheme = ""
                if url.startswith("https://"):
                    scheme = "ଏଚ୍ ଟି ଟି ପି ଏସ୍ କୋଲନ୍ ସ୍ଲାଶ୍ ସ୍ଲାଶ୍ "
                    url = url[len("https://") :]
                elif url.startswith("http://"):
                    scheme = "ଏଚ୍ ଟି ଟି ପି କୋଲନ୍ ସ୍ଲାଶ୍ ସ୍ଲାଶ୍ "
                    url = url[len("http://") :]

                parts = url.split("/", 1)
                domain = parts[0]
                path = parts[1] if len(parts) > 1 else ""

                domain_parts = domain.split(".")
                spoken_domain = " ଡଟ୍ ".join(
                    " ".join(part.upper()) for part in domain_parts
                )

                spoken_path = ""
                if path:
                    path_parts = path.split("/")
                    spoken_path = " ସ୍ଲାଶ୍ ".join(
                        " ".join(p.upper()) for p in path_parts if p
                    )
                    spoken_path = " ସ୍ଲାଶ୍ " + spoken_path if spoken_path else ""

                return f"{scheme}{spoken_domain}{spoken_path}".strip()

            text = re.sub(r"https?://[^\s]+", normalize_url, text)

            # Normalize bare domains like www.amazon.in, openai.com
            def normalize_bare_url(match):
                domain_parts = match.group(0).split(".")
                return " ଡଟ୍ ".join(" ".join(part.upper()) for part in domain_parts)

            bare_url_tld_pattern = r"\b(?:www\.)?[\w-]+\.(?:com|in|org|net|edu|gov|ai|co|io|info|biz|nic\.in|ac\.in|gov\.in)\b"
            text = re.sub(
                bare_url_tld_pattern, normalize_bare_url, text, flags=re.IGNORECASE
            )

            # Email normalization
            text = re.sub(
                r"\b([\w\.-]+)@([\w\.-]+)\.(\w+)\b",
                lambda m: f"{m.group(1)} ଏଟ୍ {m.group(2)} ଡଟ୍ {m.group(3)}",
                text,
            )

            symbol_map = {
                "&": " or ",
                "@": " at ",
                "%": " percent ",
                "+": " plus ",
                "-": " minus ",
                "=": " equals ",
                "*": " star ",
                "#": " hash ",
            }

            for sym, word in symbol_map.items():
                text = text.replace(sym, word)

            def expand_acronyms(match):
                return " ".join(match.group(0))

            text = re.sub(r"\b[A-Z]{2,}\b", expand_acronyms, text)

        has_digits = any(char.isdigit() for char in text)
        if has_digits:
            digit_parts = re.findall(r"\d+", text)
            for part in digit_parts:
                text = text.replace(part, num2words(part, lang="or"))

        return text.lower()

# %% ../nbs/1b.indic_normalizer.ipynb 27
class BengaliNormalizer(BaseNormalizer):
    """
    Normalizer for the Bengali script. In addition to basic normalization by the super class,
    * Replaces the composite characters containing nuktas by their decomposed form
    * Replace the reserved character for poorna virama (if used) with the recommended generic Indic scripts poorna virama
    * Canonicalize two part dependent vowels
    * replace pipe character '|' by poorna virama character
    * replace colon ':' by visarga if the colon follows a charcter in this script
    """

    NUKTA = "\u09bc"

    def __init__(
        self,
        lang="bn",
        remove_nuktas=False,
        nasals_mode="do_nothing",
        do_normalize_chandras=False,
        do_normalize_vowel_ending=False,
        do_remap_assamese_chars=False,
        tts_mode=False,
    ):
        super(BengaliNormalizer, self).__init__(
            lang,
            remove_nuktas,
            nasals_mode,
            do_normalize_chandras,
            do_normalize_vowel_ending,
        )
        self.do_remap_assamese_chars = do_remap_assamese_chars
        self.tts_mode = True

    def __call__(self, text: str):
        # common normalization for Indic scripts
        text = super(BengaliNormalizer, self).normalize(text)

        # decomposing Nukta based composite characters
        text = text.replace("\u09dc", "\u09a1" + BengaliNormalizer.NUKTA)
        text = text.replace("\u09dd", "\u09a2" + BengaliNormalizer.NUKTA)
        text = text.replace("\u09df", "\u09af" + BengaliNormalizer.NUKTA)

        if self.remove_nuktas:
            text = text.replace(BengaliNormalizer.NUKTA, "")

        if self.do_remap_assamese_chars and self.lang == "as":
            text = text.replace("\u09f0", "\u09b0")  #  'ra' character
            text = text.replace("\u09f1", "\u09ac")  #  'va' character

        # replace the poorna virama codes specific to script
        # with generic Indic script codes
        text = text.replace("\u09e4", "\u0964")
        text = text.replace("\u09e5", "\u0965")

        # replace pipe character for poorna virama
        text = text.replace("\u007c", "\u0964")
        # replace bengali currency numerator four for poorna virama  (it looks similar and is used as a substitute)
        text = text.replace("\u09f7", "\u0964")

        # two part dependent vowels
        text = text.replace("\u09c7\u09be", "\u09cb")
        text = text.replace("\u09c7\u09d7", "\u09cc")

        # correct visarge
        text = re.sub(r"([\u0980-\u09ff]):", "\\1\u0983", text)

        if self.tts_mode:
            text = re.sub(r"INR\s+(\d+)", r"রুপি \1", text)
            text = re.sub(r"Rs\.\s+(\d+)", r"রুপি \1", text)
            text = re.sub(r"₹\s*(\d+)", r"রুপি \1", text)
            text = re.sub(r"USD\s+(\d+)", r"ডলার \1", text)
            text = re.sub(r"\$\s*(\d+)", r"ডলার \1", text)
            text = re.sub(r"KRW\s+(\d+)", r"কোরিয়ান ওন \1", text)
            text = re.sub(r"₩\s*(\d+)", r"কোরিয়ান ওন \1", text)

            # Handle decimal numbers
            def replace_decimal(match):
                whole, frac = match.group(1), match.group(2)
                return f"{whole} পয়েন্ট {frac}"

            text = re.sub(r"(\d+)\.(\d+)", replace_decimal, text)

            # Full URL normalization
            def normalize_url(match):
                url = match.group(0)
                scheme = ""
                if url.startswith("https://"):
                    scheme = "এইচ টি টি পি এস কোলন স্ল্যাশ স্ল্যাশ "
                    url = url[len("https://") :]
                elif url.startswith("http://"):
                    scheme = "এইচ টি টি পি কোলন স্ল্যাশ স্ল্যাশ "
                    url = url[len("http://") :]
                parts = url.split("/", 1)
                domain = parts[0]
                path = parts[1] if len(parts) > 1 else ""

                domain_parts = domain.split(".")
                spoken_domain = " ডট ".join(
                    " ".join(part.upper()) for part in domain_parts
                )

                spoken_path = ""
                if path:
                    path_parts = path.split("/")
                    spoken_path = " স্ল্যাশ ".join(
                        " ".join(p.upper()) for p in path_parts if p
                    )
                    spoken_path = " স্ল্যাশ " + spoken_path if spoken_path else ""

                return f"{scheme}{spoken_domain}{spoken_path}".strip()

            text = re.sub(r"https?://[^\s]+", normalize_url, text)

            # Bare domain normalization
            def normalize_bare_url(match):
                domain_parts = match.group(0).split(".")
                return " ডট ".join(" ".join(part.upper()) for part in domain_parts)

            bare_url_tld_pattern = r"\b(?:www\.)?[\w-]+\.(?:com|in|org|net|edu|gov|ai|co|io|info|biz|nic\.in|ac\.in|gov\.in)\b"
            text = re.sub(
                bare_url_tld_pattern, normalize_bare_url, text, flags=re.IGNORECASE
            )

            # Emails
            text = re.sub(
                r"\b([\w\.-]+)@([\w\.-]+)\.(\w+)\b",
                lambda m: f"{m.group(1)} অ্যাট {m.group(2)} ডট {m.group(3)}",
                text,
            )
            # Replace standalone special characters
            symbol_map = {
                "&": " or ",
                "@": " at ",
                "%": " percent ",
                "+": " plus ",
                "-": " minus ",
                "=": " equals ",
                "*": " star ",
                "#": " hash ",
            }

            for sym, word in symbol_map.items():
                text = text.replace(sym, word)

            def expand_acronyms(match):
                return " ".join(match.group(0))

            text = re.sub(r"\b[A-Z]{2,}\b", expand_acronyms, text)

        has_digits = any(char.isdigit() for char in text)
        if has_digits:
            digit_parts = re.findall(r"\d+", text)
            for part in digit_parts:
                text = text.replace(part, num2words(part, lang="bn"))

        return text.lower()

# %% ../nbs/1b.indic_normalizer.ipynb 29
class TamilNormalizer(BaseNormalizer):
    """
    Normalizer for the Tamil script. In addition to basic normalization by the super class,
    * Replace the reserved character for poorna virama (if used) with the recommended generic Indic scripts poorna virama
    * canonicalize two-part dependent vowel signs
    * replace colon ':' by visarga if the colon follows a charcter in this script
    """

    def __init__(
        self,
        lang="ta",
        remove_nuktas=False,
        nasals_mode="do_nothing",
        do_normalize_chandras=False,
        do_normalize_vowel_ending=False,
        tts_mode=False,
    ):
        super(TamilNormalizer, self).__init__(
            lang,
            remove_nuktas,
            nasals_mode,
            do_normalize_chandras,
            do_normalize_vowel_ending,
        )
        self.tts_mode = tts_mode

    def __call__(self, text: str):
        # common normalization for Indic scripts
        text = super(TamilNormalizer, self).normalize(text)

        # replace the poorna virama codes specific to script
        # with generic Indic script codes
        text = text.replace("\u0be4", "\u0964")
        text = text.replace("\u0be5", "\u0965")

        # two part dependent vowels
        text = text.replace("\u0b92\u0bd7", "\u0b94")
        text = text.replace("\u0bc6\u0bbe", "\u0bca")
        text = text.replace("\u0bc7\u0bbe", "\u0bcb")
        text = text.replace("\u0bc6\u0bd7", "\u0bcc")

        # correct visarge
        text = re.sub(r"([\u0b80-\u0bff]):", "\\1\u0b83", text)

        if self.tts_mode:
            # Handle currencies
            text = re.sub(r"INR\s+(\d+)", r"ரூபாய் \1", text)
            text = re.sub(r"Rs\.\s+(\d+)", r"ரூபாய் \1", text)
            text = re.sub(r"₹\s*(\d+)", r"ரூபாய் \1", text)
            text = re.sub(r"USD\s+(\d+)", r"டாலர் \1", text)
            text = re.sub(r"\$\s*(\d+)", r"டாலர் \1", text)
            text = re.sub(r"KRW\s+(\d+)", r"கொரிய வான் \1", text)
            text = re.sub(r"₩\s*(\d+)", r"கொரிய வான் \1", text)

            # Handle decimal numbers
            def replace_decimal(match):
                whole, frac = match.group(1), match.group(2)
                return f"{whole} பாயிண்ட் {frac}"

            text = re.sub(r"(\d+)\.(\d+)", replace_decimal, text)

            # Normalize full URLs
            def normalize_url(match):
                url = match.group(0)
                scheme = ""
                if url.startswith("https://"):
                    scheme = "எச் டி டி பி எஸ் கோலன் ஸ்லாஷ் ஸ்லாஷ் "
                    url = url[len("https://") :]
                elif url.startswith("http://"):
                    scheme = "எச் டி டி பி கோலன் ஸ்லாஷ் ஸ்லாஷ் "
                    url = url[len("http://") :]

                parts = url.split("/", 1)
                domain = parts[0]
                path = parts[1] if len(parts) > 1 else ""

                domain_parts = domain.split(".")
                spoken_domain = " டாட் ".join(
                    " ".join(part.upper()) for part in domain_parts
                )

                spoken_path = ""
                if path:
                    path_parts = path.split("/")
                    spoken_path = " ஸ்லாஷ் ".join(
                        " ".join(p.upper()) for p in path_parts if p
                    )
                    spoken_path = " ஸ்லாஷ் " + spoken_path if spoken_path else ""

                return f"{scheme}{spoken_domain}{spoken_path}".strip()

            text = re.sub(r"https?://[^\s]+", normalize_url, text)

            # Normalize bare domains (www.amazon.in)
            def normalize_bare_url(match):
                domain_parts = match.group(0).split(".")
                return " டாட் ".join(" ".join(part.upper()) for part in domain_parts)

            bare_url_tld_pattern = r"\b(?:www\.)?[\w-]+\.(?:com|in|org|net|edu|gov|ai|co|io|info|biz|nic\.in|ac\.in|gov\.in)\b"
            text = re.sub(
                bare_url_tld_pattern, normalize_bare_url, text, flags=re.IGNORECASE
            )

            # Emails
            text = re.sub(
                r"\b([\w\.-]+)@([\w\.-]+)\.(\w+)\b",
                lambda m: f"{m.group(1)} அட் {m.group(2)} டாட் {m.group(3)}",
                text,
            )

            # Replace standalone special characters
            symbol_map = {
                "&": " or ",
                "@": " at ",
                "%": " percent ",
                "+": " plus ",
                "-": " minus ",
                "=": " equals ",
                "*": " star ",
                "#": " hash ",
            }

            for sym, word in symbol_map.items():
                text = text.replace(sym, word)

            def expand_acronyms(match):
                return " ".join(match.group(0))

            text = re.sub(r"\b[A-Z]{2,}\b", expand_acronyms, text)

        has_digits = any(char.isdigit() for char in text)
        if has_digits:
            digit_parts = re.findall(r"\d+", text)
            for part in digit_parts:
                text = text.replace(part, num2words(part, lang="ta"))

        return text.lower()

# %% ../nbs/1b.indic_normalizer.ipynb 32
class KannadaNormalizer(BaseNormalizer):
    """
    Normalizer for the Kannada script. In addition to basic normalization by the super class,
    * Replace the reserved character for poorna virama (if used) with the recommended generic Indic scripts poorna virama
    * canonicalize two-part dependent vowel signs
    * replace colon ':' by visarga if the colon follows a charcter in this script
    """

    def __init__(
        self,
        lang="kn",
        remove_nuktas=False,
        nasals_mode="do_nothing",
        do_normalize_chandras=False,
        do_normalize_vowel_ending=False,
        tts_mode=False,
    ):
        super(KannadaNormalizer, self).__init__(
            lang,
            remove_nuktas,
            nasals_mode,
            do_normalize_chandras,
            do_normalize_vowel_ending,
        )
        self.tts_mode = tts_mode

    def __call__(self, text: str):
        # common normalization for Indic scripts
        text = super(KannadaNormalizer, self).normalize(text)

        # replace the poorna virama codes specific to script
        # with generic Indic script codes
        text = text.replace("\u0ce4", "\u0964")
        text = text.replace("\u0ce5", "\u0965")

        # dependent vowels
        text = text.replace("\u0cbf\u0cd5", "\u0cc0")
        text = text.replace("\u0cc6\u0cd5", "\u0cc7")
        text = text.replace("\u0cc6\u0cd6", "\u0cc8")
        text = text.replace("\u0cc6\u0cc2", "\u0cca")
        text = text.replace("\u0cca\u0cd5", "\u0ccb")

        # correct visarge
        text = re.sub(r"([\u0c80-\u0cff]):", "\\1\u0c83", text)

        if self.tts_mode:
            # Handle currencies
            text = re.sub(r"INR\s+(\d+)", r"ರೂಪಾಯಿ \1", text)
            text = re.sub(r"Rs\.\s+(\d+)", r"ರೂಪಾಯಿ \1", text)
            text = re.sub(r"₹\s*(\d+)", r"ರೂಪಾಯಿ \1", text)
            text = re.sub(r"USD\s+(\d+)", r"ಡಾಲರ್ \1", text)
            text = re.sub(r"\$\s*(\d+)", r"ಡಾಲರ್ \1", text)
            text = re.sub(r"KRW\s+(\d+)", r"ಕೊರಿಯನ್ ವಾನ್ \1", text)
            text = re.sub(r"₩\s*(\d+)", r"ಕೊರಿಯನ್ ವಾನ್ \1", text)

            # Handle decimal numbers
            def replace_decimal(match):
                whole, frac = match.group(1), match.group(2)
                return f"{whole} ಪಾಯಿಂಟ್ {frac}"

            text = re.sub(r"(\d+)\.(\d+)", replace_decimal, text)

            # Normalize full URLs
            def normalize_url(match):
                url = match.group(0)
                scheme = ""
                if url.startswith("https://"):
                    scheme = "ಎಚ್ ಟಿ ಟಿ ಪಿ ಎಸ್ ಕೋಲನ್ ಸ್ಲ್ಯಾಶ್ ಸ್ಲ್ಯಾಶ್ "
                    url = url[len("https://") :]
                elif url.startswith("http://"):
                    scheme = "ಎಚ್ ಟಿ ಟಿ ಪಿ ಕೋಲನ್ ಸ್ಲ್ಯಾಶ್ ಸ್ಲ್ಯಾಶ್ "
                    url = url[len("http://") :]

                parts = url.split("/", 1)
                domain = parts[0]
                path = parts[1] if len(parts) > 1 else ""

                domain_parts = domain.split(".")
                spoken_domain = " ಡಾಟ್ ".join(
                    " ".join(part.upper()) for part in domain_parts
                )

                spoken_path = ""
                if path:
                    path_parts = path.split("/")
                    spoken_path = " ಸ್ಲ್ಯಾಶ್ ".join(
                        " ".join(p.upper()) for p in path_parts if p
                    )
                    spoken_path = " ಸ್ಲ್ಯಾಶ್ " + spoken_path if spoken_path else ""

                return f"{scheme}{spoken_domain}{spoken_path}".strip()

            text = re.sub(r"https?://[^\s]+", normalize_url, text)

            # Normalize bare domains like www.amazon.in
            def normalize_bare_url(match):
                url = match.group(0)
                domain_parts = url.split(".")
                spoken = " ಡಾಟ್ ".join(" ".join(part.upper()) for part in domain_parts)
                return spoken

            bare_url_tld_pattern = r"\b(?:www\.)?[\w-]+\.(?:com|in|org|net|edu|gov|ai|co|io|info|biz|nic\.in|ac\.in|gov\.in)\b"
            text = re.sub(
                bare_url_tld_pattern, normalize_bare_url, text, flags=re.IGNORECASE
            )

            # Normalize email
            text = re.sub(
                r"\b([\w\.-]+)@([\w\.-]+)\.(\w+)\b",
                lambda m: f"{m.group(1)} ಅಟ್ {m.group(2)} ಡಾಟ್ {m.group(3)}",
                text,
            )

            # Replace standalone special characters
            symbol_map = {
                "&": " or ",
                "@": " at ",
                "%": " percent ",
                "+": " plus ",
                "-": " minus ",
                "=": " equals ",
                "*": " star ",
                "#": " hash ",
            }
            for sym, word in symbol_map.items():
                text = text.replace(sym, word)

            def expand_acronyms(match):
                return " ".join(match.group(0))

            text = re.sub(r"\b[A-Z]{2,}\b", expand_acronyms, text)

        has_digits = any(char.isdigit() for char in text)
        if has_digits:
            digit_parts = re.findall(r"\d+", text)
            for part in digit_parts:
                text = text.replace(part, num2words(part, lang="kn"))

        return text.lower()

# %% ../nbs/1b.indic_normalizer.ipynb 34
class MalayalamNormalizer(BaseNormalizer):
    """
    Normalizer for the Malayalam script. In addition to basic normalization by the super class,
    * Replace the reserved character for poorna virama (if used) with the recommended generic Indic scripts poorna virama
    * canonicalize two-part dependent vowel signs
    * Change from old encoding of chillus (till Unicode 5.0) to new encoding
    * replace colon ':' by visarga if the colon follows a charcter in this script
    """

    CHILLU_CHAR_MAP = {
        "\u0d7a": "\u0d23",
        "\u0d7b": "\u0d28",
        "\u0d7c": "\u0d30",
        "\u0d7d": "\u0d32",
        "\u0d7e": "\u0d33",
        "\u0d7f": "\u0d15",
    }

    def _canonicalize_chillus(self, text):
        for chillu, char in MalayalamNormalizer.CHILLU_CHAR_MAP.items():
            text = text.replace(chillu, "{}\u0d4d".format(char))
        return text

    def _correct_geminated_T(self, text):
        return text.replace("\u0d31\u0d4d\u0d31", "\u0d1f\u0d4d\u0d1f")

    def __init__(
        self,
        lang="ml",
        remove_nuktas=False,
        nasals_mode="do_nothing",
        do_normalize_chandras=False,
        do_normalize_vowel_ending=False,
        do_canonicalize_chillus=False,
        do_correct_geminated_T=False,
        tts_mode=False,
    ):
        super(MalayalamNormalizer, self).__init__(
            lang,
            remove_nuktas,
            nasals_mode,
            do_normalize_chandras,
            do_normalize_vowel_ending,
        )
        self.do_canonicalize_chillus = do_canonicalize_chillus
        self.do_correct_geminated_T = do_correct_geminated_T
        self.tts_mode = tts_mode

    def __call__(self, text: str):
        # Change from old encoding of chillus (till Unicode 5.0) to new encoding
        text = text.replace("\u0d23\u0d4d\u200d", "\u0d7a")
        text = text.replace("\u0d28\u0d4d\u200d", "\u0d7b")
        text = text.replace("\u0d30\u0d4d\u200d", "\u0d7c")
        text = text.replace("\u0d32\u0d4d\u200d", "\u0d7d")
        text = text.replace("\u0d33\u0d4d\u200d", "\u0d7e")
        text = text.replace("\u0d15\u0d4d\u200d", "\u0d7f")

        # Normalize chillus
        if self.do_canonicalize_chillus:
            text = self._canonicalize_chillus(text)

        # common normalization for Indic scripts
        text = super(MalayalamNormalizer, self).normalize(text)

        # replace the poorna virama codes specific to script
        # with generic Indic script codes
        text = text.replace("\u0d64", "\u0964")
        text = text.replace("\u0d65", "\u0965")

        # dependent vowels
        text = text.replace("\u0d46\u0d3e", "\u0d4a")
        text = text.replace("\u0d47\u0d3e", "\u0d4b")

        # au forms
        text = text.replace("\u0d46\u0d57", "\u0d4c")
        text = text.replace("\u0d57", "\u0d4c")

        # correct geminated T
        if self.do_correct_geminated_T:
            text = self._correct_geminated_T(text)

        # correct visarga
        text = re.sub(r"([\u0d00-\u0d7f]):", "\\1\u0d03", text)
        # remove samvruthokaram
        text = text.replace("\u0d41\u0d4d", "\u0d4d")

        if self.tts_mode:
            # Handle currencies
            text = re.sub(r"INR\s+(\d+)", r"രൂപ \1", text)
            text = re.sub(r"Rs\.\s+(\d+)", r"രൂപ \1", text)
            text = re.sub(r"₹\s*(\d+)", r"രൂപ \1", text)
            text = re.sub(r"USD\s+(\d+)", r"ഡോളർ \1", text)
            text = re.sub(r"\$\s*(\d+)", r"ഡോളർ \1", text)
            text = re.sub(r"KRW\s+(\d+)", r"കൊറിയൻ വോൺ \1", text)
            text = re.sub(r"₩\s*(\d+)", r"കൊറിയൻ വോൺ \1", text)

            # Decimal numbers
            def replace_decimal(match):
                whole, frac = match.group(1), match.group(2)
                return f"{whole} പോയിന്റ് {frac}"

            text = re.sub(r"(\d+)\.(\d+)", replace_decimal, text)

            # Normalize full URLs (http/https)
            def normalize_url(match):
                url = match.group(0)
                scheme = ""
                if url.startswith("https://"):
                    scheme = "എച്ച് ടി ടി പി എസ് കോളൺ സ്ലാഷ് സ്ലാഷ് "
                    url = url[len("https://") :]
                elif url.startswith("http://"):
                    scheme = "എച്ച് ടി ടി പി കോളൺ സ്ലാഷ് സ്ലാഷ് "
                    url = url[len("http://") :]
                parts = url.split("/", 1)
                domain = parts[0]
                path = parts[1] if len(parts) > 1 else ""
                domain_parts = domain.split(".")
                spoken_domain = " ഡോട്ട് ".join(
                    " ".join(part.upper()) for part in domain_parts
                )
                spoken_path = ""
                if path:
                    path_parts = path.split("/")
                    spoken_path = " സ്ലാഷ് ".join(
                        " ".join(p.upper()) for p in path_parts if p
                    )
                    spoken_path = " സ്ലാഷ് " + spoken_path if spoken_path else ""
                return f"{scheme}{spoken_domain}{spoken_path}".strip()

            text = re.sub(r"https?://[^\s]+", normalize_url, text)

            # Normalize bare domain URLs (like www.amazon.in)
            def normalize_bare_url(match):
                url = match.group(0)
                domain_parts = url.split(".")
                spoken = " ഡോട്ട് ".join(" ".join(part.upper()) for part in domain_parts)
                return spoken

            bare_url_tld_pattern = r"\b(?:www\.)?[\w-]+\.(?:com|in|org|net|edu|gov|ai|co|io|info|biz|nic\.in|ac\.in|gov\.in)\b"
            text = re.sub(
                bare_url_tld_pattern, normalize_bare_url, text, flags=re.IGNORECASE
            )

            # Normalize emails
            text = re.sub(
                r"\b([\w\.-]+)@([\w\.-]+)\.(\w+)\b",
                lambda m: f"{m.group(1)} അറ്റ് {m.group(2)} ഡോട്ട് {m.group(3)}",
                text,
            )

            # Replace special characters
            symbol_map = {
                "&": " or ",
                "@": " at ",
                "%": " percent ",
                "+": " plus ",
                "-": " minus ",
                "=": " equals ",
                "*": " star ",
                "#": " hash ",
            }
            for sym, word in symbol_map.items():
                text = text.replace(sym, word)

            def expand_acronyms(match):
                return " ".join(match.group(0))

            text = re.sub(r"\b[A-Z]{2,}\b", expand_acronyms, text)

        has_digits = any(char.isdigit() for char in text)
        if has_digits:
            digit_parts = re.findall(r"\d+", text)
            for part in digit_parts:
                text = text.replace(part, num2words(part, lang="ml"))

        return text.lower()
