# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/02_text_transformation.ipynb.

# %% ../nbs/02_text_transformation.ipynb 3
from __future__ import annotations
from underthesea import word_tokenize, sent_tokenize, text_normalize

# %% auto 0
__all__ = ['apply_vnmese_word_tokenize']

# %% ../nbs/02_text_transformation.ipynb 4
def apply_vnmese_word_tokenize(sentence:str, # Input sentence
                        normalize_text=False, # To 'normalize' the text before tokenization
                        fixed_words=[]
                       ):
    "Applying UnderTheSea Vietnamese word tokenization"
    if normalize_text:
        sentence = text_normalize(sentence)
    sens = sent_tokenize(sentence)

    tokenized_sen = []
    for sen in sens:
        tokenized_sen.append(word_tokenize(sen,format='text',fixed_words=fixed_words))
    return ' '.join(tokenized_sen)
