#TODO: Expand contractions in French starting with l' and taking gender into account

from aruana import stopwords
from collections import Counter
from nltk.stem.snowball import SnowballStemmer
from nltk.tokenize import word_tokenize
import re
from string import punctuation as pct
from tqdm import tqdm
import unicodedata

class Aruana:
    """Aruana is a collection of methods that can be used for simple NLP tasks.
    It can be used for tasks involving text preprocessing for machine learning.
    Aruana works mainly with text strings and lists of strings. 

    It requires NLTK and TQDM to be installed.

        Attributes
        ----------
        language : str
            The language used to initialize Aruana
        nltk_language : str
            The language used for NLTK, generated by the method language_for_nltk
        vocab : Counter()
            The number of unique tokens found on the text

        Methods
        -------
        language_for_nltk()
            Converts the language inputed during Aruana init to a format that can be used by NLTK
        
        remove_html_tags(text:str)
            Removes html tags from text

        replace_html_tags(text:str, placeholder:str)
            Replaces html tags from text by a given placeholder
        
        remove_urls(text:str)
            Removes urls from text
        
        replace_urls(text:str, placeholder:str)
            Replaces links from text by a given placeholder
        
        remove_hashtags(text:str)
            Removes hashtags from text
        
        replace_hashtags(text:str, placeholder:str)
            Replaces hashtags from text by a given placeholder
        
        remove_ips(text:str)
            Removes ips from text

        replace_ips(text:str, placeholder:str)
            Replaces ips from text by a given placeholder

        remove_handles(text:str)
            Removes handles from text

        replace_handles(text:str, placeholder:str)
            Replaces handles from text by a given placeholder

        strip_accents(text:str)
            Strip accents from tokens
        
        replace_quotes(text:str, placeholder:str)
            Replaces quotes by a placeholder

        remove_punctuation(text:str)
            Removes ponctuation and special chars
        
        replace_punctuation(text:str, sign='', placeholder='')
            Replaces punctuation by a placeholder
        
        remove_numbers(text:str)
            Removes all numbers

        replace_numbers(text:str)
            Replaces all numbers for string

        stem_sentence(text:str)
            Uses NLTK stemmer to stem tokens

        remove_stopwords(text:str, custom_list=[], extend_set=False)
            Removes stopwords

        reduce_words_with_repeated_chars(text:str)
            Reduces words with chars repeated more than 3 times to a single char. 
            Useful to replace words such as loooooooong by long. Be careful, 
            as it can change abreviations such as AAA to single A 
        
        remove_excessive_spaces(text:str)
            Removes excessive space from text

        merge_same_tokens(text:str)
            If two tokens are the same, this will merge them. 
            Specially useful when dealing with social media text.

        expand_contractions(text:str)
            Expands some of the most popular English contractions

        lower_remove_white(text:str)
            Use it to lower text and to remove training whitespaces

        tokenize(text:str):
            Tokenizes text using Word Tokenizer. Attention: this method returns a list instead of a string.

        preprocess(text:str, tokenize=True, stem=True, remove_stopwords=False)
            Preprocess text before data handling with most common settings.
            Text is processed in the following order:

            - lower text
            - strip trailing whitespaces
            - merge same words
            - expand contractions
            - replace urls
            - remove tags
            - replace hashtags
            - replace ips
            - remove social media handles
            - remove stopwords if True
            - remove accents
            - remove punctuation        
            - replace numbers
            - reduce loooooong words
            - remove excessive spaces
            - stem text if True
            - tokenize text (if True, will return a list of tokens)
        
        preprocess_list(text_list:list, tokenize=True, stem=True, remove_stopwords=False)
            Preprocess every text in a list of strings

        create_corpus(text_list:list)
            Receives a list of strings and returns a single string with all the text

        vocab_size(corpus:str)
            Returns the number of tokens found on the corpus

        lexical_diversity(corpus_slice:str, corpus:str)
            Counts the vocab of one part of the corpus and compares it to the entire corpus    
    """

    def __init__(self, language:str):
        """
        Parameters
        ----------
        language : str
            The language used to initialize Aruana. Accepts 'pt-br', 'en' and 'fr'
        nltk_language : str
            The language used for NLTK, generated by the method language_for_nltk
        vocab : Counter()
            The number of unique tokens found on the text
        """
        self.language = language
        self.nltk_language = self.language_for_nltk()
        self.vocab = Counter()

    def language_for_nltk(self):
        """ Converts the language inputed during Aruana init to a format that can be used by NLTK
        
        >>> language_for_nltk('pt-br')
        'portuguese'
        
        """
        if self.language == 'pt-br':
            return 'portuguese'
        if self.language == 'en':
            return 'english'
        if self.language == 'fr':
            return 'french'
        elif self.language == 'custom':
            return 'portuguese'

    def remove_html_tags(self, text:str):
        """Removes html tags from text

        >>> remove_html_tags('<h1><strong>Hello world!</strong></h1>')
        'Hello world!'

        Parameters
        ----------
        text : str
            The string that will be transformed
        """
        text = re.sub(r'<[^>]*>', '', text)
        return text

    def replace_html_tags(self, text:str, placeholder:str):
        """Replaces html tags from text by a given placeholder
        
        >>> replace_html_tags('<h1><strong>Hello world!</strong></h1>', 'HTML')
        'HTML HTML Hello world! HTML HTML'

        Parameters
        ----------
        text : str
            The string that will be transformed
        placeholder : str
            The string that will replace the html tag
        """
        text = re.sub(r'<[^>]*>', ' ' + placeholder + ' ', text)
        text = self.remove_excessive_spaces(text)
        return text

    def remove_urls(self, text:str):
        """Removes urls from text

        >>> remove_urls('You can go to the page http://homepage.com to see the content.')
        'You can go to the page to see the content.'
        
        Parameters
        ----------
        text : str
            The string that will be transformed
        """
        text = re.sub(r'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', '', text)
        text = self.remove_excessive_spaces(text)
        return text

    def replace_urls(self, text:str, placeholder:str):
        """Replaces links from text by a given placeholder
        
        >>> replace_urls('You can go to the page http://homepage.com to see the content.', 'URL')
        'You can go to the page URL to see the content.'

        Parameters
        ----------
        text : str
            The string that will be transformed
        placeholder : str
            The string that will replace the url
        """
        text = re.sub(r'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', placeholder, text)
        return text

    def remove_hashtags(self, text:str):
        """Removes hashtags from text
        
        >>> remove_hashtags('I wish you all #love and #peanceandlove')
        'I wish you all and !'

        Parameters
        ----------
        text : str
            The string that will be transformed
        """
        text = re.sub(r'\B(\#[a-zA-Z]+\b)(?!;)', '', text)
        text = self.remove_excessive_spaces(text)
        return text

    def replace_hashtags(self, text:str, placeholder:str):
        """Replaces hashtags from text by a given placeholder

        >>> replace_hashtags('I wish you all #love and #peace', 'HASHTAG')
        'I wish you all HASHTAG and HASHTAG'
        
        Parameters
        ----------
        text : str
            The string that will be transformed
        placeholder : str
            The string that will replace the hashtag
        """
        text = re.sub(r'\B(\#[a-zA-Z]+\b)(?!;)', placeholder, text)
        return text

    def remove_ips(self, text:str):
        """Removes ips from text

        >>> remove_ips('This can be accessed on the address 198.162.0.1.')
        'This can be accessed on the address .'

        Parameters
        ----------
        text : str
            The string that will be transformed
        """
        text = re.sub(r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}', '', text)
        text = self.remove_excessive_spaces(text)
        return text

    def replace_ips(self, text:str, placeholder:str):
        """Replaces ips from text by a given placeholder

        >>> replace_ips('This can be accessed on the address 198.162.0.1.', 'IP')
        'This can be accessed on the address IP.'

        Parameters
        ----------
        text : str
            The string that will be transformed
        placeholder : str
            The string that will replace the ip
        """
        text = re.sub(r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}', placeholder, text)
        return text

    def remove_handles(self, text:str):
        """Removes handles from text

        >>> remove_handles('Can you come tonight, @alice?')
        'Can you come tonight, ?'

        Parameters
        ----------
        text : str
            The string that will be transformed
        """
        text = re.sub(r'\B(\@[a-zA-Z_0-9]+\b)(?!;)', '', text)
        text = self.remove_excessive_spaces(text)
        return text

    def replace_handles(self, text:str, placeholder:str):
        """Replaces handles from text by a given placeholder

        >>> replace_handles('Can you come tonight, @alice?', 'USERNAME')
        'Can you come tonight, USERNAME?'

        Parameters
        ----------
        text : str
            The string that will be transformed
        placeholder : str
            The string that will replace the handle
        """
        text = re.sub(r'\B(\@[a-zA-Z_0-9]+\b)(?!;)', placeholder, text)
        return text

    def strip_accents(self, text:str):
        """Strip accents from tokens

        >>> strip_accents('Mamãe me disse: você é o menino mais ágil do time.')
        'Mamae me disse: voce e o menino mais agil do time.'
        
        Parameters
        ----------
        text : str
            The string that will be transformed
        """
        try:
            text = unicode(text, 'utf-8')
        except NameError: 
            pass

        text = unicodedata.normalize('NFD', text)\
            .encode('ascii', 'ignore')\
            .decode("utf-8")

        return str(text)

    def replace_quotes(self, text:str, placeholder:str):
        """Replaces quotes by a placeholder

        >>> replace_quotes('She told me: "you are a confident and strong woman".', 'QUOTED')
        'She told me: QUOTED you are a confident and strong woman QUOTED .'

        Parameters
        ----------
        text : str
            The string that will be transformed
        placeholder : str
            The string that will replace the quote sign
        """
        text = re.sub(r'"', ' ' + placeholder + ' ', text)
        text = re.sub(r"'", ' ' + placeholder + ' ', text)
        text = self.remove_excessive_spaces(text)
        return text

    def remove_punctuation(self, text:str):
        """Removes ponctuation and special chars

        >>> remove_punctuation('Hey, are you here??? I really need your help!')
        'Hey are you here I really need your help'
        
        Parameters
        ----------
        text : str
            The string that will be transformed
        placeholder : str
            The string that will replace the punctuation and the special chars found
        """
        text = re.sub(r'[^\w\s]', ' ', text)
        text = self.remove_excessive_spaces(text)
        return text

    def replace_punctuation(self, text:str, sign='', placeholder=''):
        """Replaces punctuation by a placeholder.

        >>> replace_punctuation('Hey, are you here??? I really need your help!', sign='?', placeholder='QUESTION')
        'Hey, are you here QUESTION QUESTION QUESTION I really need your help!'
        
        Parameters
        ----------
        text : str
            The string that will be transformed
        sign : str
            The symbol you want to replace. If no symbol is provided, will replace all special chars and punctuation by the placeholder
        placeholder : str
            The string that will replace the punctuation and the special chars found
        """
        if sign is '':
            text = re.sub(r'[^\w\s]', ' ' + placeholder + ' ', text)    
        else:
            text = text.replace(sign, ' ' + placeholder + ' ')
        
        text = self.remove_excessive_spaces(text)
        return text

    def remove_numbers(self, text:str):
        """Removes all numbers

        >>> remove_numbers('I told him 1,2,3,4 times that he could not do that!')
        'I told him ,,, times that he could not do that!'

        Parameters
        ----------
        text : str
            The string that will be transformed
        """
        text = re.sub(r'[0-9]', '', text)
        text = self.remove_excessive_spaces(text)
        return text

    def replace_numbers(self, text:str):
        """Replaces all numbers for string

        >>> replace_numbers('I told him 1,2,3,4 times that he could not do that!')
        'I told him one, two, three, four times that he could not do that!'. 
        Attention: It does NOT convert numbers like 20 to twenty, but to
        two zero.

        Parameters
        ----------
        text : str
            The string that will be transformed
        """
        if self.language == 'pt-br':
            text = text.replace('0', ' zero ')
            text = text.replace('1', ' um ')
            text = text.replace('2', ' dois ')
            text = text.replace('3', ' três ')
            text = text.replace('4', ' quatro ')
            text = text.replace('5', ' cinco ')
            text = text.replace('6', ' seis ')
            text = text.replace('7', ' sete ')
            text = text.replace('8', ' oito ')
            text = text.replace('9', ' nove ')
        elif self.language == 'fr':
            text = text.replace('0', ' zéro ')
            text = text.replace('1', ' un ')
            text = text.replace('2', ' deux ')
            text = text.replace('3', ' trois ')
            text = text.replace('4', ' quatre ')
            text = text.replace('5', ' cinc ')
            text = text.replace('6', ' six ')
            text = text.replace('7', ' sept ')
            text = text.replace('8', ' huit ')
            text = text.replace('9', ' neuf ')
        elif self.language == 'en':
            text = text.replace('0', ' zero ')
            text = text.replace('1', ' one ')
            text = text.replace('2', ' two ')
            text = text.replace('3', ' three ')
            text = text.replace('4', ' four ')
            text = text.replace('5', ' five ')
            text = text.replace('6', ' six ')
            text = text.replace('7', ' seven ')
            text = text.replace('8', ' eight ')
            text = text.replace('9', ' nine ')
            
        text = self.remove_excessive_spaces(text)
        return text

    def stem_sentence(self, text:str):
        """Uses NLTK stemmer to stem tokens

        >>> stem_sentence('I love to go shopping with my mother and friends')
        'i love to go shop with my mother and friend'

        Parameters
        ----------
        text : str
            The string that will be transformed
        """
        stemmer = SnowballStemmer(self.nltk_language)
        sentence = []

        # removes more than one whitespace 
        text = re.sub(r'\s{2,}', ' ', text)

        for word in text.split(' '):
            word = stemmer.stem(str(word))
            sentence.append(word)

        text = ' '.join(sentence)
        return text

    def remove_stopwords(self, text:str, custom_list=[], extend_set=False):
        """ Removes stopwords

        >>> remove_stopwords('I love to go shopping with my mother and friends')
        'love go shopping mother friends'

        Parameters
        ----------
        text : str
            The string that will be transformed
        custom_list : list
            A list with the custom stopwords list
        extend_set : boolean
            Use this option to extend the stopwords of a given language with your custom list 
        """
        # define the stopwords set to use
        if self.language == 'pt-br':
            stop_words = stopwords.portuguese
        elif self.language == 'en':
            stop_words = stopwords.english
        elif self.language == 'fr':
            stop_words = stopwords.french
        elif self.language == 'custom':
            stop_words = custom_list
        else:
            print('This set is not available. No stopword will be removed!')
            stop_words = []

        # extend with a list of custom words
        if extend_set == True:
            stop_words = stop_words + custom_list

        # add spaces to the beginning and to the end of the sentence, 
        # so stopwords on these locations can be spotted
        text = ' ' + text + ' '

        for word in stop_words:
            text = re.sub(r'\s' + word + r'\s', ' ', text, flags=re.I)
            if self.language == 'fr':
                text = re.sub(re.escape('j\''), ' ', text, flags=re.I)
                text = re.sub(re.escape('d\''), ' ', text, flags=re.I)
                text = re.sub(re.escape('l\''), ' ', text, flags=re.I)
                text = re.sub(re.escape('m\''), ' ', text, flags=re.I)
                text = re.sub(re.escape('s\''), ' ', text, flags=re.I)
                text = re.sub(re.escape('t\''), ' ', text, flags=re.I)
            
        # strip adititonal the spaces
        text = text.strip()
        text = self.remove_excessive_spaces(text)
        return text

    def reduce_words_with_repeated_chars(self, text:str):
        """Reduces words with chars repeated more than 3 times to a single char. 
        Useful to replace words such as loooooooong by long. Be careful, 
        as it can change abreviations such as AAA to single A 

        >>> reduce_words_with_repeated_chars('I loooooooooooooove pizza so muuuchhhh')
        'I love pizza so much'

        Parameters
        ----------
        text : str
            The string that will be transformed
        """
        findings = re.findall(r'(\w)\1{2,}', text)
        for char in findings:
            find = char + '{3,}'
            replace = char + '\1' + '???'
            text = re.sub(find, replace, text)

            # doing this replacement will cause an error during tokenization, as the replace chars
            # have been mapped as punctuation. We have to remove these marks and replace them by nothing
            text = text.translate(pct)
            text = text.replace('"???','')

        text = self.remove_excessive_spaces(text)
        return text
        
    def remove_excessive_spaces(self, text:str):
        """Removes excessive space from text

        >>> remove_excessive_spaces('I  can\'t     stop looking at  you')
        'I can't stop looking at you'

        Parameters
        ----------
        text : str
            The string that will be transformed
        """
        # remove more than one space
        text = re.sub(r'(\s)\1{1,}', ' ', text)
        # remove spaces in the beginning and in the end of the string
        text = text.strip()
        return text

    def merge_same_tokens(self, text:str):
        """If two tokens are the same, this will merge them. Specially useful when dealing with social
        media text

        >>> merge_same_tokens("eu acho q vc tb estava lá")
        'eu acho que você também estava lá'

        Parameters
        ----------
        text : str
            The string that will be transformed
        """

        if self.language == 'pt-br':
            text = ' ' + text + ' '
            text = re.sub(re.escape(" abs "), " abraços ", text, flags=re.I)
            text = re.sub(re.escape(" agr "), " agora ", text, flags=re.I)
            text = re.sub(re.escape(" amg "), " amigo ", text, flags=re.I)
            text = re.sub(re.escape(" bb "), " bebê ", text, flags=re.I)
            text = re.sub(re.escape(" bj "), " beijo ", text, flags=re.I)
            text = re.sub(re.escape(" bjs "), " beijos ", text, flags=re.I)
            text = re.sub(re.escape(" blz "), " beleza ", text, flags=re.I)
            text = re.sub(re.escape(" bpn "), " bom para nós ", text, flags=re.I)
            text = re.sub(re.escape(" brb "), " volto já ", text, flags=re.I)
            text = re.sub(re.escape(" cmg "), " comigo ", text, flags=re.I)
            text = re.sub(re.escape(" cm "), " com ", text, flags=re.I)
            text = re.sub(re.escape(" cvs "), " conversa ", text, flags=re.I)
            text = re.sub(re.escape(" d+ "), " demais ", text, flags=re.I)
            text = re.sub(re.escape(" dlc "), " delícia ", text, flags=re.I)
            text = re.sub(re.escape(" dlç "), " delícia ", text, flags=re.I)
            text = re.sub(re.escape(" dm "), " mensagem privada ", text, flags=re.I)
            text = re.sub(re.escape(" dmr "), " demorou ", text, flags=re.I)
            text = re.sub(re.escape(" dms "), " demais ", text, flags=re.I)
            text = re.sub(re.escape(" doq "), " do que ", text, flags=re.I)
            text = re.sub(re.escape(" erridê "), " rodado ", text, flags=re.I)
            text = re.sub(re.escape(" erride "), " rodado ", text, flags=re.I)
            text = re.sub(re.escape(" fdp "), " filho da puta ", text, flags=re.I)
            text = re.sub(re.escape(" fdpta "), " filho da puta ", text, flags=re.I)
            text = re.sub(re.escape(" glr "), " galera ", text, flags=re.I)
            text = re.sub(re.escape(" hj "), " hoje ", text, flags=re.I)
            text = re.sub(re.escape(" ja "), " já ", text, flags=re.I)
            text = re.sub(re.escape(" k "), " risos ", text, flags=re.I)
            text = re.sub(re.escape(" kg "), " quilo ", text, flags=re.I)
            text = re.sub(re.escape(" kgs "), " quilos ", text, flags=re.I)
            text = re.sub(re.escape(" kk "), " risos ", text, flags=re.I)
            text = re.sub(re.escape(" lol "), " risos ", text, flags=re.I)
            text = re.sub(re.escape(" miga "), " amiga ", text, flags=re.I)
            text = re.sub(re.escape(" migo "), " amigo ", text, flags=re.I)
            text = re.sub(re.escape(" mlc "), " maluco ", text, flags=re.I)
            text = re.sub(re.escape(" mp "), " ministério público ", text, flags=re.I)
            text = re.sub(re.escape(" msm "), " mesmo ", text, flags=re.I)
            text = re.sub(re.escape(" nº "), " número ", text, flags=re.I)
            text = re.sub(re.escape(" n° "), " número ", text, flags=re.I)
            text = re.sub(re.escape(" n "), " não ", text, flags=re.I)
            text = re.sub(re.escape(" ng "), " ninguém ", text, flags=re.I)
            text = re.sub(re.escape(" np "), " sem problemas ", text, flags=re.I)
            text = re.sub(re.escape(" ngm "), " ninguém ", text, flags=re.I)
            text = re.sub(re.escape(" obg "), " obrigado ", text, flags=re.I)
            text = re.sub(re.escape(" omg "), " meu deus ", text, flags=re.I)
            text = re.sub(re.escape(" oq "), " o que ", text, flags=re.I)
            text = re.sub(re.escape(" pfv "), " por favor ", text, flags=re.I)
            text = re.sub(re.escape(" pls "), " por favor ", text, flags=re.I)
            text = re.sub(re.escape(" plmns "), " pelo menos ", text, flags=re.I)
            text = re.sub(re.escape(" pdc "), " pode crer ", text, flags=re.I)
            text = re.sub(re.escape(" pprt "), " papo reto ", text, flags=re.I)
            text = re.sub(re.escape(" pq "), " porque ", text, flags=re.I)
            text = re.sub(re.escape(" pqp "), " puta que pariu ", text, flags=re.I)
            text = re.sub(re.escape(" pra "), " para ", text, flags=re.I)
            text = re.sub(re.escape(" pro "), " para o ", text, flags=re.I)
            text = re.sub(re.escape(" q "), " que ", text, flags=re.I)
            text = re.sub(re.escape(" qd "), " quando ", text, flags=re.I)
            text = re.sub(re.escape(" qdo "), " quando ", text, flags=re.I)
            text = re.sub(re.escape(" rd "), " rodado ", text, flags=re.I)
            text = re.sub(re.escape(" rs "), " risos ", text, flags=re.I)
            text = re.sub(re.escape(" sdd "), " saudade ", text, flags=re.I)
            text = re.sub(re.escape(" sdds "), " saudades ", text, flags=re.I)
            text = re.sub(re.escape(" sfd "), " safado ", text, flags=re.I)
            text = re.sub(re.escape(" slc "), " você é louco ", text, flags=re.I)
            text = re.sub(re.escape(" smdd "), " sem maldade ", text, flags=re.I)
            text = re.sub(re.escape(" so "), " só ", text, flags=re.I)
            text = re.sub(re.escape(" smp "), " sempre ", text, flags=re.I)
            text = re.sub(re.escape(" sqn "), " só que não ", text, flags=re.I)
            text = re.sub(re.escape(" sr "), " senhor ", text, flags=re.I)
            text = re.sub(re.escape(" tá "), " está ", text, flags=re.I)
            text = re.sub(re.escape(" ta "), " está ", text, flags=re.I)
            text = re.sub(re.escape(" tava "), " estava ", text, flags=re.I)
            text = re.sub(re.escape(" tb "), " também ", text, flags=re.I)
            text = re.sub(re.escape(" tbm "), " também ", text, flags=re.I)
            text = re.sub(re.escape(" td "), " tudo ", text, flags=re.I)
            text = re.sub(re.escape(" tdb "), " tudo bom ", text, flags=re.I)
            text = re.sub(re.escape(" tds "), " todos ", text, flags=re.I)
            text = re.sub(re.escape(" tlg "), " está ligado ", text, flags=re.I)
            text = re.sub(re.escape(" tmj "), " estamos juntos ", text, flags=re.I)
            text = re.sub(re.escape(" tô "), " estou ", text, flags=re.I)
            text = re.sub(re.escape(" to "), " estou ", text, flags=re.I)
            text = re.sub(re.escape(" tqr "), " tem que respeitar ", text, flags=re.I)
            text = re.sub(re.escape(" vdb "), " vai dar bom ", text, flags=re.I)
            text = re.sub(re.escape(" vei "), " velho ", text, flags=re.I)
            text = re.sub(re.escape(" vc "), " você ", text, flags=re.I)
            text = re.sub(re.escape(" vcs "), " vocês ", text, flags=re.I)
            text = re.sub(re.escape(" wpp "), " whatsapp ", text, flags=re.I)
            
            text = text.strip()
            
        return text

    def expand_contractions(self, text:str):
        """ Expands some of the most popular English contractions
        
        >>> expand_contractions("you're solely responsible for your success and your failure.")
        'you are solely responsible for your success and your failure.'

        Parameters
        ----------
        text : str
            The string that will be transformed
        """

        if self.language == 'en':
           # Expand contractions
            text = ' ' + text + ' '
            text = re.sub(re.escape(" i'm "), " I am ", text, flags=re.I)
            text = re.sub(re.escape(" i'm'a "), " I am about to ", text, flags=re.I)
            text = re.sub(re.escape(" i'm'o "), " I am going to ", text, flags=re.I)
            text = re.sub(re.escape(" ive "), " I have ", text, flags=re.I)
            text = re.sub(re.escape(" he's "), " he is ", text, flags=re.I)
            text = re.sub(re.escape(" she's "), " she is ", text, flags=re.I)
            text = re.sub(re.escape(" that's "), " that is ", text, flags=re.I)
            text = re.sub(re.escape(" what's "), " what is ", text, flags=re.I)
            text = re.sub(re.escape(" where's "), " where is ", text, flags=re.I)
            text = re.sub(re.escape(" haven't "), " have not ", text, flags=re.I)
            text = re.sub(re.escape(" mustn't've "), " must not have ", text, flags=re.I)
            text = re.sub(re.escape(" shouldn't've "), " should not have ", text, flags=re.I)
            text = re.sub(re.escape(" shouldn't "), " should not ", text, flags=re.I)
            text = re.sub(re.escape(" ur "), " you are ", text, flags=re.I)
            text = re.sub(re.escape(" somebody's "), " somebody is ", text, flags=re.I)
            text = re.sub(re.escape(" someone's "), " someone is ", text, flags=re.I)
            text = re.sub(re.escape(" something's "), " something is ", text, flags=re.I)
            text = re.sub(re.escape(" that's "), " that is ", text, flags=re.I)
            text = re.sub(re.escape(" 'tis "), " it is ", text, flags=re.I)
            text = re.sub(re.escape(" 'twas "), " it was ", text, flags=re.I)
            text = re.sub(re.escape(" wasn't "), " was not ", text, flags=re.I)
            text = re.sub(re.escape(" we'd've "), " we would have ", text, flags=re.I)
            text = re.sub(re.escape(" weren't "), " were not ", text, flags=re.I)
            text = re.sub(re.escape(" when's "), " when is ", text, flags=re.I)
            text = re.sub(re.escape(" who's "), " who is ", text, flags=re.I)
            text = re.sub(re.escape(" there's "), " there is ", text, flags=re.I)
            text = re.sub(re.escape(" why's "), " why is ", text, flags=re.I)
            text = re.sub(re.escape(" let's "), " let us ", text, flags=re.I)
            text = re.sub(re.escape(" how'd "), " how did ", text, flags=re.I)
            text = re.sub(re.escape(" needn't "), " need not ", text, flags=re.I)
            text = re.sub(re.escape(" ne'er "), " never ", text, flags=re.I)
            text = re.sub(re.escape(" o'clock "), " of the clock ", text, flags=re.I)
            text = re.sub(re.escape(" o'er "), " over ", text, flags=re.I)
            text = re.sub(re.escape(" o'l "), " old ", text, flags=re.I)
            text = re.sub(re.escape(" y'all "), " you all ", text, flags=re.I)
            text = re.sub(re.escape(" shan't "), " shall not ", text, flags=re.I)
            text = re.sub(re.escape("\'ll"), " will", text, flags=re.I)
            text = re.sub(re.escape("\'d\'ve"), " would have", text, flags=re.I)
            text = re.sub(re.escape("\'ve"), " have", text, flags=re.I)
            text = re.sub(re.escape("\'re"), " are", text, flags=re.I)
            text = re.sub(re.escape("\'d"), " would", text, flags=re.I)
            text = re.sub(re.escape(" won't "), " will not ", text, flags=re.I)
            text = re.sub(re.escape(" mayn't "), " may not ", text, flags=re.I)
            text = re.sub(re.escape(" mightn't "), " might not ", text, flags=re.I)
            text = re.sub(re.escape(" mustn't "), " must not ", text, flags=re.I)
            text = re.sub(re.escape(" aren't "), " are not ", text, flags=re.I)
            text = re.sub(re.escape(" isn't "), " is not ", text, flags=re.I)
            text = re.sub(re.escape(" wouldn't "), " would not ", text, flags=re.I)
            text = re.sub(re.escape(" can't "), " can not ", text, flags=re.I)
            text = re.sub(re.escape(" cannot "), " can not ", text, flags=re.I)
            text = re.sub(re.escape(" don't "), " do not ", text, flags=re.I)
            text = re.sub(re.escape(" didn't "), " did not ", text, flags=re.I)
            text = re.sub(re.escape(" doesn't "), " does not ", text, flags=re.I)
            text = re.sub(re.escape(" it's "), " it is ", text, flags=re.I)
            text = re.sub(re.escape(" weren't "), " were not ", text, flags=re.I)
            text = re.sub(re.escape(" okay "), " ok ", text, flags=re.I)
            text = re.sub(re.escape(" c'mon "), " come on ", text, flags=re.I)
            text = re.sub(re.escape("in'"), "ing", text, flags=re.I)
            text = re.sub(re.escape(" everyone's "), " everyone is ", text, flags=re.I)
            text = re.sub(re.escape("\'s"), " s", text, flags=re.I)
            text = re.sub(re.escape(" ain't "), " am not ", text, flags=re.I)
            text = re.sub(re.escape(" amn't "), " am not ", text, flags=re.I)
            text = re.sub(re.escape(" cain't "), " can not ", text, flags=re.I)
            text = re.sub(re.escape(" 'cause "), " because ", text, flags=re.I)
            text = re.sub(re.escape(" could've "), " could have ", text, flags=re.I)
            text = re.sub(re.escape(" couldn't've "), " could not have ", text, flags=re.I)
            text = re.sub(re.escape(" daren't "), " dare not ", text, flags=re.I)
            text = re.sub(re.escape(" daresn't "), " dare not ", text, flags=re.I)
            text = re.sub(re.escape(" e'er "), " ever ", text, flags=re.I)
            text = re.sub(re.escape(" finna "), " fixing to ", text, flags=re.I)
            text = re.sub(re.escape(" gimme "), " give me ", text, flags=re.I)
            text = re.sub(re.escape(" gonna "), " going to ", text, flags=re.I)
            text = re.sub(re.escape(" gon't "), " go not ", text, flags=re.I)
            text = re.sub(re.escape(" gotta "), " got to ", text, flags=re.I)
            text = re.sub(re.escape(" hadn't "), " had not ", text, flags=re.I)
            text = re.sub(re.escape(" hasn't "), " has not ", text, flags=re.I)
            text = text.strip()
        
        elif self.language == 'fr':
            # Expand contractions
            text = ' ' + text + ' '
            text = re.sub(re.escape(" c'est "), " ce est ", text, flags=re.I)
            text = re.sub(re.escape("d'"), "de ", text, flags=re.I)
            text = re.sub(re.escape("j'"), "je ", text, flags=re.I)
            text = re.sub(re.escape("s'"), "se ", text, flags=re.I)
            text = re.sub(re.escape("qu'"), "se ", text, flags=re.I)
            text = re.sub(re.escape("t'"), "te ", text, flags=re.I)
            text = text.strip()

        return text

    def tokenize(self, text:str):
        """ Tokenizes text using Word Tokenizer. Attention: this method returns a list instead of a string.
        
        >>> expand_contractions("At the end of the day, @john you're solely responsible for your #success and your #failure. And the sooner you realize that, you accept that, and integrate that into your work ethic, you will start being #successful.")
        '['At', 'the', 'end', 'of', 'the', 'day', ',', '@', 'john', 'you', "'re", 'solely', 'responsible', 'for', 'your', '#', 'success', 'and', 'your', '#', 'failure', '.', 'And', 'the', 'sooner', 'you', 'realize', 'that', ',', 'you', 'accept', 'that', ',', 'and', 'integrate', 'that', 'into', 'your', 'work', 'ethic', ',', 'you', 'will', 'start', 'being', '#', 'successful', '.']'

        Parameters
        ----------
        text : str
            The string that will be transformed
        """
        text = word_tokenize(text)
        return text

    def lower_remove_white(self, text:str):
        """ Use it to lower text and to remove training whitespaces
        
        >>> lower_remove_white("- Mary is coming! \n- When? \n- Today!")
        '- mary is coming!  - when?  - today!'

        Parameters
        ----------
        text : str
            The string that will be transformed
        """
        
        # lower text
        text = text.lower()
        # strip trailing whitespaces
        text = re.sub(r'\n', ' ', text)

        return text

    def preprocess(self, text:str, tokenize=True, stem=True, remove_stopwords=False):
        """Preprocess text before data handling with most common settings.
        Text is processed in the following order:

            - lower text
            - strip trailing whitespaces
            - merge same words
            - expand contractions
            - replace urls
            - remove tags
            - replace hashtags
            - replace ips
            - remove social media handles
            - remove stopwords if True
            - remove accents
            - remove punctuation        
            - replace numbers
            - reduce loooooong words
            - remove excessive spaces
            - stem text if True
            - tokenize text (if True, will return a list of tokens)

        >>> preprocess("At the end of the day, you're solely responsible for your success and your failure. And the sooner you realize that, you accept that, and integrate that into your work ethic, you will start being successful. As long as you blame others for the reason you aren't where you want to be, you will always be a failure.")
        '['end', 'day', 'sole', 'respons', 'success', 'failur', 'sooner', 'realiz', 'that', 'accept', 'that', 'integr', 'work', 'ethic', 'start', 'success', 'long', 'blame', 'other', 'reason', 'want', 'be', 'alway', 'failur']'
        
        Parameters
        ----------
        text : str
            The string that will be transformed
        tokenize : boolean
            Defines if the text should be tokenized or not
        stem : boolean
            Defines if the tokens should be stemmed or not
        remove_stopwords : boolean
            Defines if stopwords should be removed.
        """
        text = str(text)
        # lowers text and removes trailing spaces
        text = self.lower_remove_white(text)
        # merge tokens wich are the same
        text = self.merge_same_tokens(text)
        # expands contractions
        text = self.expand_contractions(text)
        # replace urls
        text = self.replace_urls(text, 'URL')
        # remove tags
        text = self.remove_html_tags(text)
        # replace hashtags
        text = self.replace_hashtags(text, 'HASHTAG')
        # replace ips
        text = self.replace_ips(text, 'IP')
        # remove social media handles
        text = self.remove_handles(text)
        # remove stopwords
        if remove_stopwords:
            text = self.remove_stopwords(text)
        # remove accents        
        text = self.strip_accents(text)
        # remove punctuation        
        text = self.remove_punctuation(text)
        # remove numbers
        text = self.replace_numbers(text)
        # reduce loooooong words
        text = self.reduce_words_with_repeated_chars(text)
        # remove excessive spaces
        text = self.remove_excessive_spaces(text)
        # stem text
        if stem:
            text = self.stem_sentence(text)
        # tokenize
        if tokenize:
            text = self.tokenize(text)
            self.vocab.update(text)
        else:        
            # updates list of words
            self.vocab.update(list(text))
        # return text
        return text
            
    def preprocess_list(self, text_list:list, tokenize=True, stem=True, remove_stopwords=False):
        """Preprocess every text in a list of strings

        >>> list_of_strings = [
        'I love you',
        'Please, never leave me alone',
        'If you go, I will die',
        'I am watching a lot of romantic comedy lately',
        'I have to eat icecream'
        ]

        >>> list_processed = aruana_en.preprocess_list(list_of_strings, stem=False)
        '['end', 'day', 'sole', 'respons', 'success', 'failur', 'sooner', 'realiz', 'that', 'accept', 'that', 'integr', 'work', 'ethic', 'start', 'success', 'long', 'blame', 'other', 'reason', 'want', 'be', 'alway', 'failur']'

        Parameters
        ----------
        text : list
            The list of strings to be transformed
        tokenize : boolean
            Defines if the text should be tokenized or not
        stem : boolean
            Defines if the tokens should be stemmed or not
        remove_stopwords : boolean
            Defines if stopwords should be removed.
        """
        texts = []
        for text in tqdm(text_list):
            # preprocess
            text = self.preprocess(text, tokenize=tokenize, stem=stem, remove_stopwords=remove_stopwords)
            texts.append(text)
        return texts

    def create_corpus(self, text_list:list):
        """Receives a list of strings and returns a single string with all the text

        >>> list_of_strings = ['I love you', 'Please, never leave me alone', 'If you go, I will die', 'I am watching a lot of romantic comedy lately', 'I have to eat icecream']
        >>> create_corpus(list_of_strings)
        'I love you Please, never leave me alone If you go, I will die I am watching a lot of romantic comedy lately I have to eat icecream'

        Parameters
        ----------
        text : list
            The list of strings that will be used to return the corpus
        """
        corpus = ' '
        for text in text_list:
            corpus = corpus + ' ' + str(text)
        return corpus

    def vocab_size(self, corpus:str):
        """Returns the number of unique tokens found on the corpus

        >>> vocab_size('  I love you Please, never leave me alone If you go, I will die I am watching a lot of romantic comedy lately I have to eat icecream')
        24

        Parameters
        ----------
        corpus : str
            The string that holds the entire corpus
        """
        words_map_dict = Counter(corpus.split())
        unique_words = len(words_map_dict.keys())
        vocab_size = int(unique_words)

        return vocab_size

    def lexical_diversity(self, corpus_slice:str, corpus:str):
        """Counts the vocab of one part of the corpus and compares it to the entire corpus

        >>> lexical_diversity(list_of_strings[0], corpus)
        0.125

        Parameters
        ----------
        corpus_slice : str
            The string that you want to analyse
        corpus : str
            The string that holds the entire corpus
        """
        # count vocab for slice 
        slice_vocab = self.vocab_size(corpus_slice)

        # count vocab for entire corpus
        all_vocab = self.vocab_size(corpus)

        # calculate diversity
        diversity = slice_vocab/all_vocab
        return diversity
