import pickle
import re
import string
import os
from urllib.parse import urljoin

class DataHelper():
    def __init__(self):
        pass

    def clean_text(self, text_doc, new_line_elimination):
        punctuations = r')(}{:؟!،؛»«.' + r"/<>?.,:;"
        punctuations = '[' + punctuations + string.punctuation + ']'
        punctuations = punctuations.replace("@", "")

        text_doc.strip()

        # pattern = ur'\s*@[a-zA-Z0-9]*\s*'
        # tmp = re.findall(pattern, text_doc)
        # newstring = re.sub(pattern, eliminate_pattern, text_doc)


        #finding the numbers

        # pattern = r"[-+]?\d*\.\d+|\d+"
        # nums_list = re.findall(pattern, text_doc)
        # newstring = re.sub(pattern, 'floatingpointnumber', text_doc)

        # pattern = r'(?i)\b((?:[a-z][\w-]+:(?:/{1,3}|[a-z0-9%])|www\d{0,3}[.]|[a-z0-9.\-]+[.][a-z]{2,4}/)(?:[^\s()<>]+|\(([^\s()<>]+|(\([^\s()<>]+\)))*\))+(?:\(([^\s()<>]+|(\([^\s()<>]+\)))*\)|[^\s`!()\[\]{};:\'".,<>?«»“”‘’]))'
        pattern =r'(?i)\b((?:https?:(?:/{1,3}|[a-z0-9%])|[a-z0-9.\-]+[.](?:com|net|org|edu|gov|mil|aero|asia|biz|cat|coop|info|int|jobs|mobi|museum|name|post|pro|tel|travel|xxx|ac|ad|ae|af|ag|ai|al|am|an|ao|aq|ar|as|at|au|aw|ax|az|ba|bb|bd|be|bf|bg|bh|bi|bj|bm|bn|bo|br|bs|bt|bv|bw|by|bz|ca|cc|cd|cf|cg|ch|ci|ck|cl|cm|cn|co|cr|cs|cu|cv|cx|cy|cz|dd|de|dj|dk|dm|do|dz|ec|ee|eg|eh|er|es|et|eu|fi|fj|fk|fm|fo|fr|ga|gb|gd|ge|gf|gg|gh|gi|gl|gm|gn|gp|gq|gr|gs|gt|gu|gw|gy|hk|hm|hn|hr|ht|hu|id|ie|il|im|in|io|iq|ir|is|it|je|jm|jo|jp|ke|kg|kh|ki|km|kn|kp|kr|kw|ky|kz|la|lb|lc|li|lk|lr|ls|lt|lu|lv|ly|ma|mc|md|me|mg|mh|mk|ml|mm|mn|mo|mp|mq|mr|ms|mt|mu|mv|mw|mx|my|mz|na|nc|ne|nf|ng|ni|nl|no|np|nr|nu|nz|om|pa|pe|pf|pg|ph|pk|pl|pm|pn|pr|ps|pt|pw|py|qa|re|ro|rs|ru|rw|sa|sb|sc|sd|se|sg|sh|si|sj|Ja|sk|sl|sm|sn|so|sr|ss|st|su|sv|sx|sy|sz|tc|td|tf|tg|th|tj|tk|tl|tm|tn|to|tp|tr|tt|tv|tw|tz|ua|ug|uk|us|uy|uz|va|vc|ve|vg|vi|vn|vu|wf|ws|ye|yt|yu|za|zm|zw)/)(?:[^\s()<>{}\[\]]+|\([^\s()]*?\([^\s()]+\)[^\s()]*?\)|\([^\s]+?\))+(?:\([^\s()]*?\([^\s()]+\)[^\s()]*?\)|\([^\s]+?\)|[^\s`!()\[\]{};:\'".,<>?«»“”‘’])|(?:(?<!@)[a-z0-9]+(?:[.\-][a-z0-9]+)*[.](?:com|net|org|edu|gov|mil|aero|asia|biz|cat|coop|info|int|jobs|mobi|museum|name|post|pro|tel|travel|xxx|ac|ad|ae|af|ag|ai|al|am|an|ao|aq|ar|as|at|au|aw|ax|az|ba|bb|bd|be|bf|bg|bh|bi|bj|bm|bn|bo|br|bs|bt|bv|bw|by|bz|ca|cc|cd|cf|cg|ch|ci|ck|cl|cm|cn|co|cr|cs|cu|cv|cx|cy|cz|dd|de|dj|dk|dm|do|dz|ec|ee|eg|eh|er|es|et|eu|fi|fj|fk|fm|fo|fr|ga|gb|gd|ge|gf|gg|gh|gi|gl|gm|gn|gp|gq|gr|gs|gt|gu|gw|gy|hk|hm|hn|hr|ht|hu|id|ie|il|im|in|io|iq|ir|is|it|je|jm|jo|jp|ke|kg|kh|ki|km|kn|kp|kr|kw|ky|kz|la|lb|lc|li|lk|lr|ls|lt|lu|lv|ly|ma|mc|md|me|mg|mh|mk|ml|mm|mn|mo|mp|mq|mr|ms|mt|mu|mv|mw|mx|my|mz|na|nc|ne|nf|ng|ni|nl|no|np|nr|nu|nz|om|pa|pe|pf|pg|ph|pk|pl|pm|pn|pr|ps|pt|pw|py|qa|re|ro|rs|ru|rw|sa|sb|sc|sd|se|sg|sh|si|sj|Ja|sk|sl|sm|sn|so|sr|ss|st|su|sv|sx|sy|sz|tc|td|tf|tg|th|tj|tk|tl|tm|tn|to|tp|tr|tt|tv|tw|tz|ua|ug|uk|us|uy|uz|va|vc|ve|vg|vi|vn|vu|wf|ws|ye|yt|yu|za|zm|zw)\b/?(?!@)))'

        newstring = re.sub(pattern,'',text_doc)

        pattern = '\s*' + punctuations + '+' + '\s*'
        tmp = re.findall(pattern, newstring)
        newstring = re.sub(pattern, self.add_space, newstring)

        # pattern = u'([a-zA-Z0-9]+)(\s*)(' + punctuations + u')(\s*)([a-zA-Z0-9]+)'
        # rep = ur'\1\3\5'
        # tmp = re.findall(pattern, newstring)
        # newstring = re.sub(pattern, rep, newstring)

        pattern = r'[\n]+'
        tmp = re.findall(pattern, newstring)
        if new_line_elimination:
            newstring = re.sub(pattern, " ", newstring)
        else:
            # newstring = re.sub(pattern, "\n", newstring)
            pass

        punctuations = r")(}{:؟!-،؛»«.@$&%" + r"/<>?.,:;"
        latinLettersDigits = r"a-zA-Z0-9"
        pattern = r'[^' + punctuations + latinLettersDigits + 'آ-ی' + '‌' + '\d\s:]'
        tmp = re.findall(pattern, newstring)
        newstring = re.sub(pattern, self.eliminate_pattern, newstring)

        pattern = r'[ ]+'
        tmp = re.findall(pattern, newstring)
        newstring = re.sub(pattern, ' ', newstring)

        #
        # for number in nums_list:
        #     pattern = 'floatingpointnumber'
        #     newstring = re.sub(pattern, number, newstring, 1)


        return newstring



    def replace_newline_with_dot(self, mystring):
        return ' . '

    def eliminate_pattern(self, mystring):
        return ""

    def load_var(self, load_path):
        file = open(load_path, 'rb')
        variable = pickle.load(file)
        file.close()
        return variable

    def save_var(self, save_path, variable):
        print("saving vars ...")
        file = open(save_path, 'wb')
        pickle.dump(variable, file)
        print("variable saved.")
        file.close()

    def build_stem_dictionary(self, normalizer, verb_tense_path, mokasar_noun_path):
        path_dir = "resource/Persian_Dependency_Treebank/Data/2ndRep"
        lexicon_stem = set()
        verb_stem = set()
        #verb_tense_map = {}
        verb_p2f_map = {}
        verb_f2p_map = {}
        for fileName in os.listdir(path_dir):
            file_path = path_dir + "/" + fileName
            with open(file_path, "r") as input:
                input_content = input.readlines()
                for el in input_content:
                    el = normalizer.sub_alphabets(el)
                    el = el.split("\t")
                    if (len(el) > 2):
                        if (el[3] == 'V'):
                            tmp_pos = "V"
                        else:
                            tmp_pos = "N"
                        stem_word = el[2]
                        stem_word = stem_word.split("#")
                        stem_word = [x.strip('\u200c') for x in stem_word]
                        if (tmp_pos == "V" and len(stem_word) == 2):
                            if (len(stem_word[0]) != 0 and len(stem_word[1]) != 0):
                                verb_p2f_map[stem_word[0]] = stem_word[1]
                                verb_f2p_map[stem_word[1]] = stem_word[0]
                                verb_stem.add(stem_word[0])
                                verb_stem.add(stem_word[1])
                        if(tmp_pos == 'V' and len(stem_word) == 3):
                            if(len(stem_word[0]) != 0 and len(stem_word[1]) != 0 and len(stem_word[2]) !=0):
                                #verb_prifix.add(stem_word[0])
                                verb_p2f_map[stem_word[1]] = stem_word[2]
                                verb_f2p_map[stem_word[2]] = stem_word[1]
                                verb_stem.add(stem_word[1])
                                verb_stem.add(stem_word[2])
                        for t in stem_word:
                            if len(t) > 1:
                                if (tmp_pos == 'N'):
                                    lexicon_stem.add(t)

        with open(verb_tense_path, "r") as bon_file:
            bon_file_content = bon_file.readlines()
            for el in bon_file_content:
                el = el.strip()
                el = normalizer.sub_alphabets(el)
                el = el.split()
                el = [x.strip('\u200c') for x in el]

                verb_p2f_map[el[0]] = el[1]
                verb_f2p_map[el[1]] = el[0]
                verb_stem.add(el[0])
                verb_stem.add(el[1])

        irregular_noun = {}
        with open(mokasar_noun_path, "r") as input:
            input_content = input.readlines()
            for el in input_content:
                el = normalizer.sub_alphabets(el)
                el = el.replace("\t\t", "\t")
                el = el.strip().split("\t")
                el = [x.strip('\u200c') for x in el]
                irregular_noun[el[0]] = el[1]
                lexicon_stem.add(el[0])

        verb_tense_map = [verb_p2f_map, verb_f2p_map]
        return lexicon_stem, verb_stem, verb_tense_map, irregular_noun
