# -*- coding: utf-8 -*-
"""
Created on Wed Nov 21 13:03:21 2018

@author: fabio
"""
import warnings
warnings.filterwarnings("ignore")
from collections import Counter
from itertools import chain
from sklearn.model_selection import *
from sklearn.metrics import *
from sklearn.preprocessing import LabelBinarizer,LabelEncoder
from sklearn.decomposition import PCA
from sklearn.metrics.pairwise import cosine_similarity
from scipy import spatial
import re
import pandas as pd
from pandas import *
import numpy as np
import os
import itertools
import pickle
#import matplotlib.pyplot as plt
from sklearn_crfsuite import CRF
from sklearn_crfsuite import metrics
from sklearn_crfsuite import scorers
from pathlib import Path
start_path = Path(__file__).parent.absolute()

def print_transitions(trans_features):
    string = ""
    for (label_from, label_to), weight in trans_features:
        string += ("%-6s -> %-7s %0.6f" % (label_from, label_to, weight))
        string+="\n"
    return string
def print_state_features(state_features):
    string = ""
    for (attr, label), weight in state_features:
        string+=("%0.6f %-8s %s" % (weight, label, attr))
        string+="\n"
    return string

class GenerateFeatures():
    def __init__(self):
        self.features={}
    def generateMorphologyFeatures(self,word,tag):
        self.features[tag+'contains_punct']=str(0)
        self.features[tag+'word_shape']=''
        for char in word:
            if str(char).isalpha():
                if str(char).islower():
                    self.features[tag+'word_shape']+='a'
                else:
                    self.features[tag+'word_shape']+='A'
            elif str(char).isdigit():
                self.features[tag+'word_shape']+='#'
            else:
                self.features[tag+'word_shape']+='-'
                self.features[tag+'contains_punct']=str(1)
        self.features[tag+'ends_in_s']=str(word[-1:]=='s')
        self.features[tag+'ends_in_a']=str(word[-1:]=='a')
        self.features[tag+'ascii']=str(len(word)==len(word.encode()))
        self.features[tag+'is_all_lowercase']=str(word.lower()==word and word.isdigit()==False)
        self.features[tag+'is_first_uppercase']=str(word.isupper())
        self.features[tag+'is_complete_uppercase']=str(word.upper()==word and word.isdigit()==False)
        self.features[tag+'is_numeric']=str(word.isdigit())
        self.features[tag+'is_alpha']=str(word.isalpha())
        self.features[tag+'is_alphanumeric']=str(bool((re.match('^(?=.*[0-9]$)(?=.*[a-zA-Z])',word))))
        self.features[tag+'length']=str(word)
        prefix_suffix_window=5
        for i in range(1,prefix_suffix_window+1):
            self.features[tag+'prefix_'+str(i)]=str(word[:i])
            self.features[tag+'suffix_'+str(i)]=str(word[-i:])
    def wordcontextFeatures(self,word,pos,lemma,tag):
        self.features[tag+'word']=str(word)
        self.features[tag+'pos']=pos
        self.features[tag+'lemma']=str(lemma)
            
def features(document,pos_list,lemma_list,index,useful_features):
    word=document[index]
    pos=pos_list[index]
    lemma=lemma_list[index]
    if index-1>0:
        prev_prev_word=document[index-2]
        prev_prev_pos=pos_list[index-2]
        prev_prev_lemma=lemma_list[index-2]
    else:
        prev_prev_word=''
        prev_prev_pos=''
        prev_prev_lemma=''
    if index>0:
        prev_word=document[index-1]
        prev_pos=pos_list[index-1]
        prev_lemma=lemma_list[index-1]
    else:
        prev_word=''
        prev_pos=''
        prev_lemma=''
    if index<len(document)-1:
        next_word=document[index+1]
        next_pos=pos_list[index+1]
        next_lemma=lemma_list[index+1]
    else:
        next_word=''
        next_pos=''
        next_lemma=''
    if index<len(document)-2:
        next_next_word=document[index+2]
        next_next_pos=pos_list[index+2]
        next_next_lemma=lemma_list[index+2]
    else:
        next_next_word=''
        next_next_pos=''
        next_next_lemma=''
    gen_features=GenerateFeatures()
    if useful_features[0]:
        gen_features.generateMorphologyFeatures(word,'')
        gen_features.generateMorphologyFeatures(prev_word,'prev_')
        gen_features.generateMorphologyFeatures(next_word,'next_')
        gen_features.generateMorphologyFeatures(prev_prev_word,'prev_prev_')
        gen_features.generateMorphologyFeatures(next_next_word,'next_next_')
    if useful_features[1]:
        gen_features.wordcontextFeatures(word,pos,lemma,'')
        gen_features.wordcontextFeatures(prev_word,prev_pos,prev_lemma,'prev_')
        gen_features.wordcontextFeatures(next_word,next_pos,next_lemma,'next_')
        gen_features.wordcontextFeatures(prev_prev_word,prev_prev_pos,prev_prev_lemma,'prev_prev_')
        gen_features.wordcontextFeatures(next_next_word,next_next_pos,next_next_lemma,'next_next_')
    return gen_features.features

def untag(document,option):
    #Passar de colunas para tuples
    if option!='predict':
        #return [str(word) for word,pos,tag in document],[pos for word,pos,tag in document],['' for word,pos,tag in document]
        return [str(word) for word,pos,lemma,tag in document],[pos for word,pos,lemma,tag in document],[lemma for word,pos,lemma,tag in document]
    else:
        #return [str(word) for word,pos in document],[pos for word,pos in document],['' for word,pos in document]
        return [str(word) for word,pos,lemma in document],[pos for word,pos,lemma in document],[lemma for word,pos,lemma in document]

def prepareData(tagged_documents,option,useful_features):
    #Construir o dataset a partir dos ficheiros -> Gerar features para usar no CRF
    X,y=[],[]
    for document in tagged_documents:
        words,postags,lemmas =untag(document,option)
        X.append([features(words,postags,lemmas,index,useful_features) for index in range(len(document))])
        if option!='predict':
            y.append([tag for word,pos,lemmas,tag in document])
    return X,y

def fromListToTuple(data):
    tuple_data=[]
    for l in data:
        tuple_data.append(tuple(l))
    return tuple_data

def fromTuplesToList(data,y):
    dataset=[]
    for i in range(len(data)):
        dataset.append([])
        for j in range(len(data[i])):
            list_j=list(data[i][j])
            list_j.append(y[i][j])
            dataset[i].append(list_j)
    return dataset

def save_model(nome,modelo):
    with open(nome,'wb') as f:
        pickle.dump(modelo,f,protocol=pickle.HIGHEST_PROTOCOL)

def load_model(nome):
    with open(nome,'rb') as f:
         result = pickle.load(f)
    return result

def run_crf(data,load_file=start_path.joinpath("harem.pickle")):
    useful_features=[True,True]
    X_test=fromListToTuple(data)
    X_teste,y_teste=prepareData([X_test],'predict',useful_features)
    crf = load_model(load_file)
    y_pred=crf.predict(X_teste)
    resultados = []
    for index,elem in enumerate(y_pred[0]):
        resultados.append(str(y_pred[0][index]))
    return resultados

def test_crf(train_file,test_file,model_name=""):
    valores = []
    data=pandas.read_csv(train_file,sep="\t",header=None)
    X_dataset=fromListToTuple(data.iloc[:,[0,1,2,3]].values)
    useful_features=[True,True]
    X_train,y_train=prepareData([X_dataset],'train',useful_features)
    data2=pandas.read_csv(test_file,sep="\t",header=None)
    X_test=fromListToTuple(data2.iloc[:,[0,1,2]].values)
    X_teste,y_teste=prepareData([X_test],'predict',useful_features)
    crf = CRF(
            algorithm='lbfgs',
            c1=0.0625,
            c2=0.5,
            max_iterations=100,
            all_possible_transitions=False,
            all_possible_states=True,
            verbose=True
        )
    crf.fit(X_train, y_train)
    if(model_name!=""):
        save_model(model_name + ".pickle",crf)
    useful_features=[True,True]
    data2=pandas.read_csv(test_file,sep="\t",header=None)
    X_test=fromListToTuple(data2.iloc[:,[0,1,2]].values)
    X_teste,y_teste=prepareData([X_test],'predict',useful_features)
    y_pred=crf.predict(X_teste)
    resultados = []
    for index,elem in enumerate(y_pred[0]):
        resultados.append(str(y_pred[0][index]))
    return resultados

