# Copyright 2020 Alberto Martín Mateos and Niloufar Shoeibi
# See LICENSE for details.

# Copyright 2019-2020 Alberto Martín Mateos & Niloufar Shoeibi for TWINPICS project
# See LICENSE for details.
# -*- coding: utf-8 -*-

import subprocess
import sys
import warnings
warnings.filterwarnings("ignore")

#---------------------------------------------------------------INSTALLATION---------------------------------------------------------#

def install(package):
    subprocess.check_call([sys.executable, "-m", "pip", "install", package])

install("networkx==2.2")
install("pandas")
install("numpy")
install ("urllib3")
install("requests")
install("matplotlib")
install("tweepy")
##----------------------------------------------IMPORTS---------------------------------------------------------------------------#	
import json
import pandas as pd
import numpy as np
import urllib3
import re
import time
import networkx as nx
import requests
import matplotlib.pyplot as plt
import tweepy
from os import path
from datetime import timedelta

#----------------------------------------FUNCTIONS-----------------------------------------------------------------------------#

def load_file(df):
    df = pd.read_json(df)
    return df

def merge_dataframes(df1, df2):
    df_final = pd.merge(df1, df2, on = ["screen_name"])
    return df_final

#----------------------------------Structure for the graph-----------------------------------------#
def profile_connections (data_tweets):    
    screen_name = []
    screen_name_mention = []
    data = pd.DataFrame()
    data_tweets = data_tweets.rename(columns = {"created_at": "created_at_tw"})
    data_tweets = pd.concat([data_tweets.drop(['user'], axis=1), data_tweets['user'].apply(pd.Series)],
                            axis=1).drop_duplicates().reset_index(drop=True)
    if(data_tweets.empty == False):
        for i in range(len(data_tweets)):
            screen_name.append(data_tweets["screen_name"][i])
            if "@" in (data_tweets["text"][i]):
                split_text = data_tweets["text"][i].split()
                check = "@"
                screen_name_mention1 = [idx for idx in split_text if idx.lower().startswith(check.lower())]
                #Consider that @ is from a email or another words that is not the screen_name
                if not screen_name_mention1:
                    screen_name_mention.append(["Empty"])
                else:
                    screen_name_mention1 = re.sub("[\[\]\'\"!@?¿#$:….,]", '',str(screen_name_mention1)).split(", ")  
                    if(data_tweets["text"][i].startswith('RT',0)):
                        screen_name_mention.append([screen_name_mention1[0]])                    
                    else:
                        screen_name_mention.append(re.sub("[\[\]\'\"!@?¿#$:….,]", '',str(screen_name_mention1)).split(" "))
            else:
                screen_name_mention.append(["Empty"])
    data["screen_name"] = screen_name    
    data["screen_name_mention"] = screen_name_mention  
    #Agruping
    lst_col = "screen_name_mention"
    data = data[["screen_name","screen_name_mention"]]
    data_ext =pd.DataFrame({col:np.repeat(data[col].values, data[lst_col].str.len())for col in data.columns.difference(
        [lst_col])}).assign(**{lst_col:np.concatenate(data[lst_col].values)})[data.columns.tolist()]
    data1_ext = pd.DataFrame({'self_loop_iteration' : data_ext.groupby(data_ext.columns.tolist(),as_index=False).size().sort_values(
                                                                        ascending = False)}).reset_index()
    return data1_ext

#--------------------------------------Graph building functions---------------------------

def building_node_list(df):

    node_list = pd.DataFrame()
    n = []
    for i in range(len(df)):
        n.append(df['screen_name'].iloc[i])
        n.append(df['screen_name_mention'].iloc[i])
    node_list['screen_name']= n
    node_list = node_list.drop_duplicates()
    node_list = node_list[node_list["screen_name"] !="Empty"].reset_index()
    return list(node_list["screen_name"])

def building_DiGraph(node_list, df):
    DiG = nx.DiGraph()
    DiG.add_nodes_from(node_list)
    for i in range(len(df)):
        if(df['screen_name_mention'].iloc[i] !="Empty"):
            DiG.add_edge(df['screen_name'].iloc[i],df['screen_name_mention'].iloc[i])
    return DiG


def self_loop_iteration_nodes(df):
    df_self_loop_iteration_nodes = pd.DataFrame()
    n = []
    self_loop_iteration = []
    for i in range(len(df)):
        if(df["screen_name"].iloc[i] ==df["screen_name_mention"].iloc[i]):
            n.append(df['screen_name'].iloc[i])
            self_loop_iteration.append(df["self_loop_iteration"].iloc[i])            
        else:
            n.append(df['screen_name'].iloc[i])
            n.append(df['screen_name_mention'].iloc[i])
            self_loop_iteration.append(0)
            self_loop_iteration.append(0)
    df_self_loop_iteration_nodes['screen_name']= n
    df_self_loop_iteration_nodes['self_loop_iteration']= self_loop_iteration
    df = pd.DataFrame({'times': df_self_loop_iteration_nodes.groupby(["screen_name","self_loop_iteration"]).size().sort_values(
    ascending=False)}).reset_index().drop_duplicates(["screen_name"],keep="first")
    del df["times"]
    return df
	
def DiGraph_visualization (DiG):
    plt.figure(figsize=(30,18))
    nx.draw(DiG, with_labels=True, node_size=10, node_color="skyblue", node_shape="s", alpha=0.5, linewidths=4)
    plt.show()

#---------------------------------Graph features functions ----------------------
def in_degree_centrality(DiG, output):
    l = []
    l = nx.in_degree_centrality(DiG)
    sc = []
    indegcent = []
    for k, v in l.items():
        indegcent.append(v)
        sc.append(k)
    output['in_degree_centrality']=indegcent
    output["screen_name"] = sc    
    return output

def out_degree_centrality(DiG, output):
    l=[]
    l=nx.out_degree_centrality(DiG)
    sc=[]
    outd=[]
    for k, v in l.items():
        outd.append(v)
    output['outdegree_centrality']=outd
    return output

def degree_centrality(DiG,output):
    l=[]
    l=nx.degree_centrality(DiG)
    deg=[]
    for k, v in l.items():
        deg.append(v)
    output['degree_centrality']=deg
    return output

def in_degree(DiG, output):
    indegs=[]
    for i in range(len(list(DiG.nodes))):
        indegs.append(DiG.in_degree(list(DiG.nodes)[i]))
    output['in_degree']=indegs
    return output

def out_degree(DiG, output):
    outdegs=[]
    for i in range(len(list(DiG.nodes))):
        outdegs.append(DiG.out_degree(list(DiG.nodes)[i]))
    output['out_degree']=outdegs
    return output

def degree(DiG, output):
    degs=[]
    for i in range(len(list(DiG.nodes))):
        degs.append(DiG.degree(list(DiG.nodes)[i]))
    output['degree']=degs
    return output

def self_loops(DiG, output):
    SL = list(DiG.nodes_with_selfloops())
    output['self_loop']= False
    l=[]
    for i in range(len(SL)):
        for j in range(len(output)):
            if SL[i]==output['screen_name'].iloc[j]:
                output.self_loop.iloc[j]=True
    return output


def graph_features(DiG,df):

    output = pd.DataFrame()
    output = in_degree_centrality(DiG, output)
    output = out_degree_centrality(DiG, output)
    output = degree_centrality(DiG, output)
    output = in_degree(DiG, output)
    output = out_degree(DiG, output)
    output = degree(DiG, output)
    output = self_loops(DiG, output)
    df_nodes_self_loop_iteration = self_loop_iteration_nodes(df)
    output = merge_dataframes(output, df_nodes_self_loop_iteration)
    return output

#-------------------------------------Request to extract metadata for each user-------------------------------#

def body_users(user_list):
    len_user_list = len(user_list)
    body_tweets = json.dumps({
	  "screen_names": user_list,
	  "item_limit": len_user_list,
	  "show_params": [
		"created_at",
		  "default_profile",
		  "default_profile_image",
		  "description",
		  "favourites_count",
		  "followees","followers",
		  "listed_count","screen_name",
		  "statuses_count",
		  "verified"],
	  "filter_options": { }
	})
    return body_tweets


    

def user_timeline_extraction (node_list, consumerKey, consumerSecret,accessToken,accessTokenSecret, iter_tw, fichero):
    # get_user Returns the 20 most recent statuses posted from the authenticating user or the user specified. 
    profile_iter = 300
    k=0
    auth = tweepy.OAuthHandler(consumerKey[0], consumerSecret[0])
    auth.set_access_token(accessToken[0], accessTokenSecret[0])
    api = tweepy.API(auth)
    nc= 0
    ultima = 0
    origen1 = time.time()
    while ((k<= (len(node_list)+profile_iter)) & (ultima !=2) ):
      tweets_user=[]
      list_users_noInfo = []
      origen2 = time.time()
      i=0
      if ((len(node_list)>= k) & ((len(node_list)-profile_iter) <=k)):
        node_list1 = node_list[k:len(node_list)]
        ultima = 1
      else:
        node_list1 = node_list[k:k+profile_iter]
      while(i <(len(node_list1))):         
        pri = 0
        itera=0
        max_id=""
        intentos = 0
        while itera != iter_tw:
          try:
            if pri ==0:
              user=api.user_timeline(node_list1[i], count=200,tweet_mode="extended")      
              tweets_user.append(user)
              max_id = str(tweets_user[len(tweets_user)-1][len(tweets_user[len(tweets_user)-1])-1]._json["id_str"])
              pri=1
              itera = itera+1
            else:
              user=api.user_timeline(node_list1[i], count=200,tweet_mode="extended", max_id=max_id)
              tweets_user.append(user)
              max_id = str(tweets_user[len(tweets_user)-1][len(tweets_user[len(tweets_user)-1])-1]._json["id_str"])
              itera = itera+1
              intentos=0 # Para si justo es el primero el que no se puede extraer y no es problema de agotar claves
          except:
            nc = nc+1
            n_clave = len(consumerKey)
            if(nc == n_clave):
              nc= 0
            auth = tweepy.OAuthHandler(consumerKey[nc], consumerSecret[nc])
            auth.set_access_token(accessToken[nc], accessTokenSecret[nc])
            api = tweepy.API(auth)
            intentos = intentos + 1
            if(intentos == 2):         
              list_users_noInfo.append(node_list1[i])
              intentos = 0
              itera=iter_tw
        i = i+1       
      destino2 = time.time()
      aux=[]
      for i in range( len(tweets_user)):
        for j in range(len(tweets_user[i])):          
            aux.append(tweets_user[i][j]._json)     
      if(len(aux) != 0):  
        df = pd.DataFrame(aux)
        df = df.rename(columns = {"created_at": "created_at_tw"})
        df= df.rename(columns = {"lang": "lang_tw"})
        df = pd.concat([df.drop(['user'], axis=1), df['user'].apply(pd.Series)], axis=1)
        df = df.rename(columns={"followers_count": "followers",
                              "friends_count": "followees",
                              "retweet_count": "retweets",
                              "favorite_count": "favorites",
                              "favourites_count": "favorites_count",
                              "full_text": "text"})
        df2 = metadata_extraction(df)
        if path.exists("/content/"+ fichero):
          df2.to_csv(fichero,mode="a", index=False, header=False)
        else:
          df2.to_csv(fichero,index=False)         
        if len(list_users_noInfo) !=0:
          nodes = pd.DataFrame()
          nodes["nodos"] = list_users_noInfo
          if path.exists("/content/node_without_"+fichero):
            nodes.to_csv("node_without_"+ fichero, mode ="a",index=False, header=False)          
          else:
             nodes.to_csv("node_without_"+ fichero, index=False)
        k = k+profile_iter        
        if ultima ==1:          
          ultima =2
      else:
        print("Repetimos")
    print("Lista de usuarios sin metadata")
    print(list_users_noInfo)
    destino1 = time.time()
    return df2, list_users_noInfo


#----------------------------------Extract metadata-----------------------------------------------------------#
def metadata_extraction (df_users):
    
    users = df_users["screen_name"].unique()
    count_users=0
    df = pd.DataFrame() 
    for i in range(len(users)):
        data_tweets = df_users[df_users["screen_name"]==users[i]].reset_index()
        
        data = pd.DataFrame()              
        if(data_tweets.empty == False):       
            count_users = count_users + 1
            print("Usuarios: %f" % count_users)
            [screen_name_mention_user,seasonalityDF,tweets_urlDF,mentionsDF,text_tweetsDF,
             fav_twDF_list,ret_twDF_list,fav_twDF,ret_twDF,RT_tw,ntwPro_DF,minu_tw,tw_per_days_list,
             text_tweets_days_list,tweet_date_days_list,duplicated,minu_tw_answer,
             tw_day_list] = text_tweets_features_extraction(data_tweets)
            data["screen_name_mention_user"] = screen_name_mention_user
            data["screen_name"] = data_tweets["screen_name"].iloc[0]  
            data["created_at"] = data_tweets["created_at"].iloc[0]
            data["following"] = data_tweets["followees"].iloc[0]
            data["followers"] = data_tweets["followers"].iloc[0]
            data["statuses_count"] = data_tweets["statuses_count"].iloc[0]
            data["default_profile"] = data_tweets["default_profile"].iloc[0]
            data["default_profile_image"] = data_tweets["default_profile_image"].iloc[0]
            data["biography_profile"] = data_tweets["description"].iloc[0]
            data["listed_count"] = data_tweets["listed_count"].iloc[0]    
            data["favourite_count"] = data_tweets["favorites_count"].iloc[0]
            data["seasonality"] = seasonalityDF    
            data["tweets_url"] = tweets_urlDF    
            data["mentions"] = mentionsDF 
            data["text_tweets_days_list"] = text_tweets_days_list
            data["tweet_date_days_list"] = tweet_date_days_list   
            data["text_tweets"] = text_tweetsDF
            data["favorite_tweets_list"] = fav_twDF_list
            data["retweet_tweets_list"] = ret_twDF_list
            data["favorite_tweets_count"] = fav_twDF
            data["retweet_tweets_count"] = ret_twDF
            data["RT"] = RT_tw
            data["num_ownTw"] = ntwPro_DF    
            data["time_btw_tw"] = minu_tw
            data["tw_day_list"] = tw_day_list
            data["tw_per_day_list"] = tw_per_days_list
            data["tw_duplicated"] = duplicated
            data["minu_tw_answer"] = minu_tw_answer
            data["profile_type"] = data_tweets["verified"].iloc[0]
            data["screen_name_mention_user"] = data["screen_name_mention_user"].astype(str)
            probably_fake = []
            for j in range(len(data)):
                if data["biography_profile"].any()==False:# Si se lee el dataframe
                #if len(data["biography_profile"][j]) == 0:# Si se hace la extraccion antes de guardar los datos
                    data["biography_profile"][j] = 1
                else:
                    data["biography_profile"][j] = 0
                if data["following"].iloc[j] == 2001:
                    probably_fake.append(1)
                else:
                    probably_fake.append(0)
            data["probably_fake"] = probably_fake
            df = df.append(data, ignore_index=True)
    df =df.loc[df.astype(str).drop_duplicates().index].reset_index()
    
    df = time_twitter_account(df)
    df = time_series_tw(df)

    return df



def text_tweets_features_extraction(data_tweets):
   
    seasonalityDF = []
    tweets_urlDF = []
    mentionsDF = []
    text_tweetsDF = []
    fav_twDF= []
    ret_twDF = []
    ret_twDF_list = []
    fav_twDF_list = []
    ntwPro_DF= []
    RT_tw = []
    minu_tw = []
    minu_tw_answer = []
    tw_duplicated = []
    duplicated = []
    tw_per_day_list= []
    screen_name_mention_user = []
    tweets = []
    tweets_url = 0
    num_tw_day = 1
    is_RT=0
    is_RT_list= []
    screen_name_mention = []
    RT = []
    retweet_list = []
    favorite_list = []
    mentions = 0
    time_btw_tw = []
    tw_answer = []
    num_tw_days = []
    text_duplicated = []
    tw_day_list = []
    text_tweets_day = []
    text_tweets_days = []
    text_tweets_days_list = []
    #Almacenar la fecha entera de los tweets para si hay que consultar las horas de ese dia
    tweet_date_day = []
    tweet_date_days = []
    tweet_date_days_list = []
    lang_text_tweet = list(data_tweets["lang_tw"])
    text_tweets_user= list(data_tweets["text"])
    print(len(text_tweets_user))
    favorites= list(data_tweets["favorites"])
    retweets = list(data_tweets["retweets"])
    tweets_date= pd.to_datetime(data_tweets["created_at_tw"])
    for j in range(len(text_tweets_user)): #Comprobamos cuantos tweets han sido escritos por el usuario 
        if(text_tweets_user[j].startswith('RT',0)):
            RT.append(text_tweets_user[j])
            is_RT = 1
        else:
            is_RT=0          
            #Tweets with url from users
            if "http" in text_tweets_user[j]:
                tweets_url = tweets_url + 1
            #Tweets with mentions from users
            if "@" in text_tweets_user[j]:
                mentions = mentions + 1                        
            else:
                #Time of the answer of a tweet(tw --> @)
                if(j!=(len(text_tweets_user)-1)):
                    if ("@" in text_tweets_user[j+1]):
                        tw_answer.append((tweets_date[j] - tweets_date[j+1]) / np.timedelta64(1,'m'))
                #Found duplicated tweets where the only change the photo  that the users publish
                text_sinHttp = " ".join(filter(lambda x:x[0:4]!='http', text_tweets_user[j].split()))
                text_sinMencion = " ".join(filter(lambda x:x[0]!='@', text_sinHttp.split()))
                text_duplicated.append(text_sinMencion)
            tweets.append({"text":text_tweets_user[j], "lang": lang_text_tweet[j], "RT": 0})
            retweet_list.append(int(retweets[j]))
            favorite_list.append(int(favorites[j]))
        is_RT_list.append(is_RT)                                
        if(j!=0):
          #Time between tweets (RT + propios)
          time_btw_tw.append((tweets_date[j-1] - tweets_date[j]) / np.timedelta64(1,'m'))
          #Found if tweets are the same day
          #Second condition if all the tweets are the same day          
          if((tweets_date[j-1].date() == tweets_date[j].date()) & (j!=(len(text_tweets_user)-1))):
                            
              num_tw_day = num_tw_day + 1
              #Create a list with the tweets of a day
              text_tweets_day.append({"text": text_tweets_user[j-1],"lang": lang_text_tweet[j-1],"RT": is_RT_list[j-1]})
              text_tweets_day.append({"text": text_tweets_user[j],"lang": lang_text_tweet[j], "RT": is_RT_list[j]})              
              tweet_date_day.append(tweets_date[j-1])
              tweet_date_day.append(tweets_date[j])                         
          else:
              if(j==(len(text_tweets_user)-1)):                  
                  if(tweets_date[j-1].date() == tweets_date[j].date()):                     
                      text_tweets_day.append({"text": text_tweets_user[j-1],"lang": lang_text_tweet[j-1],"RT": is_RT_list[j-1]})
                      text_tweets_day.append({"text": text_tweets_user[j],"lang": lang_text_tweet[j], "RT": is_RT_list[j]})              
                      tweet_date_day.append(tweets_date[j-1])
                      tweet_date_day.append(tweets_date[j])
                      num_tw_day = num_tw_day + 1
                      num_tw_days.append(num_tw_day)
                      text_tweets_days.append([i for n, i in enumerate(text_tweets_day) if i not in text_tweets_day[n + 1:]] )
                      tweet_date_days.append(list(dict.fromkeys(tweet_date_day)))
                      num_tw_day = 1 
                      text_tweets_day = []
                      tweet_date_day = []
                  else:                      
                      text_tweets_day.append({"text": text_tweets_user[j-1],"lang": lang_text_tweet[j-1],"RT": is_RT_list[j-1]})                                   
                      tweet_date_day.append(tweets_date[j-1]) 
                      num_tw_days.append(1)                     
                      text_tweets_days.append([i for n, i in enumerate(text_tweets_day) if i not in text_tweets_day[n + 1:]])
                      tweet_date_days.append(list(dict.fromkeys(tweet_date_day)))
                      text_tweets_day = []
                      tweet_date_day = []
                      
                      tweet_date_day.append(tweets_date[j])                     
                      text_tweets_day.append({"text": text_tweets_user[j],"lang": lang_text_tweet[j], "RT": is_RT_list[j]}) 
                      num_tw_days.append(1)
                      text_tweets_days.append([i for n, i in enumerate(text_tweets_day) if i not in text_tweets_day[n + 1:]] )
                      tweet_date_days.append(list(dict.fromkeys(tweet_date_day)))
                      num_tw_day = 1 
                      text_tweets_day = []
                      tweet_date_day = []
                      
              else:
                  if(num_tw_day ==1):
                    text_tweets_day.append({"text": text_tweets_user[j-1],"lang": lang_text_tweet[j-1],"RT": is_RT_list[j-1]})
                    tweet_date_day.append(tweets_date[j-1])
                  #+1 because dont sum the last interation
                  num_tw_days.append(num_tw_day)
                  text_tweets_days.append([i for n, i in enumerate(text_tweets_day) if i not in text_tweets_day[n + 1:]] )
                  tweet_date_days.append(list(dict.fromkeys(tweet_date_day)))           
                  num_tw_day = 1 
                  text_tweets_day = []
                  tweet_date_day = []
        if "@" in (text_tweets_user[j]):
            split_text = text_tweets_user[j].split()
            check = "@"
            screen_name_mention1 = [idx for idx in split_text if idx.lower().startswith(check.lower())] 
            if not screen_name_mention1:
                    screen_name_mention.append(["Empty"])
            else:
                screen_name_mention1 = re.sub("[\[\]\'\"!@?¿#$:….,]", '',str(screen_name_mention1)).split(", ")
                if(text_tweets_user[j].startswith('RT',0)):
                    screen_name_mention.append([screen_name_mention1[0]])                    
                else:
                    screen_name_mention.append(screen_name_mention1)
        else:
            screen_name_mention.append(["Empty"])

    if(len(set(text_duplicated)) != len((text_duplicated))):
        duplicated.append(len(text_duplicated) - len(set(text_duplicated)))
    else:
        duplicated.append(0)
    
    #Maximun tweets for user
    if not num_tw_days:
        text_tweets_days_list.append([{"text":"Empty", "lang":"und","RT":2}])
        tweet_date_days_list.append(["Empty"])
        tw_per_day_list.append([0])
    else:
        text_tweets_days_list.append(text_tweets_days)
        tweet_date_days_list.append(tweet_date_days)
        tw_per_day_list.append(num_tw_days)
   
    tw_day_list.append(list(sorted(set(tweets_date.dt.date), reverse=True)))
    screen_name_mention_user.append(screen_name_mention)
    seasonalityDF.append(np.std(time_btw_tw))
    tweets_urlDF.append(tweets_url)
    mentionsDF.append(mentions)
    text_tweetsDF.append(tweets)
    fav_twDF_list.append(favorite_list)
    ret_twDF_list.append(retweet_list)
    fav_twDF.append(int(sum(favorite_list)))
    ret_twDF.append(int(sum(retweet_list)))
    ntwPro_DF.append(int(len(tweets)))
    RT_tw.append(int(len(RT)))
    minu_tw.append(int(sum(time_btw_tw)))
    #Time between tweets and its answer
    if not tw_answer:
        minu_tw_answer.append(0)
    else:   
        minu_tw_answer.append(int(sum(tw_answer))/len(tw_answer))
        
    return [screen_name_mention_user,seasonalityDF,tweets_urlDF,mentionsDF,text_tweetsDF,
            fav_twDF_list,ret_twDF_list,fav_twDF,ret_twDF,RT_tw,ntwPro_DF,minu_tw,tw_per_day_list,
            text_tweets_days_list,tweet_date_days_list,duplicated,minu_tw_answer,tw_day_list]

def time_twitter_account(df):

    anio_actual = time.strftime("%Y")
    df["creation_day"] = 0
    df["twitter_years"] = 0
    for i in range(len(df)):
        df["creation_day"][i] = df.iloc[i]["created_at"].split(' ')[-1]
        df["twitter_years"][i] = int(anio_actual) - int(df['creation_day'][i]) + 1
    return df

def time_series_tw (df):
    for index, row in df.iterrows():
        restart=True
        while restart == True:
            restart=False
            for k in range(len(df.loc[index, "tw_day_list"])-1):
                if(int((df.loc[index, "tw_day_list"][k] - df.loc[index,"tw_day_list"][k+1]).days))>1:
                    times = int((df.loc[index, "tw_day_list"][k] - df.loc[index,"tw_day_list"][k+1]).days)
                    for j in range(times-1):
                        df.loc[index,"tw_per_day_list"].insert(k+1,0)
                        df.loc[index, "tweet_date_days_list"].insert(k+1,["Empty"])
                        df.loc[index,"text_tweets_days_list"].insert(k+1, [{"text":"Empty","lang": "und","RT":2}])
                        df.loc[index, "tw_day_list"].insert(k+j+1, (df.loc[index,"tw_day_list"][k] - timedelta(days=j+1)))
                    restart = True
    return df 

#-------------------------------------------Ratios and other advanced features----------------------------------#
def advanced_features(data):

    #Posible bot
    data["screen_name_bot"] = 0 
    data["days_with_tw"] = 0
    data["max_fav_tw"] = 0
    data["max_ret_tw"] = 0
    data["follow_rate"] = 0
    data["max_tw_day"] = 0
    data["index_max_day_tw"]=0
    for i in range(len(data)):
        if (len(data.iloc[i]["screen_name"]) > 4):
            if((data.iloc[i]["screen_name"][-1].isdigit()) & (data.iloc[i]["screen_name"][-2].isdigit()) & 
                (data.iloc[i]["screen_name"][-3].isdigit()) &(data.iloc[i]["screen_name"][-4].isdigit())):
                data["screen_name_bot"].iloc[i] = 1
        else:
            data["screen_name_bot"].iloc[i] = 1   

        if(data.iloc[i]["following"]== 0):
            data["follow_rate"].iloc[i] = 0
        else:
            data["follow_rate"].iloc[i] = (data.iloc[i]["following"] / data.iloc[i]["followers"])
        
        #ast.literal_eval cuando se leen los datos del csv porque no recoge bien la lista de valores
        #Favorites and Retweets in their published tweets
        if not ((data["favorite_tweets_list"].iloc[i])):
            data["max_fav_tw"].iloc[i] = 0
        else:
            data["max_fav_tw"].iloc[i] = max((data["favorite_tweets_list"].iloc[i]))
        
        if not ((data["retweet_tweets_list"].iloc[i])):
            data["max_ret_tw"].iloc[i] = 0
        else:
            data["max_ret_tw"].iloc[i] = max((data["retweet_tweets_list"].iloc[i]))

        data["max_tw_day"].iloc[i] =max((data["tw_per_day_list"].iloc[i]))
        data["days_with_tw"].iloc[i] = len(data["tw_per_day_list"].iloc[i])
        data["index_max_day_tw"].iloc[i] = data["tw_per_day_list"].iloc[i].index(data["max_tw_day"].iloc[i])
        data["follow_rate"][data["followers"]== 0] = np.median(data["follow_rate"])
        data["num_userTw"]= data["num_ownTw"] + data["RT"]
        data["tw_RT_rate"] = ((((data["num_ownTw"]))) / (data["num_userTw"])).replace([np.inf, -np.inf], np.nan).fillna(0)
        data["sum_fav_RT_ownTw"] = (data["retweet_tweets_count"] + data["favorite_tweets_count"]).replace([np.inf, -np.inf], np.nan).fillna(0)
        data["iter_fav_RT_rate"] = (data["sum_fav_RT_ownTw"] / data["num_ownTw"]).replace([np.inf, -np.inf], np.nan).fillna(0)
        data["tw_year_rate"] = ((data["statuses_count"]) / (data["twitter_years"])).replace([np.inf, -np.inf], np.nan).fillna(0)
        data["time_tw_rate"] = (data["time_btw_tw"] / data["num_userTw"]).replace([np.inf, -np.inf], np.nan).fillna(0)
        data["seasonality"] = data["seasonality"].replace([np.inf, -np.inf], np.nan).fillna(0)
        data["fake_sum"] = data["default_profile_image"] + data["biography_profile"]+ data["screen_name_bot"]
        data["fake_rate"] = ((data["tweets_url"] + data["mentions"]) /data["num_ownTw"]).replace([np.inf, -np.inf], np.nan).fillna(0)
    data = trend_tweets_features(data) 
        
    return data

def trend_tweets_features(data):
    data["num_anom"] = 0
    data["trend"] = 0
    data["anom_rate"] = 0
    for i in range(len(data)):
        #trend rates
        if len(data.iloc[i]["tw_per_day_list"]) >4:
            df_aux= pd.DataFrame(data.iloc[i]["tw_per_day_list"])
            df_aux["floor"] = df_aux[0].mean() - 1.33 * df_aux[0].std()
            df_aux["roof"] = df_aux[0].mean() + 1.33 * df_aux[0].std() 
            df_aux["anom"] = df_aux.apply(lambda row: row[0] if (row[0]<= row["floor"] or 
                                                                 row[0]>= row["roof"]) else 0, axis =1) 
            data["num_anom"].iloc[i] = len(list(filter(lambda x: x != 0, df_aux["anom"])))
            data["anom_rate"].iloc[i] = data["num_anom"].iloc[i] / len(data["tw_per_day_list"].iloc[i])
            if (data["num_anom"].iloc[i] ==1):
                if max(df_aux["anom"])== data["max_tw_day"].iloc[i]:
                    data["trend"].iloc[i] = 0
                else:
                    data["trend"].iloc[i]=1                    
            else:
                if data["max_tw_day"].iloc[i] in list(filter(lambda x: x != 0, df_aux["anom"])):                    
                    data["trend"].iloc[i] = 0
                else:
                    data["trend"].iloc[i]=1  
        else:
            data["trend"].iloc[i] =2 
            data["num_anom"].iloc[i] =0
            data["anom_rate"].iloc[i]=0
    return data
#----------------------------------------------Filters-------------------------------------------------------#



def filter_possible_irregular_profiles(data_join):
    data_join["evaluate_NLP"] ="No"   
    
    
    #Profiles old spreader
    data_join["evaluate_NLP"][(data_join["index_max_day_tw"] >=30) & (data_join["profile_type"]==False) &
                  (data_join["max_tw_day"]>60)] ="NLP-F0"  
    
    #Profiles with low number of tweets but with a high iteration
    data_join["evaluate_NLP"][(data_join["trend"] == 0) & (data_join["profile_type"]==False) &
                  (data_join["max_tw_day"]<40) & (data_join["tw_RT_rate"]<0.2) &
                 ((data_join["max_ret_tw"]>500) | (data_join["max_fav_tw"]>500))] ="NLP-F1"
    
    #Profiles with a high number of RT in a especific day (try to spread information)
    data_join["evaluate_NLP"][(data_join["trend"] == 0) & (data_join["profile_type"]==False) &
                  (data_join["max_tw_day"]>150) & (data_join["tw_RT_rate"]<0.1) & 
                (data_join["evaluate_NLP"]=="No")] = "NLP-F2"
    
    #Possible influencer profile
    data_join["evaluate_NLP"][(data_join["trend"]==1) & (data_join["profile_type"]==False) &
             (data_join["in_degree"]>=5) & (data_join["followers"]>=3000) &
             (data_join["evaluate_NLP"]=="No")] = "NLP-F3"
    
    #Possible constant spreader
    data_join["evaluate_NLP"][(data_join["trend"]==1) & (data_join["profile_type"]==False) &
             (data_join["out_degree"]>=2) & (data_join["following"]>=3000) & (data_join["evaluate_NLP"]=="No")] = "NLP-F4"                

    #Possible new account with a high activity last days
    data_join["evaluate_NLP"][(data_join["trend"]==2) & (data_join["profile_type"]==False) &
             (data_join["twitter_years"]==1) & (data_join["max_tw_day"]>180) &
              (data_join["evaluate_NLP"]=="No")] = "NLP-F5"                     

    #Possible fake behaviour profile in the last days
    data_join["evaluate_NLP"][(data_join["trend"]==2) & (data_join["profile_type"]==False) &
             (data_join["fake_sum"]>=2) & (data_join["evaluate_NLP"]=="No")] = "NLP-F6" 
    
    #Possible influencers profile in the last days with a high activity
    data_join["evaluate_NLP"][(data_join["trend"]==2) & (data_join["profile_type"]==False) &
             ((data_join["max_ret_tw"]>500) | (data_join["max_fav_tw"]>500)) &
                  (data_join["max_tw_day"]>180) &  (data_join["evaluate_NLP"]=="No")] = "NLP-F7"
    
    #Profiles BOT (more than tweet or RT each 5 minutes the 24 hours)
    data_join["evaluate_NLP"][(data_join["max_tw_day"]>288) &(data_join["profile_type"]==False) &
                             (data_join["evaluate_NLP"]=="No")] ="NLP-8"
    
    data_join["evaluate_NLP"][(data_join["trend"]==2) & (data_join["profile_type"]==False) & 
                              (data_join["max_tw_day"]>180) & (data_join["evaluate_NLP"]=="No")] = "More Data"

    return data_join

	
def evaluate_NLP_profiles(df):
	for i in range(len(df)):
		tweets = df.iloc[i]["text_tweets_days_list"]
		tweets_join = [j for i in tweets for j in i]
		tweets_df = pd.DataFrame(tweets_join)
		tweets_df.to_csv("tweets_profile_example.csv",index=False)
	return tweets_df