import os
import datetime as dt
import re
import pickle
import csv
from os import listdir
from os.path import isfile, join
import datetime as dt

class FileMatchException(Exception):
    def __init__(self, message):
        super().__init__(message)
        
def save_lists(json_response_list, json_sample_list, sentiment_list, sentiment_sample, start_t, end_t, new_end_t, json_max, interval_len):
    """
    Stores tweet data collected in files for later access

    Parameters
    --------
    json_response_list : list of strings
        cleaned text of tweets collected containing certain keywords
    json_sample_list : list of strings
        cleaned text of random tweets collected 
    sentiment_list- : list of floats
        sentiment scores of tweets collected containing certain keywords
    sentiment_sample : list of floats
        sentiment scores of random tweets collected 
    start_t- : string
        Start date of data collection.
    end_t : string
        End date of previous data collection file
    new_end_t : string
        End date of data collection
    json_max : int
        number of tweets collected per interval
    interval_len : int
        number of minutes per interval

    Returns
    --------
    None
    
    Raises
    --------
    FileMatchException
        Raised if files are unexpectedly missing when searched for
    """
    
    #string format of the time at the current moment
    time_now = dt.datetime.now().isoformat()
    
    #get path to parent directory of this file
    rel_path = os.path.dirname(os.path.realpath(__file__))

    #construct the full path to store collected tweet data
    datadir = "storedqueries"
    fullpath = os.path.join(rel_path, datadir)
    
    #remove illegal characters from filename
    tweetfile = "tweet_" + start_t + "_" + end_t + "_" + str(json_max) + "_" + str(interval_len) + "_"
    tweetfile = re.sub(r":",".",tweetfile)
    tweetfile = re.sub(r"/",".",tweetfile)
    tweetfile = re.sub(r"\.",".",tweetfile)
    sample_tweetfile = tweetfile
    tweetfile += ".csv"
    sample_tweetfile += "sample.csv"
    os.makedirs(os.path.abspath(fullpath), mode=0o666, exist_ok=True)
    
    #make new filename for updated enddate
    newtweetfile = "tweet_" + start_t + "_" + new_end_t + "_" + str(json_max) + "_" + str(interval_len) + "_"
    newtweetfile = re.sub(r":",".",newtweetfile)
    newtweetfile = re.sub(r"/",".",newtweetfile)
    newtweetfile = re.sub(r"\.",".",newtweetfile)
    new_sample_tweetfile = newtweetfile
    newtweetfile += ".csv"
    new_sample_tweetfile += "sample.csv"
    
    #remove illegal characters from filename
    sentifile = "senti_" + start_t + "_" + end_t + "_" + str(json_max) + "_" + str(interval_len) + "_"
    sentifile = re.sub(r":",".",sentifile)
    sentifile = re.sub(r"/",".",sentifile)
    sentifile = re.sub(r"\.",".",sentifile)
    sample_sentifile = sentifile
    sentifile += ".csv"
    sample_sentifile += "sample.csv"
    
    #make new filename for updated enddate
    newsentifile = "senti_" + start_t + "_" + new_end_t + "_" + str(json_max) + "_" + str(interval_len) + "_"
    newsentifile = re.sub(r":",".",newsentifile)
    newsentifile = re.sub(r"/",".",newsentifile)
    newsentifile = re.sub(r"\.",".",newsentifile)
    new_sample_sentifile = newsentifile
    newsentifile += ".csv"
    new_sample_sentifile += "sample.csv"
    
    #store tweet text array if needed
    with open(os.path.join(fullpath,tweetfile), "a", newline='') as csvfile:
        tweetwriter = csv.writer(csvfile, delimiter=',',quotechar='|', quoting=csv.QUOTE_MINIMAL)
        for row in json_response_list:
            tweetwriter.writerow(row)
    with open(os.path.join(fullpath,sample_tweetfile), "a", newline='') as csvfile:
        tweetwriter = csv.writer(csvfile, delimiter=',',quotechar='|', quoting=csv.QUOTE_MINIMAL)
        for row in json_sample_list:
            tweetwriter.writerow(row)
            
    #store tweet sentiment array if needed
    with open(os.path.join(fullpath,sentifile), "a", newline='') as csvfile:
        sentiwriter = csv.writer(csvfile, delimiter=',',quotechar='|', quoting=csv.QUOTE_MINIMAL)
        for row in sentiment_list:
            sentiwriter.writerow(row)
    with open(os.path.join(fullpath,sample_sentifile), "a", newline='') as csvfile:
        sentiwriter = csv.writer(csvfile, delimiter=',',quotechar='|', quoting=csv.QUOTE_MINIMAL)
        for row in sentiment_sample:
            sentiwriter.writerow(row)
    
    #check if tweet and sentiment files with matching parameters exists
    if not (os.path.exists(os.path.join(fullpath,tweetfile)) and os.path.exists(os.path.join(fullpath,sentifile))):
        raise FileMatchException("No matching tweet and sentiment files found")
        
    #rename file to reflect updated enddate
    if os.path.exists(os.path.join(fullpath,tweetfile)) and tweetfile != newtweetfile:
        os.rename(os.path.join(fullpath,tweetfile), os.path.join(fullpath,newtweetfile))
        
    if os.path.exists(os.path.join(fullpath,sample_tweetfile)) and tweetfile != new_sample_tweetfile:
        os.rename(os.path.join(fullpath,sample_tweetfile), os.path.join(fullpath,new_sample_tweetfile))
        
    #rename file to reflect updated enddate
    if os.path.exists(os.path.join(fullpath,sentifile)) and sentifile != newsentifile:
        os.rename(os.path.join(fullpath,sentifile), os.path.join(fullpath,newsentifile))
        
    if os.path.exists(os.path.join(fullpath,sample_sentifile)) and sentifile != new_sample_sentifile:
        os.rename(os.path.join(fullpath,sample_sentifile), os.path.join(fullpath,new_sample_sentifile))
        
def find_file(onlyfiles, prefix, json_max, interval_len, has_sample):
    """
    Given a list of parameters, this method returns data about the
    file that matches those parameters
    
    Parameters
    --------
    onlyfiles : list of paths
        Contains references to the files to be searched on with this method
    prefix : string
        file prefix to be searched for
    json_max : int
        max number of results stored in file for each time interval
    interval_len : int
        Represents length of collection intervals in minutes
    has_sample : boolean
        Represents whether the file is a comparison sample or contains the
        requested keywords

    Returns
    --------
    (past_file, datestr, datestr2, totaltime) : Tuple
        Contains the string file name that was searched for, the string start
        and end date of data storage and the int length of data collection in minutes
        
    Raises
    --------
    
    """
    
    past_file = ""

    #search in specified file list for sentiment file that matches specified data storage file values (which are contained within the file name)
    for file in onlyfiles:
        file_data = file.split("_")
        file_prefix = file_data[0]
        if file_prefix == "":
            continue
        file_startdate = file_data[1]
        datestr = file_startdate
        date_s = file_startdate.split(".")
        file_enddate = file_data[2]
        datestr2 = file_enddate
        date_c = file_enddate.split(".")
        file_json_max = int(file_data[3])
        file_interval_len = int(file_data[4])
        if ((file_prefix == prefix) and file_json_max == json_max and file_interval_len == interval_len):
            if has_sample:
                if "sample" in file:
                    past_file = file
                    break
            else:
                if "sample" not in file:
                    past_file = file
                    break
                    
    if past_file == "":
        raise FileMatchException("No sentiment storage files found")
        
    #create dates that represent the start and stop dates of file data storage
    start_date = dt.date(int(date_s[2])+2000, int(date_s[0]), int(date_s[1])) #using the date values from tweet_date_s and tweet_date_c is arbitrary; you could also use the equivalent sentiment values
    end_date = dt.date(int(date_c[2])+2000, int(date_c[0]), int(date_c[1]))
            
    #retrieve timezone-aware time at midnight
    ti = dt.time(tzinfo = dt.timezone.utc) 
        
    #make datetime from date and utc time 
    start_dt = dt.datetime.combine(start_date,ti) 
    end_dt = dt.datetime.combine(end_date,ti)
    
    #calculate totaltime from beginning of data collection to end
    totaltime = int((end_dt - start_dt) / dt.timedelta(minutes = 1))
        
    return (past_file, datestr, datestr2, totaltime)
        
def load_lists(json_max, interval_len):
    """
    Retrieve tweet data lists from file storage based on search parameters
    
    Parameters
    --------
    json_max- : int
        number of tweets collected per interval
    interval_len : int
        number of minutes per interval
        
    Returns
    --------
    (sentiment_list,tweet_list,sentiment_sample, tweet_sample, totaltime) : Tuple
        All lists that correspond to the search parameters received by this
        method as well as the total time (int) in minutes that the files contain
        data over

    Raises
    --------
    
    """
    
    #get path to parent directory of this file
    rel_path = os.path.dirname(os.path.realpath(__file__))

    #construct the full path to store collected tweet data
    datadir = "storedqueries"
    mypath = os.path.join(rel_path, datadir)
            
    onlyfiles = [f for f in listdir(mypath) if isfile(join(mypath, f))]
    
    past_sentifile = ""
    past_tweetfile = ""
    senti_datestr = "a"
    tweet_datestr = "b"
    tweet_datestr2 = "c"
    senti_datestr2 = "d"
    
    if not onlyfiles:
        raise FileMatchException("No data storage files found")
        
    #retrieve file name with specified parameters
    past_sentifile, senti_datestr, senti_datestr2, _ = find_file(onlyfiles, 'senti', json_max, interval_len, False)
    past_tweetfile, tweet_datestr, tweet_datestr2, _ = find_file(onlyfiles, 'tweet', json_max, interval_len, False)
    
    #retrieve file name with specified parameters that contains sample comparison data
    past_sample_sentifile, sample_senti_datestr, sample_senti_datestr2, _ = find_file(onlyfiles, 'senti', json_max, interval_len, True)
    past_sample_tweetfile, sample_tweet_datestr, sample_tweet_datestr2, totaltime = find_file(onlyfiles, 'tweet', json_max, interval_len, True)
    
    if tweet_datestr != tweet_datestr or tweet_datestr2 != senti_datestr2:
        raise FileMatchException("No matching tweet and sentiment files found")
    else:
        sentiment_list = []
        if os.path.exists(os.path.join(mypath,past_sentifile)):
            #retrieve tweet sentiment array 
            with open(os.path.join(mypath,past_sentifile), "r", newline='') as csvfile:
                sentireader = csv.reader(csvfile, delimiter=',',quotechar='|', quoting=csv.QUOTE_MINIMAL)
                for row in sentireader:
                    sentiment_list.append([float(row[0]), float(row[1])])
        sentiment_sample = []
        if os.path.exists(os.path.join(mypath,past_sample_sentifile)):
            #retrieve tweet sentiment array 
            with open(os.path.join(mypath,past_sample_sentifile), "r", newline='') as csvfile:
                sentireader = csv.reader(csvfile, delimiter=',',quotechar='|', quoting=csv.QUOTE_MINIMAL)
                for row in sentireader:
                    sentiment_sample.append([float(row[0]), float(row[1])])

        tweet_list = []
        if os.path.exists(os.path.join(mypath,past_tweetfile)):
            #retrieve tweet sentiment array 
            with open(os.path.join(mypath,past_tweetfile), "r", newline='') as csvfile:
                tweetreader = csv.reader(csvfile, delimiter=',',quotechar='|', quoting=csv.QUOTE_MINIMAL)
                for row in tweetreader:
                    tweet_list.append(row)
        tweet_sample = []
        if os.path.exists(os.path.join(mypath,past_sample_tweetfile)):
            #retrieve tweet sentiment array 
            with open(os.path.join(mypath,past_sample_tweetfile), "r", newline='') as csvfile:
                tweetreader = csv.reader(csvfile, delimiter=',',quotechar='|', quoting=csv.QUOTE_MINIMAL)
                for row in tweetreader:
                    tweet_sample.append(row)

        return (sentiment_list,tweet_list,sentiment_sample, tweet_sample, totaltime)
    
