import os
import pandas as pd
from datetime import date, timedelta
import yfinance as yf
import requests
import ast
from sqlalchemy import create_engine
import math
import numpy as np

class TickerDataFetch:

    def __init__(self):
        self.todays_date = str(date.today())
        self.script_dir = os.path.dirname(os.path.abspath(__file__))
        self.output_folder = os.path.join(self.script_dir, "Ticker Data")

    def folder_scan(self):
        os.makedirs(self.output_folder, exist_ok=True)
        self.List = os.listdir(self.output_folder)
        self.namelist = {}
        for item in self.List:
            if '.csv' not in item:
                self.List.remove(item)
        for item in self.List:
            ticker_name = item.split('.')[0].split()[0]
            ticker_date = item.split('.')[0].split()[1]
            self.namelist[ticker_name] = ticker_date

    def fetch_historical_data(self, ticker):
        ticker_data = yf.download(ticker, period='max')
        os.makedirs(self.output_folder, exist_ok=True)
        csv_path = os.path.join(self.output_folder, f"{ticker} {self.todays_date}.csv")
        DataFrame = pd.DataFrame(ticker_data)
        DataFrame.columns = ['Close','High','Low','Open','Volume']
        DataFrame = DataFrame[::-1]
        DataFrame = DataFrame.dropna()
        DataFrame.to_csv(csv_path,index=True,header=True)
        print(f"Saved CSV to: {csv_path}")

    def update_tickers (self,tickers): 
        self.folder_scan()
        for ticker in tickers:
            if ticker in self.namelist:
                if self.namelist[ticker] == self.todays_date:
                    print(ticker,"data found for",self.todays_date)
                else:
                    os.remove(os.path.join(self.output_folder, f"{ticker} {self.namelist[ticker]}.csv"))
                    self.fetch_historical_data(ticker)
                    print(ticker,"data found for",self.namelist[ticker],". Updating data for",self.todays_date)
            else:
                self.fetch_historical_data(ticker)
                print(ticker,"data not found",{ticker},". Updating data for",self.todays_date)
        for ticker in self.namelist:
            if ticker in tickers:
                pass
            else:
                os.remove(os.path.join(self.output_folder, f"{ticker} {self.namelist[ticker]}.csv"))
                print("Removed",ticker,".")
        self.folder_scan()

    def import_tickers(self):
        ticker_data = {}
        self.folder_scan()
        for ticker in self.namelist:
            data = pd.read_csv(os.path.join(self.output_folder, f"{ticker} {self.namelist[ticker]}.csv"))
            data.sort_index(ascending=False)
            data.set_index('Date',inplace=True)
            ticker_data[ticker] = data
        return ticker_data

class WeatherDataFetch:

    def __init__(self):
        self.todays_date = str(date.today())
        self.script_dir = os.path.dirname(os.path.abspath(__file__))
        self.daily_output_folder = os.path.join(self.script_dir, "Daily Weather Data")
        self.hourly_output_folder = os.path.join(self.script_dir, "Hourly Weather Data")
        os.makedirs(self.daily_output_folder, exist_ok=True)
        os.makedirs(self.hourly_output_folder, exist_ok=True)

    def folder_scan(self):
        self.List = os.listdir(self.daily_output_folder)
        for item in self.List:
            if '.csv' not in item:
                self.List.remove(item)
        self.namelist = {}
        for item in self.List:
            location_name = item.split('.')[0].split(';')[0]
            location_date = item.split('.')[0].split(';')[1]
            self.namelist[location_name] = location_date

    def build_weather_data(self, locations, range):
        date1=str(date.today()-timedelta(days=range))
        date2=str(date.today()-timedelta(days=1))
        for location in locations:
            self.folder_scan()
            daily_csv_path = os.path.join(self.daily_output_folder, f"{location};{self.todays_date}.csv")
            hourly_csv_path = os.path.join(self.hourly_output_folder, f"{location};{self.todays_date}.csv")
            
            def fetch_daily_weather_data(location,date1,date2,API_key_list):
                for i in API_key_list:
                    URL = f'https://weather.visualcrossing.com/VisualCrossingWebServices/rest/services/timeline/{location}/{date1}/{date2}?key={i}'
                    response = requests.get(URL)
                    if response.status_code == 200:
                        data = response.json()
                        break
                    else:
                        print(response)
                        print(i,"key API limit reached. Trying next key . . .")
                if 'days' not in data:
                    print("API limit reached or network error occured.")
                new_weather_data = pd.DataFrame(data['days'])
                new_weather_data = new_weather_data[::-1]
                return new_weather_data

            def fetch_hourly_weather_data(location):
                hourly_weather_data = {}
                self.folder_scan()
                data = pd.read_csv(os.path.join(self.daily_output_folder, f"{location};{self.namelist[location]}.csv"))
                data.sort_index(ascending=False)
                data.set_index('datetime',inplace=True)
                data = data.iloc[:,-1]
                hourly_weather_data[location] = data.to_dict()
                hourly_weather_data_date_dict = {}
                for date in hourly_weather_data[location]:
                    hour_data = ast.literal_eval(hourly_weather_data[location][date])
                    hour_data_df = pd.DataFrame.from_dict(hour_data)
                    hour_data_df.set_index('datetime',inplace=True)
                    hour_data_df = hour_data_df.iloc[:,:-2][::-1]
                    hourly_weather_data_date_dict[date]= hour_data_df
                hourly_weather_data_date_df = pd.concat(hourly_weather_data_date_dict,axis=0)
                return hourly_weather_data_date_df

            if location in self.namelist:
                if self.namelist[location] != self.todays_date:
                    old_weather_data = pd.read_csv(os.path.join(self.daily_output_folder, f"{location};{self.namelist[location]}.csv"))
                    new_weather_data = fetch_daily_weather_data(location,date1,date2)
                    weather_data = pd.concat([old_weather_data,new_weather_data]).drop_duplicates()
                    os.remove(os.path.join(self.daily_output_folder, f"{location};{self.namelist[location]}.csv"))
                    weather_data.to_csv(daily_csv_path,index=True,header=True)
                    print(f"{location};{self.todays_date}.csv to: {daily_csv_path}")
                    new_weather_hourly_data = fetch_hourly_weather_data(location)
                    new_weather_hourly_data.to_csv(hourly_csv_path,index=True,header=True)

                else:
                    print(f"Current data for {location} found.")
                    new_weather_hourly_data = fetch_hourly_weather_data(location)
                    new_weather_hourly_data.to_csv(hourly_csv_path,index=True,header=True)
            else:
                new_weather_data = fetch_daily_weather_data(location,date1,date2)
                new_weather_data.to_csv(daily_csv_path,index=False,header=True)
                new_weather_hourly_data = fetch_hourly_weather_data(location)
                new_weather_hourly_data.to_csv(hourly_csv_path,index=True,header=True)
                print(f"{location};{self.todays_date}.csv to: {daily_csv_path}")
            
    def import_daily_weather_data(self):
        weather_data = {}
        self.folder_scan()
        for location in self.namelist:
            data = pd.read_csv(os.path.join(self.daily_output_folder, f"{location};{self.namelist[location]}.csv"))
            data.sort_index(ascending=False)
            data.set_index('datetime',inplace=True)
            weather_data[location] = data
        return weather_data

    def import_hourly_weather_data(self):
        self.folder_scan()
        hourly_weather_data ={}
        for location in self.namelist:
            hourly_data = pd.read_csv(os.path.join(self.hourly_output_folder, f"{location};{self.todays_date}.csv"),index_col=[0,1])
            hourly_weather_data[location] = hourly_data
        return hourly_weather_data
        
class DataManipulation:
    def __init__(self):
        pass

    def remove_duplicate_columns(data):
        data = data.transpose()
        data = data.drop_duplicates()
        data = data.transpose()
        print('Dropped duplicate columns.')
        return data


    def csv_to_sql(self, folder_path):
        db_name = folder_path.split('\\')[-1]
        db_name = f'{db_name}.db'
        db_path = os.path.join(folder_path,db_name)
        engine = create_engine(f'sqlite:///{db_path}')
        file_list = os.listdir(folder_path)
        print
        for i in file_list:
            if '.csv' in i:
                df = pd.read_csv(os.path.join(folder_path, i))
                print(df)
                df.to_sql(i,engine,if_exists="replace")

    def target_creation(self,data,targets,shift=0):
        def creation(data,targets,shift):
            data_dict = {}
            recent_obs = data.iloc[0]
            target_data_dict = {}
            feature_data = data

            for target in targets:
                try:
                    if shift>0:
                        feature_data = data.iloc[shift:]                    
                        target_data = data[[target]].iloc[:-shift]
                        target_data.index = feature_data.index
                        target_data.rename(columns={target:f'{target} {shift} Ahead'},inplace=True)
                    elif shift<0:
                        feature_data = data.iloc[:shift]                    
                        target_data = data[[target]].iloc[-shift:]
                        feature_data.index = target_data.index
                        target_data.rename(columns={target:f'{target} {-shift} Behind'},inplace=True)
                    else:
                        feature_data = feature_data.drop(target, axis=1)
                        target_data = data[[target]]
                    target_data_dict[target] = target_data
                except:
                    print('Target',target,'does not exist in data')
                    print(feature_data.columns)
        
            data_dict['Feature Data'] = feature_data
            data_dict['Target Data'] = target_data_dict
            data_dict['Most Recent Observation'] = recent_obs
            return data_dict
        for i in data:
            data[i] = creation(data[i],targets,shift)
        return data

    def feature_engineering(self, feature_data, operations=['all']):
        func_type = {'powers':['power 2', 'power 3', 'power 4', 'power 5'],
                     'pairs': ['pair mult','pair div','pair add','pair sub'],
                     'exp':['exp e', 'exp 2', 'exp 10'],
                     'log':['log e', 'log 2', 'log 10'],
                     'trig':['sin','cos','tan','asin','acos','atan', 'sinh','cosh', 'tanh','asinh','acosh','atanh'],
                     'window_stats':['window_stats 5', 'window_stats 7','window_stats 10', 'window_stats 25','window_stats 60','window_stats 100','window_stats 365']}
        
        for i in func_type:
            if 'all' in operations or i in operations:
                operations = operations + func_type[f'{i}']

        for i in feature_data:
            for operation in operations:
                list = operation.split()
                #powers
                powers = pd.DataFrame()
                if list[0] == 'power':
                    powers[f'Power {list[1]} {i}'] = feature_data[f'{i}']**float(list[1])
                    
                #pairs
                pairs = pd.DataFrame()
                if list[0] == 'pair':
                    for j in feature_data:
                        if list[1] == 'mult':
                            pairs[f'mult {i}-{j}'] = feature_data[f'{i}'] * feature_data[f'{j}']
                        if list[1] == 'div':
                            pairs[f'div {i}-{j}'] = feature_data[f'{i}'] / feature_data[f'{j}'].replace(0, 1e-9)
                        if list[1] == 'add':
                            pairs[f'{list[1]} {i}-{j}'] = feature_data[f'{i}'] + feature_data[f'{j}']
                        if list[1] == 'sub':
                            pairs[f'{list[1]} {i}-{j}'] = feature_data[f'{i}'] - feature_data[f'{j}']

                #exp
                exp = pd.DataFrame()
                if list[0] == 'exp':
                    powers[f'Exp {list[1]} {i}'] = float(list[1])**feature_data[f'{i}']

                #log
                log = pd.DataFrame()
                if list[0] == 'log':
                    log[f'Log {list[1]} {i}'] = np.log(feature_data[f'{i}'],float(list[1]))

                #trig
                trig = pd.DataFrame()
                if list[0] in ['sin','cos','tan','asin','acos','atan', 'sinh','cosh', 'tanh','asinh','acosh','atanh']:
                    func = getattr(math, list[0])
                    trig[f'{list[0]} {i}'] = func(feature_data[f'{i}'])

                #window_stats_n
                window_stats = pd.DataFrame()
                inverted = feature_data[f'{i}'].iloc[::-1]
                if list[0] == 'window_stats':
                    window_stats[f'min {list[1]} {i}'] = inverted.rolling(window=list[1],min_periods=1).min().iloc[::-1]
                    window_stats[f'max {list[1]} {i}'] = inverted.rolling(window=list[1],min_periods=1).max().iloc[::-1]
                    window_stats[f'mean {list[1]} {i}'] = inverted.rolling(window=list[1],min_periods=1).mean().iloc[::-1]
                    window_stats[f'median {list[1]} {i}'] = inverted.rolling(window=list[1],min_periods=1).median().iloc[::-1]
                    window_stats[f'skew {list[1]} {i}'] = inverted.rolling(window=list[1], min_periods=1).skew().iloc[::-1]
                    window_stats[f'kurt {list[1]} {i}'] = inverted.rolling(window=list[1], min_periods=1).kurt().iloc[::-1]
                    window_stats[f'std {list[1]} {i}'] = inverted.rolling(window=list[1],min_periods=1).std().iloc[::-1]
                    window_stats[f'min-max change {list[1]} {i}'] = (window_stats[f'max {list[1]} {i}'] - window_stats[f'min {list[1]} {i}'])
                    window_stats[f'min-max % change {list[1]} {i}'] = (window_stats[f'max {list[1]} {i}'] - window_stats[f'min {list[1]} {i}'])/window_stats[f'min {list[1]} {i}']
                    window_stats[f'left-right change {list[1]} {i}'] = inverted.rolling(window=list[1]).apply(lambda x: x[0]- x[-1], raw=True).iloc[::-1]
                    window_stats[f'left-right % change {list[1]} {i}'] = inverted.rolling(window=list[1]).apply(lambda x: (x[0]- x[-1])/x[-1], raw=True).iloc[::-1]
                    window_stats[f'CV {list[1]} {i}'] = window_stats[f'std {list[1]} {i}']/window_stats[f'mean {list[1]} {i}'].replace(0, 1e-9)
                    window_stats[f'zscore {list[1]} {i}'] = ((inverted - window_stats[f'mean {list[1]} {i}']) / window_stats[f'std {list[1]} {i}']).iloc[::-1]
                    window_stats[f'momentum {list[1]} {i}'] = inverted.diff(list[1]).iloc[::-1]
                    window_stats[f'entropy {list[1]} {i}'] = (inverted.rolling(window=list[1], min_periods=1).apply(lambda x: -np.sum((p:=x.value_counts(normalize=True)) * np.log(p + 1e-9))).iloc[::-1])
                    window_stats[f'slope {list[1]} {i}'] = (inverted.rolling(window=list[1], min_periods=1).apply(lambda x: np.polyfit(range(len(x)), x, 1)[0]).iloc[::-1])

                    #RSI
                    avg_gain = inverted.diff().clip(lower=0).rolling(window=list[1], min_periods=1).mean()
                    avg_loss = -inverted.diff().clip(upper=0).rolling(window=list[1], min_periods=1).mean()
                    rs = avg_gain / avg_loss.replace(0, 1e-9)
                    rsi = 100 - (100 / (1 + rs))
                    window_stats[f'RSI {list[1]} {i}'] = rsi.iloc[::-1]
                    #MACD
                    short_span = max(2, int(int(list[1]) / 2))
                    long_span = max(3, int(int(list[1]) * 1.5))
                    short_ema = inverted.ewm(span=short_span, adjust=False).mean()
                    long_ema  = inverted.ewm(span=long_span,  adjust=False).mean()
                    macd = short_ema - long_ema
                    window_stats[f'MACD {list[1]} {i}'] = macd.iloc[::-1]

            feature_data = pd.concat([feature_data,powers,pairs, exp, log, trig, window_stats],axis=1)

        print(feature_data)



