"""
Created on 08.06.2024

@author: Mathias Berg Rosendal, PhD Student at DTU Management (Energy Economics & Modelling)
"""
#%% ------------------------------- ###
###        0. Script Settings       ###
### ------------------------------- ###

import os
import sys
import shutil
import gams
import pandas as pd
import numpy as np
import pickle as pkl
import tsam
import requests
from datetime import datetime
from dataclasses import dataclass, field
from urllib.parse import urljoin
from pathlib import Path
from typing import Union, Tuple
from matplotlib.figure import Figure
from matplotlib.axes import Axes
from .utils import symbol_to_df, search_in_incfiles, prepare_incfile
from .formatting import SSS_TTT_index
from .interactive.interactive_functions import interactive_bar_chart
from .interactive.dashboard.eel_dashboard import interactive_geofilemaker
from .plotting.production_profile import plot_profile
from .plotting.maps_balmorel import plot_map

#%% ------------------------------- ###
###           1. Outputs            ###
### ------------------------------- ###

class MainResults:
    def __init__(self, files: Union[str, list, tuple], 
                 paths: Union[str, list, tuple] = '.', 
                 scenario_names: str | list | tuple | None = None,
                 system_directory: str | None = None,
                 result_type: str = 'balmorel'):
        """
        Initialises the MainResults class and loads gdx result file(s)

        Args:
            files (str, list, tuple): Name(s) of the gdx result file(s)
            paths (str, list, tuple): Path(s) to the gdx result file(s), assumed in same path if only one path given, defaults to working directory
            scenario_names (str, list, tuple): Name of scenarios corresponding to each gdx file, defaults to ['SC1', 'SC2', ..., 'SCN'] if None given
            system_directory (str, optional): GAMS system directory. Is not used if not specified.
            result_type (str, optional): Specifies the type of result to extract. Use 'optiflow' for OptiFlow results. If not specified, it defaults to extracting Balmorel GDX results.
        """

        ## Loading scenarios
        if type(files) is str:
            # Change filenames to list if just one string
            files = [files]
            
        ## File paths
        if type(paths) is str:
            # Create identical paths if only one given
            paths = [paths]*len(files)
            
        elif ((type(paths) is list) or (type(paths) is tuple)) and (len(paths) == 1):
            paths = paths*len(files)
            
        elif len(files) != len(paths):
            # Raise error if not given same amount of paths and files     
            raise Exception("%d files, but %d paths given!\nProvide only one path or the same amount of paths as files"%(len(files), len(paths)))
        
        ## Scenario Names
        if scenario_names is None:
            # Try to make scenario names from filenames, if None given
            scenario_names = pd.Series(files).str.replace('MainResults_', '').str.replace('MainResults','').str.replace('.gdx', '')

            if scenario_names is None: 
                raise FileNotFoundError("Couldn't find any results")
            
            # Rename MainResults with no suffix
            if np.any(scenario_names == ''):
                idx = list(scenario_names[scenario_names == ''].index)
                for ind in idx:
                    scenario_names[ind] = paths[ind].split('/model')[0].split(r'Balmorel')[1].replace('\\','')
                                
            # Check if some names are identical
            if len(scenario_names.unique()) != len(scenario_names):
                print('\n--------------WARNING!--------------\nIdentical scenario names detected, which will result in double counting when analysing')
                print('Scenarios: ', ', '.join(list(scenario_names)), '\n--------------WARNING!--------------\n')                

            scenario_names = list(scenario_names)

        elif type(scenario_names) is str:
            scenario_names = [scenario_names]
            
        if len(files) != len(scenario_names):    
            # Raise error if not given same amount of scenario_names and files
            raise Exception("%d files, but %d scenario names given!\nProvide none or the same amount of scenario names as files"%(len(files), len(scenario_names)))
        
        ## Store MainResult databases
        self.files = files
        self.paths = paths
        self.sc = scenario_names
        self.type = result_type
        self.db = {}
            
        if system_directory is not None:
            ws = gams.GamsWorkspace(system_directory=system_directory)
            self._gams_system_directory = system_directory
        else:
            ws = gams.GamsWorkspace()
            
        for i in range(len(files)):    
            print('Loading', str(Path(paths[i]) / files[i]))
            try:
                self.db[scenario_names[i]] = ws.add_database_from_gdx(str((Path(paths[i]) / files[i]).absolute()))
            except gams.GamsException:
                raise FileNotFoundError(f'\nCouldnt add file {files[i]}!\nBeware of æ,ø,å,ö,ü,ä or other non-english letters in the folders of your absolute path: {os.path.abspath(paths[i])}.\nThe GAMS API requires an absolute path with no non-english letters.')
     
    # Getting a certain result
    def get_result(self, symbol: str, cols: list | None = None) -> pd.DataFrame:
        """Get a certain result from the loaded gdx file(s) into a pandas DataFrame

        Args:
            symbol (str): The desired result, e.g. PRO_YCRAGF
            cols (str, optional): Specify custom columns. Defaults to pre-defined formats.

        Returns:
            pd.DataFrame: The output DataFrame
        """
        # Placeholder
        df = pd.DataFrame()
        
        for SC in self.sc:
            # Get results from each scenario
            try :
                temp = symbol_to_df(self.db[SC], symbol, cols, result_type=self.type)
                temp['Scenario'] = SC 
                # Put scenario in first column
                temp = temp.loc[:, ['Scenario'] + list(temp.columns[:-1])]
                # Save
                df = pd.concat((df, temp), ignore_index=True)
            
            except ValueError :
                print(f'{SC} doesn\'t have any value in the table {symbol}')
            
        return df  
    
    ## Plotting tools
    # Interactive bar chart plotting
    def interactive_bar_chart(self, plot_style: str = 'light'):
        """
        GUI for bar chart plotting
        """
        return  interactive_bar_chart(self, plot_style)        
    
    # Plotting a production profile
    def plot_profile(self,
                     commodity: str,  
                     year: int, 
                     scenario: str | int = 0,
                     columns: str = 'Technology',
                     region: str = 'ALL',
                     style: str = 'light') -> Tuple[Figure, Axes]:
        """Plots the production profile of a commodity, in a year, for a certain scenario

        Args:
            commodity (str): The commodity (Electricity, Heat or Hydrogen)
            year (int): The model year to plot
            scenario (str, optional): Defaults to the first scenario in MainResults.
            columns (str, optional): Technology or Fuel as . Defaults to 'Technology'.
            region (str, optional): Which country, region or area to plot. Defaults to 'ALL'.
            style (str, optional): Plot style, light or dark. Defaults to 'light'.

        Returns:
            Figure, Axes: The figure and axes objects for further manipulations 
        """
        return plot_profile(self, commodity, year, scenario, columns, region, style)
        
    
    def plot_map(self, 
                 scenario: str, 
                 year: int,
                 commodity: str | None = None,
                 lines: str | None = None, 
                 generation: str | None = None,
                 background : str | None = None,
                 save_fig: bool = False,
                 path_to_geofile: str | None = None,
                 geo_file_region_column: str = 'id',
                 **kwargs) -> Tuple[Figure, Axes]:
        """Plots the transmission capacities or flow in a scenario, of a certain commodity and the generation capacities or production of the regions.

        Args:
            path_to_result (str): Path to the .gdx file
            scenario (str): The scenario name       
            year (int): The year of the results
            commodity (str, optional): Commodity to be shown in the map. Choose from ['Electricity', 'Hydrogen'].
            lines (str, optional): Information plots with the lines. Choose from ['Capacity', 'FlowYear', 'FlowTime', 'UtilizationYear', 'UtilizationTime].
            generation (str, optional): Generation information plots on the countries. Choose from ['Capacity', 'Production', 'ProductionTime].
            background (str, optional): Background information to be shown on the map. Choose from ['H2 Storage', 'Elec Storage']. Defaults to 'None'.
            save_fig (bool, optional): Save the figure or not. Defaults to False.
            path_to_geofile (str, optional): Path to a personalized geofile. Defaults to None.
            geo_file_region_column (str, optional): Column name of the region names in the geofile. Defaults to 'id'.
            Structural additional options:
                **generation_commodity (str, optional): Commodity to be shown in the generation map, if not specified, same as line commodity. Defaults to commodity.
                **S (str, optional): Season for FlowTime or UtilizationTime. Will pick one random if not specified.
                **T (str, optional): Hour for FlowTime or UtilizationTime. Will pick one random if not specified.
                **exo_end (str, optional): Show only exogenous or endogenous capacities. Choose from ['Both', 'Endogenous', 'Exogenous']. Defaults to 'Both'.
                **generation_exclude_Import_Cap_H2 (bool, optional): Do not plot the capacities and production related to H2 Import (will be shown as line). Defaults to True.
                **generation_exclude_H2Storage (bool, optional): Do not plot capacity or production of the H2 storage. Defaults to True.
                **generation_exclude_ElectricStorage (bool, optional): Do not plot capacity or production of Electric storage. Defaults to True.
                **generation_exclude_Geothermal (bool, optional): Do not plot the production of Geothermal. Defaults to True.
                **coordinates_geofile_offset (float, optional): Geofile coordinates offset from the min and max of the geofile. Defaults to 0.5.
                **filename (str, optional): The name of the file to save, if save_fig = True. Defaults to .png if no extension is included.
            Visual additional options:
                **title_show (bool, optional): Show title or not. Defaults to True.
                **legend_show (bool, optional): Show legend_show or not. Defaults to True.
                **show_country_out (bool, optional): Show countries outside the model or not. Defaults to True.
                **choosen_map_coordinates (str, optional): Choose the map to be shown. Choose from ['EU', 'DK', 'Nordic']. Defaults to 'EU'.
                **map_coordinates (list, optional): Coordinates of the map if custom coordinates needed. Defaults to ''.
                Lines options :
                    **line_width_cat (str, optional): Way of determining lines width. Choose from ['log', 'linear', 'cluster']. Defaults to 'log'.
                    **line_show_min (int, optional): Minimum transmission capacity (GW) or flow (TWh) shown on map. Defaults to 0.
                    **line_width_min (float, optional): Minimum width of lines, used if cat is linear or log. Defaults to 0.5. Value in point.
                    **line_width_max (float, optional): Maximum width of lines, used if cat is linear or log. Defaults to 12. Value in point.
                    **line_cluster_values (list, optional): The capacity grouping necessary if cat is 'cluster'. Defaults values depends on commodity. Used for the legend if defined.
                    **line_cluster_widths (list, optional): The widths for the corresponding capacity group if cat is cluster (has to be same size as line_cluster_values). Used for the legend if defined. Values in point.
                    **line_legend_cluster_values (list, optional): The legend capacity grouping if a specific legend is needed. Is handled automatically if not defined. Not used if cat is 'cluster'.
                    **line_opacity (float, optional): Opacity of lines. Defaults to 1.
                    **line_label_show (bool, optional): Showing or not the value of the lines. Defaults to False.
                    **line_label_min (int, optional): Minimum transmission capacity (GW) or flow (TWh) shown on map in text. Defaults to 0.
                    **line_label_decimals (int, optional): Number of decimals shown for line capacities. Defaults to 1.
                    **line_label_fontsize (int, optional): Font size of transmission line labels. Defaults to 10.
                    **line_flow_show (bool, optional): Showing or not the arrows on the lines. Defaults to True.
                Generation options :
                    **generation_show_min (float, optional): Minimum generation capacity (GW) or production (TWh) shown on map. Defaults to 0.001.
                    **generation_display_type (str, optional): Type of display on regions. Choose from ['Pie']. Defaults to 'Pie'.
                    **generation_var (str, optional): Variable to be shown in the pie chart. Choose from ['TECH_TYPE', 'FFF']. Defaults to 'TECH_TYPE'.
                    **pie_radius_cat (str, optional): Way of determining pie size. Choose from ['log', 'linear', 'cluster']. Defaults to 'log'.
                    **pie_show_min (int, optional): Minimum transmission capacity (GW) or flow (TWh) shown on map. Defaults to 0. Value in data unit.
                    **pie_radius_min (float, optional): Minimum width of lines, used if cat is linear or log. Defaults to 0.2. Value in data unit.
                    **pie_radius_max (float, optional): Maximum width of lines, used if cat is linear or log. Defaults to 1.4. Value in data unit.
                    **pie_cluster_values (list, optional) = The capacity groupings necessary if cat is 'cluster'. Defaults values depends on commodity. Used for the legend if defined.
                    **pie_cluster_radius (list, optional) = The radius for the corresponding capacity group if cat is cluster (has to be same size as pie_cluster_values). Used for the legend if defined. Values in data unit.
                    **pie_legend_cluster_radius (list, optional) = The legend capacity grouping if a specific legend is needed. Is handled automatically if not defined. Not used if cat is 'cluster'. 
                Background options :
                    **background_name (str, optional): Personalized name of the background (mostly useful for Custom).
                    **background_unit (str, optional): Personalized unit of the background (mostly useful for Custom).
                    **background_scale (list, optional) : Scale used for the background coloring. Defaults to (0, Max value found in results).
                    **background_scale_tick (int, optional) : A tick every x units in the background legend. Defaults to 2.
                    **background_label_show (bool, optional): Showing or not the background label on the countries. Defaults to False.
                    **background_label_fontsize (int, optional): Font size of the background labels. Defaults to 10.
            Colors additional options:
                **background_color (str, optional): Background color of the map. Defaults to 'white'.
                **regions_ext_color (str, optional): Color of regions outside the model. Defaults to '#d3d3d3'.
                **regions_model_color (str, optional): Color of regions inside the model. Defaults to 'linen'.
                **line_color (str, optional): Color of lines network. Defaults to 'green' for electricity and '#13EAC9' for hydrogen.
                **line_label_color (str, optional): Color of line labels. Defaults to 'black'.
                **generation_tech_color (dict, optional): Dictionnary of colors for each technology. Defaults to colors for electricity and hydrogen.
                **generation_fuel_color (dict, optional): Dictionnary of colors for each fuel. Defaults to colors for electricity and hydrogen.
                **background_colormap (str, optional): Personalized background colormap on the countries.
                **background_label_color (str, optional): Color of the background labels. Defaults to 'black'.
            Geography additional options:
                **coordinates_RRR_path = Path to the csv file containing the coordinates of the regions centers.
                **bypass_path = Path to the csv file containing the coordinates of 'hooks' in indirect lines, to avoid going trespassing third regions.
                **hydrogen_third_nations_path = Path to the csv file containing the coordinates of h2 import lines from third nations.
                **countries_color_path = Path to the csv file containing the personnalized colors of the countries
                **countries_background_path = Path to the csv file containing the personnalized background of the countries

        Returns:
            Tuple[Figure, Axes]: The figure and axes objects of the plot
        """
        # Find path of scenario
        idx = np.array(self.sc) == scenario
        path = np.array(self.paths)[idx][0]
        files = np.array(self.files)[idx][0]
        path = os.path.join(path, files)
        
        if hasattr(self, "_gams_system_directory"):
            return plot_map(path, scenario, year, commodity, lines, generation, background, save_fig, path_to_geofile, geo_file_region_column, 
                            system_directory=self._gams_system_directory, **kwargs)
        else:
            return plot_map(path, scenario, year, commodity, lines, generation, background, save_fig, path_to_geofile, geo_file_region_column, **kwargs)
        
    # For wrapping functions, makes it possible to add imported functions in __init__ easily
    def _existing_func_wrapper(self, function, *args, **kwargs):
        return function(self, *args, **kwargs)     


#%% ------------------------------- ###
###            2. Inputs            ###
### ------------------------------- ###

class IncFile:
    """A useful class for creating .inc-files for GAMS models
    Args:
    prefix (str): The first part of the .inc file.
    body (str): The main part of the .inc file.
    suffix (str): The last part of the .inc file.
    name (str): The name of the .inc file.
    path (str): The path to save the file, defaults to 'Balmorel/base/data'.
    """
    def __init__(self, prefix: str = '', body: str | pd.DataFrame = '', 
                 suffix: str = '', name: str = 'name', 
                 path: str = 'Balmorel/base/data/'):
        self.prefix = prefix
        self.body = body
        self.suffix = suffix
        self.name = name
        self.path = path

    def body_concat(self, df: pd.DataFrame):
        """Concatenate a body temporarily being a dataframe to another dataframe
        """
        if type(self.body) is not pd.DataFrame:
            raise TypeError("Body must be a pandas DataFrame for this to work!")

        self.body = pd.concat((self.body, df)) # perhaps make a IncFile.body.concat function.. 

    def body_prepare(self, index: list, columns: list | None = None,
                    values: str = 'Value',
                    aggfunc: str ='sum',
                    fill_value: Union[str, int] = ''):
    
        # Pivot
        self.body = self.body.pivot_table(index=index, columns=columns, 
                            values=values, aggfunc=aggfunc,
                            fill_value=fill_value)
        
        # Check if there are multiple levels in index and 
        # concatenate with " . "
        if hasattr(self.body.index, 'levels'):
            new_ind = pd.Series(self.body.index.get_level_values(0))
            for level in range(1, len(self.body.index.levels)):
                new_ind += ' . ' + self.body.index.get_level_values(level) 

            self.body.index = new_ind
        
        # Check if there are multiple levels in columns and 
        # concatenate with " . "
        if hasattr(self.body.columns, 'levels'):
            new_ind = pd.Series(self.body.columns.get_level_values(0))
            for level in range(1, len(self.body.columns.levels)):
                new_ind += ' . ' + self.body.columns.get_level_values(level) 

            self.body.columns = new_ind
            
        # Delete names
        self.body.columns.name = ''
        self.body.index.name = ''


    def save(self):
        if self.name[-4:] != '.inc':
            self.name += '.inc'  
       
        with open(Path(self.path) / self.name, 'w') as f:
            f.write(self.prefix)
            if type(self.body) is str:
                f.write(self.body)
            elif type(self.body) is pd.DataFrame:
                f.write(self.body.to_string())
            else:
                print('Wrong format of %s.body!'%self.name)
                print('No body written')
            f.write(self.suffix)
 
class Balmorel:
    """A class that recognises the Balmorel folder structure, can be used to run scenarios or results

    Args:
        model_folder (str): The top level folder of Balmorel, where base and simex are located
    """

    def __init__(self, model_folder: str | Path, gams_system_directory: str | None  = None):
        
        # Get GAMS system directory (the default none will make GAMS find it by itself)
        self._gams_system_directory = gams_system_directory
        
        # Get full path
        self.path = Path(model_folder)
        directories = [file.name for file in self.path.iterdir() if file.is_dir()]
        
        if 'base' not in directories:
            raise Exception(f"Incorrect Balmorel folder, couldn't find base in {self.path}")
        
        # Get scenario folders
        scenarios = [SC for SC in self.path.iterdir() if SC.is_dir() and SC.name != 'simex' and SC.name[0] != '.']
        
        # Check validity of scenario folders and make list of scenarios
        self.scenarios = []
        self.input_data = {}
        for SC in scenarios:
            if (SC / 'model/Balmorel.gms').exists() and ((SC / 'model/cplex.op4').exists() or (SC / 'model/cplex.op2').exists()):
                self.scenarios.append(SC.name)
            else:
                print(f'Folder {SC} not added to scenario as the necessary {SC}/model/Balmorel.gms and/or {SC}/model/cplex.op4 or {SC}/model/cplex.op2 did not exist')

    def locate_results(self):
        """
        Locates results, which is faster than collecting them if you just want an overview
        """
                
        self.files = []
        self.paths = []
        self.scenario_names = []
        self.scfolder_to_scname = {}
        self.scname_to_scfolder = {}
        for SC in self.scenarios:
            path = self.path / f'{SC}/model'
            mainresults_files = pd.Series([file.name for file in path.iterdir()])
            mainresults_files = mainresults_files[(mainresults_files.str.find('MainResults') != -1) & (mainresults_files.str.find('.gdx') != -1)]
            self.files += list(mainresults_files)
            self.paths += [path]*len(mainresults_files)
            if len(mainresults_files) == 1:
                self.scenario_names += [SC]
                self.scfolder_to_scname[SC] = [SC]
                self.scname_to_scfolder[SC] = SC
            else:
                mainresults_files = (
                    mainresults_files
                    .str.replace('MainResults_', '')
                    .str.replace('.gdx', '')
                )
                self.scenario_names += list(mainresults_files)
                self.scfolder_to_scname[SC] = list(mainresults_files)
                
                for scenario_name in mainresults_files:
                    self.scname_to_scfolder[scenario_name] = SC 

    def collect_results(self):
        """
        Collects results
        """

        self.locate_results()

        self.results = MainResults(files=self.files, paths=self.paths, scenario_names=self.scenario_names, system_directory=self._gams_system_directory)
            
    def run(self, scenario: str, cmd_line_options: dict = {}):
        
        # Working directory
        wk_dir = os.path.join(self.path, '%s/model'%scenario)
        
        # Add options
        ws = gams.GamsWorkspace(working_directory=wk_dir, system_directory=self._gams_system_directory)
        opt = ws.add_options()
        for key in cmd_line_options.keys():
            opt.defines[key] = cmd_line_options[key]
        
        # Run Balmorel
        try:
            job = ws.add_job_from_file(os.path.join(wk_dir, 'Balmorel'), job_name=scenario)
            job.run(opt)
        except gams.GamsExceptionExecution as e:
            exec_error = e
            print('Execution error! Check output, division by zero in OUTPUT_SUMMARY can happen and may not be a problem')
        
        # Check feasibility
        with open(os.path.join(wk_dir, '%s.lst'%scenario), 'r') as f:
            output = pd.Series(f.readlines())

        output = output[(output.str.find('LP status') != -1) | (output.str.find('MIP status') != -1)] # Find all status
        all_feasible = output[output.str.find('infeasible') != -1].empty # Check if none are infeasible
        if not(all_feasible):
            raise Exception('Model run infeasible!')
        
        # Raise error from before
        if 'exec_error' in locals():
            raise exec_error
        
        
    def load_incfiles(self, 
                      scenario: str = 'base', 
                      use_provided_read_files: bool = True,
                      read_file: str = 'Balmorel_ReadData',
                      overwrite: bool = False):
        """Will load .inc files from the specific scenario

        Args:
            scenario (str, optional): The scenario that you . Defaults to 'base'.
            use_provided_read_files (bool, optional): Use provided Balmorel_ReadData.gms and Balmorelbb4_ReadData.inc. Defaults to True.
            read_file (str, optional): The name of the read file to be executed. Defaults to Balmorel_ReadData
            overwrite (bool, optional): Will overwrite an existing %scenario%_input_data.gdx file from a previous .load_incfiles execution 
            
        Raises:
            KeyError: _description_
        """
        
        if scenario not in self.scenarios:
            raise KeyError('%s scenario wasnt found.\nRun this Balmorel(...) class again if you just created the %s scenario.'%(scenario, scenario))
        
        
        # Path to the GAMS system directory
        model_folder = self.path / scenario / 'model'

        # Path to input .gdx file of scenario
        input_gdx = model_folder / ('%s_input_data.gdx'%scenario)
        
        if input_gdx.exists() and not(overwrite):
            ws = gams.GamsWorkspace(system_directory=self._gams_system_directory)
            db = ws.add_database_from_gdx(str((model_folder / ('%s_input_data.gdx'%scenario)).absolute()))
            self.input_data[scenario] = db
            print(
                '-'*20,
                '\nLoaded existing input data - pass "overwrite = True" if you changed data since last incfile loading\n',
                'Loaded .gdx file:\n', 
                input_gdx,
                '\n',
                '-'*20,
            )
            
        else:
            # Are you using the provided 'ReadData'-Balmorel files or a custom one?
            use_provided_read_files = True
            if use_provided_read_files:
                pkgdir = sys.modules['pybalmorel'].__path__[0]
                print('-'*150, '\npkgdir\n', pkgdir, '\n', '-'*150)
                # Copy Balmorel_ReadData and Balmorelbb4_ReadData 
                # into the model folder if there isn't one already
                for file in ['Balmorel_ReadData.gms', 'Balmorelbb4_ReadData.inc']:
                    if not (model_folder / file).exists():
                        shutil.copyfile(Path(pkgdir) / file, model_folder / file)
                        print(Path(model_folder) / file)

            # Initialize GAMS Workspace
            ws = gams.GamsWorkspace(working_directory=model_folder, 
                                    system_directory=self._gams_system_directory)

            # Set options
            opt = ws.add_options()
            opt.gdx = '%s_input_data.gdx'%scenario # Setting the output gdx name (note, could be overwritten by the cmd line options, which is intended)        
            
            # Load the GAMS model
            model_db = ws.add_job_from_file(str((model_folder / read_file).absolute()), job_name=scenario)

            # Run the GAMS file
            model_db.run(opt)

            # Store the database (will take some minutes)
            self.input_data[scenario] = model_db.get_out_db()

    def get_input(self, symbol: str, cols: list | None = None) -> pd.DataFrame:
        """Get a certain input from the loaded input file(s) into a pandas DataFrame

        Args:
            symbol (str): The desired input, e.g. DE_VAR_T
            cols (str, optional): Specify custom columns. Defaults to pre-defined formats or domain names.

        Returns:
            pd.DataFrame: The output DataFrame
        """
        # Placeholder
        df = pd.DataFrame()
        
        for SC in self.input_data.keys():
            # Get results from each scenario
            try :
                temp = symbol_to_df(self.input_data[SC], symbol, cols)
                temp['Scenario'] = SC 
                # Put scenario in first column
                temp = temp.loc[:, ['Scenario'] + list(temp.columns[:-1])]
                # Save
                df = pd.concat((df, temp), ignore_index=True)
            
            except ValueError :
                print(f'{SC} doesn\'t have any value in the table {symbol}')
            
        return df  

    def temporal_aggregation(self, 
                             scenario: str, 
                             seasons: int, 
                             terms: int, 
                             method: str = 'contiguous', 
                             representation: str = 'distribution_minmax',
                             symbols_to_aggregate: dict | str = 'auto',
                             incfile_symbol_relation: dict = {},
                             overwrite: bool = False):
        """
        Do temporal aggregation of scenario, using tsam.
        If symbols_to_aggregate is 'auto' (default setting), the 
        script will try to automatically find the timeseries data.

        NOTE:   If .inc files of the scenario you want to aggregate define both
        time-related symbols AND symbols without S or T sets, this script will
        produce new .inc files that does NOT include the non-S/T dependant
        symbols. Hence, remember to copy paste those parts into the newly
        generated .inc files afterwards.
                

        Args:
           scenario (str): scenario to aggregate.
           seasons (int): amount of seasons to aggregate to 
           terms (int): amount of terms to aggregate to
           method (str, optional): Aggregation method. Defaults to 'contiguous', options are: averaging, kmeans, kmedoids, kmaxoids, hierarchical, contiguous (default) and random choice
           representation (str, optional): How to represent cluster centers. Options are: mean, medoid, maxoid, distribution, distribution_minmax (default), minmax_mean
           symbols_to_aggregate (dict | str): 'auto' for automatically finding 
            the data. Otherwise, a dictionary with keys 'SSS,TTT', 'SSS'
            and 'TTT' that each point to a list of symbols to aggregate for manual 
            choice of aggregation.
           incfile_symbol_relation (dict): if manually defining symbols to 
            aggregate, this dict must be passed too. Keys should be symbols, 
            and the values should be a list of .inc file paths, where the first one 
            will be the new .inc file with aggregated data, and the others will be 
            empty. E.g.: {'DE_VAR_T' : ['base/data/DE_VAR_T.inc',
                                        'base/data/INDIVUSERS_DE_VAR_T.inc', 
                                        '/base/data/TRANSPORT_DE_VAR_T.inc']}
        """

        # Create temporal aggregation class
        self.ts = self.TimeAgg(parent=self)

        # Collect and standardise time series input 
        self.ts.collect_and_standardise(scenario, symbols_to_aggregate, 
                                        incfile_symbol_relation, overwrite)

        # Cluster collected time series input
        self.ts.cluster(seasons, terms, method, representation)

        # Prepare and save incfiles
        self.ts.save_incfiles(scenario, excluded_incfiles=['HYRSDATASET.inc'])


    class TimeAgg:
        def __init__(self, parent):
            self.parent = parent
            self.symbols = {}
            self.incfiles = {}
            self.incfiles_to_save = {}
            self.data = pd.DataFrame(index=SSS_TTT_index)

        def collect_and_standardise(self, 
                                    scenario: str, 
                                    symbols_to_aggregate: dict | str = 'auto',
                                    incfile_symbol_relation: dict = {},
                                    overwrite: bool = False):
            
            # Collect .inc files
            self.parent.load_incfiles(scenario, overwrite=overwrite)

            # Check standardised time series data have already been collected, return if so and overwrite = False
            std_data_file = self.parent.path / scenario / 'std_ts_data.pkl'
            symbols_file = self.parent.path / scenario / 'symbols_to_agg.pkl'
            incfiles_file = self.parent.path / scenario / 'incfile_symbol_relation.pkl'
            if not overwrite and std_data_file.exists() and symbols_file.exists() and incfiles_file.exists():
                with open(std_data_file, 'rb') as f:
                    self.data = pkl.load(f)

                # Load old standardised input
                if symbols_to_aggregate == 'auto':
                    with open(symbols_file, 'rb') as f:
                        self.symbols = pkl.load(f)
                # Manually defined, following a previous standardisation
                elif type(symbols_to_aggregate) is not str:
                    self.symbols = symbols_to_aggregate

                # Load old relations
                if incfile_symbol_relation == {}:
                    with open(incfiles_file, 'rb') as f:
                        self.incfiles = pkl.load(f)
                # Manually defined, following a previous standardisation
                else:
                    self.incfiles = incfile_symbol_relation

                return

            # Collect input - start checking if input is correct
            if type(symbols_to_aggregate) is dict and incfile_symbol_relation == {}:
                # Make sure input is correct for manual entry
                raise ValueError("incfile_symbol_relation must be defined if writing symbols to aggregate manually!")

            elif type(symbols_to_aggregate) is dict:
                # Check that incfiles were defined for each symbol
                for timeseries_type in ['SSS,TTT', 'SSS', 'TTT']:
                    for symbol in symbols_to_aggregate[timeseries_type]:
                        try: 
                            incfile_symbol_relation[symbol]
                        except KeyError:
                            raise ValueError(f"Missing incfile for {symbol}!")

                # Correct input
                self.symbols = symbols_to_aggregate
                self.incfiles = incfile_symbol_relation

            elif type(symbols_to_aggregate) is str and symbols_to_aggregate.lower() == 'auto':
                # Automatically find time series symbols
                self.find_timeseries_input(scenario)
                
            elif type(symbols_to_aggregate) is str and symbols_to_aggregate.lower() != 'auto':
                # Check that some other string was not passed
                raise ValueError(r"Incorrect choice - did you mean symbols_to_aggregate = 'auto' ?")

            else:
                raise ValueError("Incorrect input!")

            # Collect and standardise input 
            for symbol_type in ['SSS,TTT', 'SSS', 'TTT']:
                self.symbols_to_ignore = []
                for symbol in self.symbols[symbol_type]:
                    self.standardise_timeseries(scenario, symbol)

                self.symbols[symbol_type] = [symbol for symbol in self.symbols[symbol_type] if symbol not in self.symbols_to_ignore]

            # Save standardised input to a .pkl file
            with open(std_data_file, 'wb') as f:
                pkl.dump(self.data, f)
            with open(symbols_file, 'wb') as f:
                pkl.dump(self.symbols, f)
            with open(incfiles_file, 'wb') as f:
                pkl.dump(self.incfiles, f)

        def find_timeseries_input(self, scenario: str,
                                excluded_symbols: list = ['WEIGHT_S', 'WEIGHT_T', 
                                                            'CHRONOHOUR', 'SSIZE',
                                                            'TWORKDAY', 'TWEEKEND',
                                                            'S', 'T', 'DE_VAR_T1',
                                                            'DE_VAR_T1_INDIVHEATING',
                                                            'DH_VAR_T1', 'DH_VAR_T_IND',
                                                            'DH_VAR_T_INDIVHEATING', 
                                                            'WND_VAR_T1', 'SOLE_VAR_T1',
                                                            'SOLH_VAR_T1', 'X3FX_VAR_T1'
                                                            ]):
            """
            Locates the timeseries inputs of Balmorel by loading symbols from .inc
            files, filtering symbols with SSS or TTT domains, and filtering raw
            data input from processed by matching symbol names to the text of ALL
            .inc files in base/data and scenario/data. excluded_symbols can be used
            to exclude temporal resolution input meta data, such as CHRONOHOUR, S
            or T, or temporary profiles such as DE_VAR_T1. 

            Args:
            scenario (str): scenario timeseries data input to find.
            excluded_symbols (list): symbols to exclude.

            Returns:
            str: description.
            """
            
            # Make sure scenario data has been loaded
            assert scenario in self.parent.input_data, (
                f"Input data not loaded for {scenario}!"
                f"Run 'Balmorel.load_incfiles({scenario})' before this command"
            )

            # Get a list of all symbols in this scenario
            db = self.parent.input_data[scenario]
            symbol_list = [symbol.name for symbol in db if symbol.name not in excluded_symbols]

            # Categorise symbols
            timeseries_symbols = {'SSS,TTT' : [], 'SSS' : [], 'TTT' : []}
            symbols_incfiles = {}
            for symbol in symbol_list:
                # Only look at symbols with domains (e.g.: not CCCRRRAAA)
                domains = [domain.name for domain in db[symbol].domains if domain != '*']            

                # Use ripgrep to search for symbol in .inc files 
                incfiles_containing_symbol = search_in_incfiles(symbol, self.parent.path / scenario / 'data')

                # Try again in base/data, if symbol wasnt found in scenario/data
                if len(incfiles_containing_symbol) == 0:
                    incfiles_containing_symbol = search_in_incfiles(symbol, self.parent.path / 'base/data')

                # Only collect if symbol exists in an .inc file
                if len(incfiles_containing_symbol) > 0:
                    if ('SSS' in domains or 'S' in domains) and ('TTT' in domains or 'T' in domains):
                        timeseries_symbols['SSS,TTT'].append(symbol)
                        symbols_incfiles[symbol] = incfiles_containing_symbol
                    elif ('SSS' in domains or 'S' in domains) and not ('TTT' in domains or 'T' in domains):
                        timeseries_symbols['SSS'].append(symbol)
                        symbols_incfiles[symbol] = incfiles_containing_symbol
                    elif ('TTT' in domains or 'T' in domains) and not ('SSS' in domains or 'S' in domains):
                        timeseries_symbols['TTT'].append(symbol)
                        symbols_incfiles[symbol] = incfiles_containing_symbol

            self.symbols = timeseries_symbols
            self.incfiles = symbols_incfiles
            
        def standardise_timeseries(self, scenario: str, symbol: str):
            """
            Collect and standardise timeseries

            NOTE: This depends on the balmorel_symbol_columns in pybalmorel/formatting.py!
                  Otherwise, the following symbols will get KeyErrors in .pivot_table 
                  because of duplicate columns: 
                    XKRATE: Duplicate RRR's instead of IRRRE and IRRRI
            """

            # Get symbol 
            db = self.parent.input_data[scenario]
            df = symbol_to_df(db, symbol)

            # Make sure it is not empty, remove if so
            if df.shape == (0, 0):
                self.symbols_to_ignore.append(symbol)
                print(f'Ignoring {symbol} in time aggregation since it was empty')
                return

            # Separate time domains from other domains
            domains = [domain for domain in df.columns if domain not in ['SSS', 'S', 'TTT', 'T', 'Value']]            
            time_domains = [domain for domain in df.columns if domain in ['SSS', 'S', 'TTT', 'T']]            

            # Standardise
            df = (
                df
                .pivot_table(index=time_domains, 
                            columns=domains, 
                            values='Value',
                            fill_value=0)
            )

            # Check if symbol has only constant values
            without_constants = df.loc[:, df.nunique() > 1]
            
            if len(without_constants.columns) == 0:
                self.symbols_to_ignore.append(symbol)
                print(f'Ignoring {symbol} in time aggregation since all time series were constant')
                return

            # Flatten column to one string name
            if type(df.columns) is pd.MultiIndex:
                cols = [f"{symbol}|{'|'.join(col)}" for col in df.columns]
            else:
                cols = [f"{symbol}|{col}" for col in df.columns]
            df.columns = cols
            
            # Re-index to get full ST set for all (will duplicate S or T values to all T or S indices, respectively)
            if 'TTT' in time_domains and 'SSS' not in time_domains and 'S' not in time_domains:
                df = df.reindex(SSS_TTT_index, level='TTT').fillna(0)
            elif 'T' in time_domains and 'SSS' not in time_domains and 'S' not in time_domains:
                df.index.name = 'TTT'
                df = df.reindex(SSS_TTT_index, level='TTT').fillna(0)
            elif 'SSS' in time_domains and 'TTT' not in time_domains and 'T' not in time_domains:
                df = df.reindex(SSS_TTT_index, level='SSS').fillna(0)
            elif 'S' in time_domains and 'TTT' not in time_domains and 'T' not in time_domains:
                df.index.name = 'SSS'
                df = df.reindex(SSS_TTT_index, level='SSS').fillna(0)
            else:
                df.index.names = ['S', 'T']
                df = df.reindex(SSS_TTT_index).fillna(0)

            # Store time series data
            self.data = self.data.join(df, how="outer").fillna(0) 

        def cluster(
                self,
                seasons: int = 6,
                terms: int = 24,
                method: str = "contiguous",
                representation: str = "distribution_minmax"
        ):
            """Cluster collect input data

            Args:
                scenario (str): The scenario folder to aggregate.
                seasons (int): Amount of periods / seasons
                terms (int): Amount of hours / terms
                method (str, optional): Aggregation method. Defaults to 'contiguous', options are: averaging, kmeans, kmedoids, kmaxoids, hierarchical, contiguous (default) and random choice
                representation (str, optional): How to represent cluster centers. Options are: mean, medoid, maxoid, distribution, distribution_minmax (default), minmax_mean
            """

            # Using a Random Choice
            if method == "random":
                # # Make random time aggregation
                # N_timeslices = seasons * terms
                # N_hours = len(self.data)
                #
                # # Make choices
                # agg_steps = []
                # for i in range(N_timeslices):
                #     agg_steps.append(np.random.randint(N_hours))
                #
                # # Sort chronologically
                # agg_steps.sort()
                #
                # # Also save a small note with the chosen timesteps
                # with open(
                #     "Balmorel/%s/picked_times.txt"
                #     % (
                #         "W%dT%d_rand"
                #         % (seasons, terms)
                #     ),
                #     "w",
                # ) as f:
                #     f.write(pd.Series(self.data.iloc[agg_steps].index).to_string())
                raise NotImplementedError("To be developed. The outcommented code in this function can be used to index the collected data and create new .inc files")

            # Using tsam
            else:

                # Clip very small values (doesn't seem to be working?)
                df = self.data.clip(1e-5)

                # Aggregate collected data
                cluster_config = tsam.ClusterConfig(method, representation)
                aggregation = tsam.aggregate(
                    df,
                    n_clusters=seasons,
                    period_duration=terms,
                    temporal_resolution=1,
                    cluster=cluster_config,
                )

                # Make new Balmorel index
                data = aggregation.cluster_representatives
                data.index = pd.MultiIndex.from_product(
                    [[f'S{i:02.0f}' for i in range(1, seasons+1)],
                    [f'T{i:03.0f}' for i in range(1, terms+1)]],
                    names=['SSS', 'TTT']
                )

                # Store to self
                self.cluster_stats = aggregation
                self.agg_data = aggregation.cluster_representatives
                self.agg_resolution = {'S' : seasons, 'T' : terms}
                self.method = method
                self.representation = representation

        def prepare_clustered_data(self, scenario: str, symbol_type: str):

            # Prepare placeholders
            incfiles = self.incfiles_to_save
            symbols = self.symbols
            incfile_relations = self.incfiles
            db = self.parent.input_data[scenario]

            # Prepare new, aggregated scenario and document
            self.new_scenario_path.mkdir(parents=True, exist_ok=True)
            with open(self.new_scenario_path / '../temporal_aggregation.md', 'w') as f:
                f.write(f"Temporal aggregation made {datetime.now().strftime('%Y-%m-%d %T')}\n")
                f.write(f"Method: {self.method}\n")
                f.write(f"Representation: {self.representation}\n")
                f.write(f'Aggregated resolution: {self.agg_resolution["S"]} seasons and {self.agg_resolution["T"]} terms\n')

            # Loop through symbols
            for symbol in symbols[symbol_type]:
                # Collect metadata
                domains = db[symbol].domains_as_strings
                non_time_domains = [domain for domain in domains if domain not in ['SSS', 'TTT', 'Value']]
                explanatory_text = db[symbol].text

                # Collect aggregated data
                symbol_data_idx = self.agg_data.columns.str.contains(symbol)
                symbol_data = self.agg_data.loc[:, symbol_data_idx]

                # Un-standardise (prepare for Balmorel input)
                new_columns=symbol_data.columns.str.split('|', expand=True)
                symbol_data.columns = new_columns
                symbol_data.columns.names = ['symbol'] + non_time_domains

                # Take median of aggregated values if only T or S based symbol
                if symbol_type == 'TTT':
                    symbol_data = symbol_data.pivot_table(index='TTT', aggfunc='median')
                elif symbol_type == 'SSS':
                    symbol_data = symbol_data.pivot_table(index='SSS', aggfunc='median')

                # Get domain order correct
                for _ in range(len(non_time_domains)):
                    symbol_data = symbol_data.stack()
                symbol_data = symbol_data.reset_index()

                domain_len=len(domains)
                symbol_data = symbol_data.pivot_table(index=domains[:domain_len-1], 
                                                      columns=domains[domain_len-1], 
                                                      values=symbol)

                symbol_data.columns.name = ''
                symbol_data.index.name = ''
                symbol_data.index.names = ['']*symbol_data.index.nlevels
                if symbol_data.index.nlevels >= 2:
                    symbol_data.index = [' . '.join(ind) for ind in symbol_data.index]

                # Loop through related .inc files
                if type(incfile_relations[symbol]) is str:
                    iterable = [incfile_relations[symbol]]
                elif type(incfile_relations[symbol]) is list:
                    iterable = incfile_relations[symbol]
                else:
                    raise TypeError(f"Wrong format of .inc-file-symbol relation dictionary entry for {symbol}!")

                for incfile in iterable:
                    # Prepare a new .inc file if it doesn't already exist
                    if incfile not in incfiles.keys():
                        # Prepare filename, path, prefix and figure out if symbol == filename
                        filename, path, prefix, suffix, domains, filename_eq_symbol = prepare_incfile(incfile, symbol, domains, explanatory_text)
                        incfiles[incfile] = IncFile(
                            name=filename,
                            path=str(self.new_scenario_path),
                            body=symbol_data.to_string(),
                            prefix=prefix, 
                            suffix=suffix
                        )

                        # Store new attribute, relating to whether or not .inc filename equals symbol name 
                        incfiles[incfile].sn_eq_ifn = filename_eq_symbol
                
                    # Append to existing .inc file
                    else:
                        filename, path, prefix, suffix, domains, filename_eq_symbol = prepare_incfile(incfile, symbol, domains, explanatory_text)
                        incfiles[incfile].body += '\n'
                        incfiles[incfile].body += '\n;\n' + prefix
                        incfiles[incfile].body += symbol_data.to_string()

                # Make first related .inc file the one to save data to, if no .inc file had a name equal to symbol name
                incfiles_to_save = [incfiles[prepared_incfile].sn_eq_ifn for prepared_incfile in iterable]
                if sum(incfiles_to_save) == 0:
                    incfiles[iterable[0]].sn_eq_ifn = True
                # Make sure that only one .inc file will be saved with the data
                elif sum(incfiles_to_save) > 1:
                    raise ValueError(f"More than one .inc file will contain data for symbol {symbol}, but only one should!")


        def save_incfiles(self, scenario: str, excluded_incfiles: list = []):
            self.new_scenario_path = Path(self.parent.path / f'{scenario}_S{self.agg_resolution["S"]}T{self.agg_resolution["T"]}/data')
            # Save .inc files 
            for symbol_type in ['SSS,TTT', 'SSS', 'TTT']:
                self.prepare_clustered_data(scenario, symbol_type)

            # TODO: Fix the fact that you are randomly saving EV leave profiles
            # to one of the .inc files, but balopt chooses a specific one,
            # which then might be empty in aggregated files
            # TODO: Split DR_DATAINPUT.inc into a time-dependant one and the
            # static input. Then excluded_incfiles can be an empty list as well

            # Save aggregated files
            incfiles = self.incfiles_to_save
            for incfile in incfiles:
                # Don't save excluded .inc files
                if incfiles[incfile].name in excluded_incfiles:
                    continue

                # Only write data to one of the .inc files that symbols relate to
                if incfiles[incfile].sn_eq_ifn:
                    incfiles[incfile].save()
                # Write the rest as empty files (typically addon files)
                else:
                    # Empty file (addon files already included in previously written .inc file)
                    with open(incfiles[incfile].path + '/' + incfiles[incfile].name, 'w') as f:
                        f.write('')

            # Finally save S and T 
            bodies = {
                'S' : ", ".join([f"S{i:02.0f}" for i in range(1, self.agg_resolution['S'] + 1)]),
                'T' : ", ".join([f"T{i:03.0f}" for i in range(1, self.agg_resolution['T'] + 1)]),
            }
            for incfile in ['S', 'T']:
                f = IncFile(
                    name=incfile,
                    path=str(self.new_scenario_path),
                    prefix=f"SET {incfile}({incfile*3}) '{self.parent.input_data[scenario][incfile].text}'\n/\n",
                    body=bodies[incfile],
                    suffix='\n/;'
                ).save()


@dataclass
class TechData:
    files: dict = field(default_factory=lambda: {
        "el_and_dh" :   "technology_data_for_el_and_dh.xlsx",
        "indi_heat" :   "technology_data_heating_installations.xlsx",
        "ren_fuels" :   "data_sheets_for_renewable_fuels.xlsx",
        "storage"   :   "technology_datasheet_for_energy_storage.xlsx",
        "ccts"      :   "technology_data_for_carbon_capture_transport_storage.xlsx",
        "indu_heat" :   "technology_data_for_industrial_process_heat_0.xlsx",
        "trans"     :   "energy_transport_datasheet.xlsx"                              
    })
    path: str = r"C:\Users\mathi\gitRepos\balmorel-preprocessing\RawDataProcessing\Data\Technology Data" # Change this later
    
    def load(self, file: str):
        f = pd.read_excel(os.path.join(self.path, self.files[file]),
                          sheet_name='alldata_flat')
        return f

    # Function to download a file from a URL and save it to a specified folder
    def download_file(self, url, save_folder, filename=None):
        """
        Downloads a file from a given URL and saves it to a specified folder.
        Args:
            url (str): The URL of the file to download.
            save_folder (str): The folder where the file should be saved.
            filename (str, optional): The name to save the file as. If not provided, the filename is extracted from the URL.
        Returns:
            str: The full path to the saved file.
        Raises:
            requests.exceptions.RequestException: If the download fails.
        Notes:
            - The function ensures that the save folder exists.
            - The file is downloaded in chunks to handle large files efficiently.
        chunk_size:
            The size of each chunk of data to be written to the file. In this case, it is set to 8192 bytes (8 KB).
        """
        # Make sure the save folder exists
        os.makedirs(save_folder, exist_ok=True)

        # If no filename is provided, extract the filename from the URL
        if filename is None:
            filename = url.split("/")[-1]
        
        # Full path to save the file
        save_path = os.path.join(save_folder, filename)

        # Download the file with streaming to handle large files
        with requests.get(url, stream=True) as response:
            response.raise_for_status()  # Raise an error if the download fails
            with open(save_path, 'wb') as file:
                for chunk in response.iter_content(chunk_size=8192):
                    file.write(chunk)
        
        print(f"Downloaded file to: {save_path}")
        return save_path

    def download_catalogue(self, 
                           file: str,
                           save_to_folder: str = '.',
                           domain: str = "https://ens.dk/sites/ens.dk/files/Analyser/"):
        """Downloads technology catalogue from DEA website

        Args:
            file (str): _description_
            save_to_folder (str, optional): _description_. Defaults to '.'.
            domain (_type_, optional): _description_. Defaults to "https://ens.dk/sites/ens.dk/files/Analyser/".
        """
        try:
            # You probably used the short name used in this class
            filename = self.files[file]
        except KeyError:
            # ..in case you wrote the full name of the file
            filename = file
            
        if not(filename in os.listdir(save_to_folder)):
            self.download_file(urljoin(domain, filename), save_to_folder)
        else:
            print('\nThe file:\t\t%s\nalready exists in:\t%s'%(self.files[file], save_to_folder))

    def download_all_catalogues(self,
                                save_to_folder: str = '.'):
        for file in self.files.keys():
            self.download_catalogue(file, save_to_folder)
        

class GUI:
    def __init__(self) -> None:
        pass
    
    # Interactive bar chart plotting
    def bar_chart(self, MainResults_instance):
        """Interactive GUI to plot bar charts from MainResults

        Args:
            MainResults_instance (class): Loaded MainResults

        Returns:
            None: An interactive GUI is opened to plot bar charts 
        """                
        return  interactive_bar_chart(MainResults_instance)  
    
    def geofilemaker():
        """Opens a GUI to interactively generate necessary .inc files for Balmorel geography

        Returns:
            None: An interactive GUI to generate geographic .inc files
        """
        return interactive_geofilemaker()
