
# This file is part of pylabcontrol, software for laboratory equipment control for scientific experiments.
# Copyright (C) <2016>  Arthur Safira, Jan Gieseler, Aaron Kabcenell
#
#
# pylabcontrol is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# pylabcontrol is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with pylabcontrol.  If not, see <http://www.gnu.org/licenses/>.

import datetime
from copy import deepcopy

from pylabcontrol.core.instruments import Instrument
from pylabcontrol.core.parameter import Parameter
from pylabcontrol.core.read_write_functions import save_b26_file, load_b26_file
from pylabcontrol.core.helper_functions import module_name_from_path

from collections import deque
import os
import pandas as pd
import glob
import inspect
import warnings
import platform
from PyQt5.QtCore import pyqtSignal, QObject, pyqtSlot


import numpy as np
from builtins import len as builtin_len
from matplotlib.backends.backend_pdf import FigureCanvasPdf as FigureCanvas # use this to avoid error that plotting should only be done on main thread
from matplotlib.figure import Figure
from importlib import import_module


# cPickle module implements the same algorithm as pickle, in C instead of Python.
# It is many times faster than the Python implementation, but does not allow the user to subclass from Pickle.
import pickle

class Script(QObject):
    #This is the signal that will be emitted during the processing.
    #By including int as an argument, it lets the signal know to expect
    #an integer argument when emitting.
    updateProgress = pyqtSignal(int) # emits a progress update in percent
    started = pyqtSignal()  # signals the begin of the script
    finished = pyqtSignal() # signals the end of the script
    # current_subscript = pyqtSignal(str) # indicates the current subscript that is being excecuted

    _DEFAULT_SETTINGS = [
        Parameter('path', '', str, 'path to folder where data is saved'),
        Parameter('tag', 'default_tag'),
        Parameter('save', False, bool,'check to automatically save data'),
    ]

    RAW_DATA_DIR = 'raw_data' # dir name for rawdata
    SUBSCRIPT_DATA_DIR = 'data_subscripts' # dir name for subscript data

    def __init__(self, name=None, settings=None, instruments=None, scripts=None, log_function=None, data_path=None):
        """
        executes scripts and stores script parameters and settings
        Args:
            name (optional):  name of script, if not provided take name of function
            settings (optional): a Parameter object that contains all the information needed in the script
            instruments (optional): instruments used in the script
            scripts (optional):  sub_scripts used in the script
            log_function(optional): function reference that takes a string
        """
        QObject.__init__(self)

        self._script_class = self.__class__.__name__

        if name is None:
            name = self.__class__.__name__
        self.name = name


        self._instruments = {}
        if instruments is None:
            instruments = {}
        else:
            assert isinstance(instruments, dict)
            assert set(self._INSTRUMENTS.keys()) <= set(instruments.keys())

        self.data_path = data_path

        self.instruments = {key: instruments[key] for key in list(self._INSTRUMENTS.keys())}

        self._scripts = {}
        if scripts is None:
            scripts = {}
        self.scripts = scripts

        # set end time to be before start time if script hasn't been excecuted this tells us
        self.start_time = datetime.datetime.now()
        self.end_time = self.start_time - datetime.timedelta(seconds=1)

        self._settings = deepcopy(Parameter(self._DEFAULT_SETTINGS + Script._DEFAULT_SETTINGS))
        self._settings.update({'tag':self.name.lower()})
        if settings is not None:
            self.update(settings)
        self._abort = False
        self.is_running = False

        # data hold the data generated by the script,
        # this should either be a dictionary or a deque of dictionaries
        self.data = {}

        # a log for status outputs
        self.log_data = deque()
        # this can be overwritten
        self.log_function = log_function

        # default value is 'none', overwrite this in script if it has plotting capabilities
        self._plot_refresh = True

        self.progress = None


        self._current_subscript_stage = {
            'current_subscript': None,
            'subscript_exec_count':{},
            'subscript_exec_duration':{}
        }
    @property
    def data_path(self):
        return self._data_path

    @data_path.setter
    def data_path(self, path):
        # check is path is a valid path string
        # if path is not None and path is not '':
        #     if not os.path.isdir(path):
        #         print('{:s} created'.format(path))
        #         os.makedirs(path)

        self._data_path = path

    @pyqtSlot(bool)
    def _set_current_subscript(self, active):
        """
        sets the current subscript and keeps a counter of how ofter a particular subscript has been executed
        this information is usefull when implementing a status update or plotting functions that depend on which subscript is being executed

        keeps track of the following dictionary:
        self._current_subscript_stage = {
            'current_subscript' : reference to the current subscrit
            'subscript_exec_count' : dictionary where key is the subscript name and value how often is has been executed
            'subscript_exec_duration' : dictionary where key is the subscript name and value the average duration of executing the subscript
        }

        Args:
            active: True if the current subscript is just started, False if it just finished
        """

        current_subscript = self.sender()


        if active:
            for subscript_name in list(self._current_subscript_stage['subscript_exec_count'].keys()):
                if subscript_name == current_subscript.name:
                    self._current_subscript_stage['subscript_exec_count'][subscript_name] += 1
            self._current_subscript_stage['current_subscript'] = current_subscript
        else:
            self._current_subscript_stage['current_subscript'] = current_subscript
            for subscript_name in list(self._current_subscript_stage['subscript_exec_count'].keys()):
                # calculate the average duration to execute the subscript
                if subscript_name == current_subscript.name:
                    duration = current_subscript.end_time - current_subscript.start_time
                    if subscript_name  in self._current_subscript_stage['subscript_exec_duration']:
                        duration_old = self._current_subscript_stage['subscript_exec_duration'][subscript_name]
                    else:
                        duration_old = datetime.timedelta(0)
                    exec_count = self._current_subscript_stage['subscript_exec_count'][subscript_name]

                    duration_new = (duration_old * (exec_count - 1) + duration)
                    self._current_subscript_stage['subscript_exec_duration'][subscript_name] = (duration_old * (
                    exec_count - 1) + duration) / exec_count

    def _function(self):
        """
        This is the actual function that will be executed. It uses only information that is provided in the settings property
        will be overwritten in the __init__
        """
        # some generic function
        raise NotImplementedError

    #todo: 170202JG (search for this to find related todos)
    # make this a slot
    # @pyqtSlot(bool)
    def log(self, string):
        """
        appends input string to log file and sends it to log function (self.log_function)
        Returns:

        """

        self.log_data.append(string)
        if self.log_function is None:
            print(string)
        else:
            self.log_function(string)

    # @property
    # def _DEFAULT_SETTINGS(self):
    #     """
    #     returns the default parameter_list of the script this function should be over written in any subclass
    #     """
    #     raise NotImplementedError("Subclass did not implement _DEFAULT_SETTINGS")

    @property
    def _INSTRUMENTS(self):
        """

        Returns: a dictionary of the instruments, where key is the instrument name and value is the instrument class
        if there is not instrument it should return an empty dict

        """
        raise NotImplementedError("Subclass did not implement _INSTRUMENTS")

    @property
    def _SCRIPTS(self):
        """

        Returns: a dictionary of the instruments, where the key is the instrument name and value is the instrument class
        if there is not instrument it should return an empty dict

        """
        raise NotImplementedError("Subclass did not implement _SCRIPTS")

    def __str__(self):
        """
        :return: a description of the script in form of a string
        """

        output_string = '{:s} (class type: {:s})\n'.format(self.name, self.__class__.__name__)

        output_string += 'settings:\n'
        for key, value in self.settings.items():
            output_string += "{:s} : {:s}\n".format(key, str(value))
        return output_string

    @property
    def name(self):
        """
        script name
        """
        return self._name

    @name.setter
    def name(self, value):
        if isinstance(value, str):
            value = str(value)

        assert isinstance(value, str), str(value) + ' is not a string'
        self._name = value

    @property
    def instrumets(self):
        """
        :return: instruments that the script uses as a dictionary
        """
        return self._instruments
    @instrumets.setter
    def instruments(self, instrument_dict):
        assert isinstance(instrument_dict, dict)
        # checks if all the keys in _INSTRUMENTS are contained in instrument_dict
        assert set(self._INSTRUMENTS.keys()) <= set(instrument_dict.keys()), "{:s}: needs instruments {:s} but received {:s}".format(self.name, str( list(self._INSTRUMENTS.keys())), str(list(instrument_dict.keys())))
        for key, value in self._INSTRUMENTS.items():
            self._instruments.update({key: instrument_dict[key]})


    @property
    def scripts(self):
        """
        :return: sub_scripts that the script uses as a dictionary
        """
        return self._scripts

    @scripts.setter
    def scripts(self, script_dict):
        assert isinstance(script_dict, dict)
        assert set(script_dict.keys()) == set(self._SCRIPTS.keys()), "{:s}: set subscripts {:s}, received {:s}".format(self.name, str(list(script_dict.keys())), str( list(self._SCRIPTS.keys())))

        for key, value in self._SCRIPTS.items():
            assert isinstance(script_dict[key], self._SCRIPTS[key])
            self._scripts.update({key: script_dict[key]})

    @property
    def settings(self):
        '''
        :return: returns the settings of the script
        settings contain Parameters, Instruments and Scripts
        '''
        return self._settings

    def update(self, settings):
        '''
        updates the internal dictionary
        Args:
            settings: parameters to be set
        # mabe in the future:
        # Returns: boolean that is true if update successful

        '''
        if 'settings' in settings:
            self._settings.update(settings['settings'])
        else:
            self._settings.update(settings)

        if 'instruments' in settings:
            for instrument_name, instrument_setting in settings['instruments'].items():
                self.instruments[instrument_name]['settings'].update(instrument_setting['settings'])

        if 'scripts' in settings:
            for script_name, script_setting in settings['scripts'].items():
                self.scripts[script_name].update(script_setting)

    @property
    def end_time(self):
        """
        time when script execution ended
        :return:
        """
        return self._time_stop

    @end_time.setter
    def end_time(self, value):
        assert isinstance(value, datetime.datetime)
        self._time_stop = value

    @property
    def remaining_time(self):
        """
        estimates the time remaining until script is finished
        """
        elapsed_time = (datetime.datetime.now() - self.start_time).total_seconds()
        # safety to avoid devision by zero
        if self.progress == 0:
            self.progress = 1

        estimated_total_time = 100. / self.progress * elapsed_time

        return datetime.timedelta(seconds = max(estimated_total_time - elapsed_time, 0))

    @property
    def start_time(self):
        """
        time when script execution started
        """
        return self._time_start
    @start_time.setter
    def start_time(self, value):
        assert isinstance(value, datetime.datetime)
        self._time_start = value

    @property
    def excecution_time(self):
        """
        :return: script excecition time as time_delta object to get time in seconds use .total_seconds()
        """
        return self.end_time - self.start_time

    @pyqtSlot(int)
    def _receive_signal(self, progress):
        """
        this function takes care of signals emitted by the subscripts
        the default behaviour is that it just reemits the signal
        Args:
            progress: progress of subscript
        """
        # print(datetime.datetime.now().strftime("%B %d, %Y %H:%M:%S"), self.name,QtCore.QThread.currentThread(), self._current_subscript_stage['current_subscript'].name,
        #       'received signal. emitting....')

        self.progress = progress
        self.updateProgress.emit(progress)

    def run(self):
        """
        executes the script
        :return: boolean if execution of script finished succesfully
        """
        self.log_data.clear()
        self._plot_refresh = True  # flag that requests that plot axes are refreshed when self.plot is called next time
        self.is_running = True
        self.start_time = datetime.datetime.now()


        self._current_subscript_stage = {
            'current_subscript': None,
            'subscript_exec_count':{},
            'subscript_exec_duration':{}
        }

        # update the datapath of the subscripts, connect their progress signal to the receive slot
        for subscript in list(self.scripts.values()):
            subscript.data_path = os.path.join(self.filename(create_if_not_existing=False), self.SUBSCRIPT_DATA_DIR)
            subscript.updateProgress.connect(self._receive_signal)
            subscript.started.connect(lambda: self._set_current_subscript(True))
            subscript.finished.connect(lambda: self._set_current_subscript(False))
            self._current_subscript_stage['subscript_exec_count'].update({subscript.name:0})
            self._current_subscript_stage['subscript_exec_duration'].update({subscript.name: datetime.timedelta(0)})

                #todo: 170202JG (search for this to find related todos) need to test this:
                # do we need to connect the log functions of the subscript to the mother script?, e.g
                # subscript.log.connect(self.log)


        self.log('starting script {:s} at {:s} on {:s}'.format(self.name, self.start_time.strftime('%H:%M:%S'),self.start_time.strftime('%d/%m/%y')))
        self._abort = False

        #saves standard to disk
        if self.settings['save']:
            self.save_b26()

        self.started.emit()

        self._function()
        self.end_time  = datetime.datetime.now()
        self.log('script {:s} finished at {:s} on {:s}'.format(self.name, self.end_time.strftime('%H:%M:%S'),self.end_time.strftime('%d/%m/%y')))

        #saves standard to disk
        if self.settings['save']:
            self.save_data()
            self.save_log()
            self.save_image_to_disk()

        success = not self._abort

        # disconnect subscripts
        for subscript in list(self.scripts.values()):
            subscript.started.disconnect()
            subscript.updateProgress.disconnect()
            subscript.finished.disconnect()
        self.is_running = False
        self.finished.emit()



    def stop(self):
        """
        stops itself and all the subscript
        """
        for subscript in list(self.scripts.values()):
            subscript.stop()
        print(('--- stopping: ', self.name))
        self._abort = True

    def is_valid(self):
        """
        function to validate of the script parameters are valid:
         - check if the filename is too long (pandas can't write files if the total filepath is longer than 259 characters)

        :return: boolean
        """
        # validate = True
        # # check if filename is longer than 220, this leaves a buffer of 39 for dynamically created extentions
        # if len(self.filename()) > 220:
        #     validate = False
        #     self.log('Validation failed. Detected long filename in ', self.name)
        #
        # for s in self.scripts:
        #     if s.validate == False:
        #         validate = False
        #
        # return validate
        pass





    def filename(self, appendix=None, create_if_not_existing=False):
        """
        creates a filename based
        Args:
            appendix: appendix for file

        Returns: filename

        """

        # if provided path is a relative path and self.data_path exists, build path
        if os.path.isabs(self.settings['path']) == False and self.data_path is not None:
            path = os.path.join(self.data_path, self.settings['path'])
        else:
            path = self.settings['path']

        tag = self.settings['tag']#.replace('.','-')

        filename = os.path.join(path, "{:s}_{:s}".format(self.start_time.strftime('%y%m%d-%H_%M_%S'),tag))

        if os.path.exists(filename) == False and create_if_not_existing:
            os.makedirs(filename)

        if appendix is not None:
            filename = os.path.join(filename,  "{:s}_{:s}{:s}".format(self.start_time.strftime('%y%m%d-%H_%M_%S'),tag,appendix))

        # windows can't deal with long filenames so we have to use the prefix '\\\\?\\'
        # if len(filename.split('\\\\?\\')) == 1:
        #     filename = '\\\\?\\' + filename

        return filename
    def to_dict(self):
        """

        Returns: itself as a dictionary

        """

        from pylabcontrol.core.script_iterator import ScriptIterator


        if 'script_iterator' in self.__module__.split('.'):
            # script iterator module is of the form
            # 'pylabcontrol.core.script_iterator.b26_toolkit.dynamic_script_iterator0'
            # and the class name if of the form package.dynamic_script_iterator0
            package = self.__class__.__name__.split('.')[0]
        else:
            # if it is not a script iterator the package is the highest level of the module
            package = self.__module__.split('.')[0]


        dictator = {self.name: {
            'class' : self.__class__.__name__,
            'filepath': inspect.getfile(self.__class__),
            'info': self.__doc__,
            'package': package
        }}

        # todo JG: mayby change path so that it points to b26toolkit and not to PyLabcotnrols
        # if isinstance(self, ScriptIterator):
        #     dictator['filepath'] = inspect.getfile(self.__class__),



        if self.scripts != {}:
            dictator[self.name].update({'scripts': {} })
            for subscript_name, subscript in self.scripts.items():
                dictator[self.name]['scripts'].update(subscript.to_dict() )

        if self.instruments != {}:
            # dictator[self.name].update({'instruments': self.instruments})
            # dictator[self.name].update({'instruments': {} })
            # for instrument_name, instrument in self.instruments.iteritems():
            #     dictator[self.name]['instruments'].update(instrument.to_dict())

            dictator[self.name].update({'instruments': {
                instrument_name: {'class': instrument['instance'].__class__.__name__, 'settings':instrument['settings']}
                for instrument_name, instrument in self.instruments.items()
            }})

        dictator[self.name]['settings'] = self.settings

        return dictator
    def save_data(self, filename = None, data_tag = None):
        """
        saves the script data to a file
        filename: target filename, if not provided, it is created from internal function
        data_tag: string, if provided save only the data that matches the tag, otherwise save all data
        Returns:

        """


        def len(x):
            """
            overwrite the buildin len function to cover cases that don't have a length, like int or float
            and to catch string as objects of length 0
            Args:
                x: quantity of which we want to find the length
            Returns: length of x

            """
            if isinstance(x, (int, float, str)) or x is None:
                result = 0
            else:
                result = builtin_len(x)
            return result

        if filename is None:
            filename = self.filename('.csv')

        filename = os.path.join(os.path.join(os.path.dirname(filename),self.RAW_DATA_DIR), os.path.basename(filename))

        # windows can't deal with long filenames so we have to use the prefix '\\\\?\\'
        # if len(filename.split('\\\\?\\')) == 1:
        #     filename = '\\\\?\\' + filename

        if not os.path.exists(os.path.dirname(filename)):
            os.makedirs(os.path.dirname(filename))

        # if deque object, take the last dataset, which is the most recent
        if isinstance(self.data, deque):
            data = self.data[-1]
        elif isinstance(self.data, dict):
            data = self.data
        else:
            raise TypeError("script data variable has an invalid datatype! Must be deque or dict.")


        if data_tag is None:
            if len(set([len(v) for v in list(data.values())])) == 1 and set(
                    [len(np.shape(list(data.values())[i])) for i in range(len(list(data.values())))]) == set([0, 1]):
                # if all entries of the dictionary are the same length and single column we can write the data into a single file

                if len(np.shape(list(data.values())[0]))==1:
                    df = pd.DataFrame(data)
                else:
                    df = pd.DataFrame.from_records([data])
                df.to_csv(filename, index=False)

            else:
                # otherwise, we write each entry into a separate file
                for key, value in data.items():
                    if len(value) == 0:
                        df = pd.DataFrame([value])
                    else:
                        if isinstance(value, dict) and isinstance(list(value.values())[0], (int, float)):
                            # if dictionary values are single numbers
                            df = pd.DataFrame.from_dict({k: [v] for k, v in value.items()})
                        elif isinstance(value, dict) and isinstance(list(value.values())[0], (list, np.ndarray)):
                            # if dictionary values are lists or arrays
                            df = pd.DataFrame.from_dict(value)
                        else:
                            # if not a dictionary
                            df = pd.DataFrame(value)

                    df.to_csv(filename.replace('.csv', '-{:s}.csv'.format(key)), index=False)

        else:
            # save only the data for which a key has been provided
            assert data_tag in list(data.keys())
            value = data[data_tag]
            if len(value) == 0:
                df = pd.DataFrame([value])
            else:
                df = pd.DataFrame(value)
            df.to_csv(filename, index=False)
    def save_log(self, filename = None):
        """
        save log to file
        Returns:

        """

        if filename is None:
            filename = self.filename('-info.txt')
        # windows can't deal with long filenames so we have to use the prefix '\\\\?\\'
        # if len(filename.split('\\\\?\\')) == 1:
        #     filename = '\\\\?\\' + filename
        with open(filename, 'w') as outfile:
            for item in self.log_data:
                outfile.write("%s\n" % item)
    def save_b26(self, filename=None):
        """
        saves the script settings to a file: filename is filename is not provided, it is created from internal function
        """
        if filename is None:
            filename = self.filename('.b26')

        # if platform.system() == 'Windows':
        #     # windows can't deal with long filenames so we have to use the prefix '\\\\?\\'
        #     if len(filename.split('\\\\?\\')) == 1:
        #         filename = '\\\\?\\' + filename
        save_b26_file(filename, scripts=self.to_dict(), overwrite=True)
    def save_image_to_disk(self, filename_1 = None, filename_2 = None):
        """
        creates an image using the scripts plot function and writes it to the disk
        for single plots (plot_type: 'main', 'aux')
            - if no filname provided take default name
        for double plots (plot_type: 'main', 'aux')
            - if no filnames provided take default name
            - if only one filname provided save only the plot for which name is provided
        Args:
            filename_1: filname for figure 1
            filename_2: filname for figure 1

        Returns: None

        """

        def axes_empty(ax):
            """
            takes an axes object and checks if it is empty
            the axes object is considered empty it doesn't contain any of the following:
                - lines
                - images
                - patches
            Returns:

            """

            is_empty = True

            if ax is not None and len(ax)>0:
                for a in ax:
                    if len(a.lines)+len(a.images)+len(a.patches) != 0:
                        is_empty = False


            return is_empty

        # create and save images
        if (filename_1 is None):
            filename_1 = self.filename('-plt1.png')

        if (filename_2 is None):
            filename_2 = self.filename('-plt2.png')


        # windows can't deal with long filenames so we have to use the prefix '\\\\?\\'
        # if len(filename_1.split('\\\\?\\')) == 1:
        #     filename_1 = '\\\\?\\' + filename_1
        # if len(filename_2.split('\\\\?\\')) == 1:
        #     filename_2 = '\\\\?\\' + filename_2

        if os.path.exists(os.path.dirname(filename_1)) is False:
            os.makedirs(os.path.dirname(filename_1))
        if os.path.exists(os.path.dirname(filename_2)) is False:
            os.makedirs(os.path.dirname(filename_2))


        fig_1 = Figure()
        canvas_1 = FigureCanvas(fig_1)

        fig_2 = Figure()
        canvas_2 = FigureCanvas(fig_2)

        self.force_update()

        self.plot([fig_1, fig_2])

        if filename_1 is not None and not axes_empty(fig_1.axes):
            fig_1.savefig(filename_1)
        if filename_2 is not None and not axes_empty(fig_2.axes):
            fig_2.savefig(filename_2)

    def save(self, filename):
        """
        saves the instance of the script to a file using pickle
        Args:
            filename: target filename

        """

        if filename is None:
            filename = self.filename('.b26s')
        # if len(filename.split('\\\\?\\')) == 1:
        #     filename = '\\\\?\\' + filename
        with open(filename, 'w') as outfile:
            outfile.write(pickle.dumps(self.__dict__))

    @staticmethod
    def load(filename, instruments = None):
        """
        loads an script instance using pickle
        Args:
            filename: source filename
            instruments:
                optional - only needed if script requires instruments
                dictionary of form

                instruments = {
                name_of_instrument_1 : instance_of_instrument_1,
                name_of_instrument_2 : instance_of_instrument_2,
                ...
                }
        Returns:
            script_instance
            updated_instruments
        """
        with open(filename, 'r') as infile:
            dataPickle = infile.read()

        script_as_dict = pickle.loads(dataPickle)
        script_class = script_as_dict['_script_class']

        script_instance, _, updated_instruments = Script.load_and_append({'script': script_class}, instruments = instruments)
        script_instance = script_instance['script']

        # save references to instruments
        instruments = script_instance._instruments

        # update the script instance
        script_instance.__dict__ = script_as_dict

        # update references to instruments
        script_instance._instruments = instruments

        return script_instance, updated_instruments

    @staticmethod
    def load_data(path):
        """
        loads the data that has been save with Script.save.
        Args:
            path: path to folder saved by Script.save or raw_data folder within
        Returns:
            a dictionary with the data of form
            data = {param_1_name: param_1_data, ...}
        """


        # check that path exists
        if not os.path.exists(path):
            print(path)
            raise AttributeError('Path given does not exist!')

        # windows can't deal with long filenames (>260 chars) so we have to use the prefix '\\\\?\\'
        # if len(path.split('\\\\?\\')) == 1:
        #     path = '\\\\?\\' + os.path.abspath(path)


        # if raw_data folder exists, get a list of directories from within it; otherwise, get names of all .csv files in
        # current directory
        data = {}
        # if self.RAW_DATA_DIR in os.listdir(path): #8/26/16 AK: self not defined in static context
        #     data_files = os.listdir(os.path.join(path, self.RAW_DATA_DIR + '/'))
        #     path = os.path.join(path, self.RAW_DATA_DIR + '/')
        #
        # else:
        if 'raw_data' in os.listdir(path):  #temporarily hardcoded
            data_files = os.listdir(os.path.join(path, 'raw_data' + '/'))
            path = os.path.join(path, 'raw_data' + '/')

        else:
            data_files = glob.glob(os.path.join(path, '*.csv'))

        # If no data files were found, raise error
        if not data_files:
            raise AttributeError('Could not find data files in {:s}'.format(path))

        # import data from each csv
        for data_file in data_files:
            # get data name, read the data from the csv, and save it to dictionary
            data_name = data_file.split('-')[-1][0:-4] # JG: why do we strip of the date?
            imported_data_df = pd.read_csv(os.path.join(path, data_file))

            # check if there are real headers, if the headers are digits than we ignore them because then they are just indecies
            # real headers are strings (however, the digits are also of type str! that why we use the isdigit method)
            column_headers = list(imported_data_df.columns.values)
            if sum([int(x.isdigit()) for x in column_headers]) != len(column_headers):
                data[data_name] = {h: imported_data_df[h].as_matrix() for h in column_headers}
            else:
                # note, np.squeeze removes extraneous length-1 dimensions from the returned 'matrix' from the dataframe
                data[data_name] = np.squeeze(imported_data_df.as_matrix())

        return data

    @staticmethod
    def load_settings(path, setttings_only = True):
        """
        loads the settings that has been save with Script.save_b26.
        Args:
            path: path to folder saved by Script.save_b26
            setttings_only: if true returns only the settings if the .b26 file contains only a single script
        Returns:
            a dictionary with the settings
        """


        # check that path exists
        if not os.path.exists(path):
            print(path)
            raise AttributeError('Path given does not exist!')

        tag = '_'.join(os.path.basename(os.path.dirname(os.path.abspath(path) + '/')).split('_')[3:])

        search_str = os.path.abspath(path)+'/*'+tag +'.b26'
        fname = glob.glob(search_str)
        if len(fname)>1:
            print(('warning more than one .b26 file found, loading ', fname[0]))
        elif len(fname) == 0:
            print(('no .b26 file found in folder {:s},  check path !'.format(search_str)))
            return
        fname = fname[0]
        settings = load_b26_file(fname)['scripts']

        if len(list(settings.keys())) == 1 and setttings_only:
            settings = settings[list(settings.keys())[0]]['settings']

        return settings

    @staticmethod
    def load_and_append(script_dict, scripts=None, instruments=None, log_function=None, data_path=None,
                        raise_errors=True, package='pylabcontrol', verbose=False):
        """
        load script from script_dict and append to scripts, if additional instruments are required create them and add them to instruments

        Args:
            script_dict: dictionary of form

                script_dict = {
                name_of_script_1 :
                    {"settings" : settings_dictionary, "class" : name_of_class}
                name_of_instrument_2 :
                    {"settings" : settings_dictionary, "class" : name_of_class}
                ...
                }

            or

                script_dict = {
                name_of_script_1 : name_of_class,
                name_of_script_2 : name_of_class
                ...
                }

            where name_of_class is either a class or the name of a class

            scripts: dictionary of form

                scripts = {
                name_of_script_1 : instance_of_script_1,
                name_of_script_2 : instance_of_script_2,
                ...
                }

            instruments: dictionary of form

                instruments = {
                name_of_instrument_1 : instance_of_instrument_1,
                name_of_instrument_2 : instance_of_instrument_2,
                ...
                }
            log_function: function that takes a string

            data_path: absolute path where data is saved, in case the path in the script is definded as a relative path

            raise_errors: if True errors are raised
        Returns:
                dictionary of form
                script_dict = { name_of_script_1 : script_1_instance, name_of_script_2 : script_2_instance, ...}
                load_failed = {name_of_script_1: exception_1, name_of_script_2: exception_2, ....}
                updated_instruments = {name_of_instrument_1 : instance_of_instrument_1, ..}

        """
        if scripts is None:
            scripts = {}
        if instruments is None:
            instruments = {}

        load_failed = {}
        updated_scripts = {}
        updated_scripts.update(scripts)
        updated_instruments = {}
        updated_instruments.update(instruments)

        if verbose:
            print(('script_dict', script_dict))

        def get_instruments(class_of_script, script_instruments, instruments):
            """

            creates the dictionary with the instruments needed for the script and update the instrument dictionary if new instruments are required

            Args:
                class_of_script: the class of the script
                instruments: the instruments that have been loaded already

            Returns: dictionary with the instruments that the script needs and the updated instruments dictionary

            """

            default_instruments = getattr(class_of_script, '_INSTRUMENTS')
            instrument_dict = {}
            instruments_updated = {}
            instruments_updated.update(instruments)
            # check if instruments needed by script already exist, if not create an instance
            for instrument_name, instrument_class in default_instruments.items():
                # check if instruments needed by script already exist
                instrument = [instance for name, instance in instruments_updated.items() if
                              isinstance(instance, instrument_class) and name == instrument_name]

                if len(instrument) == 0:
                    # create new instance of instrument
                    instruments_updated, __ = Instrument.load_and_append({instrument_name: instrument_class}, instruments_updated, raise_errors)

                if script_instruments is not None and instrument_name in script_instruments:
                    instrument_settings_dict = script_instruments[instrument_name]['settings']
                else:
                    instrument_settings_dict = instruments_updated[instrument_name].settings

                instrument_instance = instruments_updated[instrument_name]

                # make a deepcopy of _DEFAULT_SETTINGS to get a parameter object
                instrument_settings = deepcopy(instrument_instance._DEFAULT_SETTINGS)

                #now update parameter object with new values
                instrument_settings.update(instrument_settings_dict)

                instrument_dict.update({instrument_name: {"instance":instrument_instance, "settings":instrument_settings}})


            return instrument_dict, instruments_updated

        def get_sub_scripts(class_of_script, instruments, sub_scripts_dict, log_function = None):
            """

            creates the dictionary with the sub scripts needed by the script and updates the instrument dictionary if new instruments are required

            Args:
                class_of_script: the class of the script
                instruments: the instruments that have been loaded already
                sub_scripts_dict: settings of script in dictionary form

            Returns:dictionary with the sub scripts that the script needs

            """

            default_scripts = getattr(class_of_script, '_SCRIPTS')
            #
            # create instruments that script needs
            sub_scripts = {}
            sub_scripts, scripts_failed, instruments_updated = Script.load_and_append(default_scripts, sub_scripts,
                                                                                      instruments,
                                                                                      log_function=log_function,
                                                                                      raise_errors=raise_errors)
            try:
                if sub_scripts_dict is not None:
                    for k, v in sub_scripts_dict.items():
                        #update settings, updates instrument and settings
                        sub_scripts[k].update(v)
            except TypeError: #if actually an object, as with dynamic scripts
                pass

            if len(scripts_failed)>0:
                raise ImportError('script {:s}: failed to load subscripts'.format(class_of_script))
            return sub_scripts, instruments_updated

        for script_name, script_info in script_dict.items():

            # check if script already exists
            if script_name in list(scripts.keys()):
                print(('WARNING: script {:s} already exists. Did not load!'.format(script_name)))
                load_failed[script_name] = ValueError('script {:s} already exists. Did not load!'.format(script_name))
            else:
                module, script_class_name, script_settings, script_instruments, script_sub_scripts, script_doc, package = Script.get_script_information(script_info, package=package)
                # creates all dynamic scripts so they can be imported following the if statement
                # if script_class_name == 'ScriptIterator':
                if 'ScriptIterator' in script_class_name:
                    # creates all the dynamic classes in the script and the class of the script itself
                    # and updates the script info with these new classes
                    from pylabcontrol.core.script_iterator import ScriptIterator #CAUTION: imports ScriptIterator, which inherits from script. Local scope should avoid circular imports.

                    script_info, _ = ScriptIterator.create_dynamic_script_class(script_info)

                    # now get the info for the dynamically created class
                    module, script_class_name, script_settings, script_instruments, script_sub_scripts, script_doc, package = Script.get_script_information(script_info)
                if verbose:
                    print(('load_and_append.module', module))
                    print(('load_and_append.script_info', script_info))
                    print(('load_and_append.package', package))

                if module is None and inspect.isclass(script_info):
                    class_of_script = script_info
                else:
                    class_of_script = getattr(module, script_class_name)

                #  ========= create the instruments that are needed by the script =========
                try:
                    script_instruments, updated_instruments = get_instruments(class_of_script, script_instruments, updated_instruments)
                except Exception as err:
                    print(('loading script {:s} failed. Could not load instruments!'.format(script_name)))
                    load_failed[script_name] = err
                    if raise_errors:
                        raise err
                    continue
                #  ========= create the subscripts that are needed by the script =========
                try:
                    sub_scripts, updated_instruments = get_sub_scripts(class_of_script, updated_instruments, script_sub_scripts, log_function = log_function)
                except Exception as err:
                    print(('loading script {:s} failed. Could not load subscripts!'.format(script_name)))
                    load_failed[script_name] = err
                    if raise_errors:
                        raise err
                    continue

                #  ========= create the script if instruments and subscripts have been loaded successfully =========
                class_creation_string = ''
                if script_instruments:
                    class_creation_string += ', instruments = script_instruments'
                if sub_scripts:
                    class_creation_string += ', scripts = sub_scripts'
                if script_settings:
                    class_creation_string += ', settings = script_settings'
                if log_function:
                    class_creation_string += ', log_function = log_function'
                if data_path:
                    class_creation_string += ', data_path = data_path'
                class_creation_string = 'class_of_script(name=script_name{:s})'.format(class_creation_string)

                if verbose:
                    print(('class_creation_string', class_creation_string))
                    print(('class_of_script', class_of_script))
                    print(('scripts', sub_scripts))

                try:
                    script_instance = eval(class_creation_string)
                except Exception as err:
                    print(('loading script {:s} failed. Could not create instance of script!'.format(script_name)))
                    load_failed[script_name] = err
                    if raise_errors:
                        raise err
                    continue

                if script_doc:
                    script_instance.__doc__ = script_doc

                updated_scripts.update({script_name: script_instance})


        return updated_scripts, load_failed, updated_instruments

    @staticmethod
    def get_script_information(script_information, package='pylabcontrol', verbose=False):
        """
        extracts all the relevant information from script_information and returns it as individual variables
        Args:
            script_information: information of the script. This can be
                - a dictionary
                - a Script instance
                - name of Script class
            package (optional): name of the package to which the script belongs, i.e. pylabcontrol or b26toolkit.
                                Only used when script_information is a string
        Returns:
            module, script_class_name, script_settings, script_instruments, script_sub_scripts, script_info, package
        """

        script_settings = None
        script_instruments = None
        script_sub_scripts = None
        script_class_name = None
        module = None  # this is the module that contains the script where we look for scripts
        script_info = None # this is the docstring that describes the script
        module_path = package + '.pylabcontrol.scripts'
        script_filepath = None

        if isinstance(script_information, dict):
            if 'settings' in script_information:
                script_settings = script_information['settings']
            if 'filepath' in script_information:
                script_filepath = str(script_information['filepath'])
                module_path, _ = module_name_from_path(script_filepath)
            if 'package' in script_information:
                package = script_information['package']
            else:
                assert 'filepath' in script_information  # there should be a filepath if we load form a b26 file
                # in the case that we generate the script_information from a .py file the package is given by the name of the highest module
                if 'filepath' in script_information:
                    package = module_path.split('.')[0]

            script_class_name = str(script_information['class'])
            if 'ScriptIterator' in script_class_name:
                module_path = package + '.pylabcontrol.core.script_iterator'

            if 'instruments' in script_information:
                script_instruments = script_information['instruments']
            if 'scripts' in script_information:
                script_sub_scripts = script_information['scripts']
            if 'info' in script_information:
                script_info = script_information['info']
                
        elif isinstance(script_information, str):
            script_class_name = script_information


        elif issubclass(script_information, Script):
            # watch out when testing this code from __main__, then classes might not be identified correctly because the path is different
            # to avoid this problem call from pylabcontrol.core import Script (otherwise the path to Script is __main__.Script)
            script_class_name = script_information.__name__
            package = script_information.__module__.split('.')[0]
            module_path = script_information.__module__

        assert isinstance(package, str)


        # if the script has not been created yet, i.e. script_class_name: ScriptIteratorB26 or ScriptIterator
        if verbose:
            print(('script_filepath', script_filepath))
            print(('path_to_module', module_path))

        if script_filepath is not None:
            # scriptiterator loaded from file
            if os.path.basename(script_filepath.split('.pyc')[0].split('.py')[0]) == 'script_iterator':
                module_path = package + '.pylabcontrol.core.script_iterator'


        # if the script has been created already, i.e. script_class_name: package.dynamic_script_iterator
        # todo: now there is the prefix package
        if len(script_class_name.split('dynamic_script_iterator')) == 2 and \
                script_class_name.split('dynamic_script_iterator')[1].isdigit():
            # package = 'pylabcontrol' # all the dynamic iterator scripts are defined in the name space of pylabcontrol
            # all the dynamic iterator scripts are defined in the name space of package.pylabcontrol.core.script_iterator
            # module = import_module(package + '.pylabcontrol.core.script_iterator')
            module_path = package

        # the package should be the highest level of the module path
        assert module_path.split('.')[0] == package
        assert isinstance(module_path, str)  # in that case we should have defined a module_path to load the module
        assert module is None  # we haven't loaded the module yet

        # try:
        #     print(module_path)
        #     module = import_module(module_path)
        #     print(module)
        # except ImportError:
        #     pass
        module = import_module(module_path)
        # check if module was found!
        if module is None or not hasattr(module, script_class_name):
            import sys
            print('here is the pythonpath')
            for path in sys.path:
                print(path)
            import time
            time.sleep(1)
            print(('Could not find the module that contains ' + script_class_name + ' in module ' + module_path))
            raise ImportError('Could not find the module that contains ' + script_class_name + ' in module ' + module_path)

        # if the module has a name of type dynamic_script_iteratorX where X is a number the module is script iterator
        return module, script_class_name, script_settings, script_instruments, script_sub_scripts, script_info, package

    @staticmethod
    def get_script_module(script_information, package='pylabcontrol', verbose=False):
        """
        wrapper to get the module for a script

        Args:
            script_information: information of the script. This can be
                - a dictionary
                - a Script instance
                - name of Script class
            package (optional): name of the package to which the script belongs, i.e. pylabcontrol or b26toolkit only used when script_information is a string
        Returns:
            module

        """

        module, _, _, _, _, _, _ = Script.get_script_information(script_information=script_information, package=package, verbose=verbose)

        return module

    def duplicate(self):
        """
        create an copy of the script

        Returns:

        """

        # get settings of script
        class_of_script = self.__class__
        script_name = self.name
        script_instruments = self.instruments
        sub_scripts = self.scripts
        script_settings = self.settings
        log_function = self.log_function
        data_path = self.data_path


        #create a new instance of same script type
        class_creation_string = ''
        if script_instruments != {}:
            class_creation_string += ', instruments = script_instruments'
        if sub_scripts != {}:
            class_creation_string += ', scripts = sub_scripts'
        if script_settings != {}:
            class_creation_string += ', settings = script_settings'
        if log_function is not None:
            class_creation_string += ', log_function = log_function'
        if data_path is not None:
            class_creation_string += ', data_path = data_path'
        class_creation_string = 'class_of_script(name=script_name{:s})'.format(class_creation_string)
        # create instance
        script_instance = eval(class_creation_string)

        # copy some other properties that might be checked later for the duplicated script
        script_instance.data = deepcopy(self.data)
        script_instance.start_time = self.start_time
        script_instance.end_time = self.end_time
        script_instance.is_running = self.is_running

        return script_instance

    def _plot(self, axes_list):
        """
        plots the data only the axes objects that are provided in axes_list
        Args:
            axes_list: a list of axes objects, this should be implemented in each subscript

        Returns: None

        """
        pass
        # not sure if to raise a not implemented error or just give a warning. For now just warning
        print(('INFO: {:s} called _plot even though it is not implemented'.format(self.name)))

    def _update_plot(self, axes_list):
        """
        updates the data in already existing plots. the axes objects are provided in axes_list
        Args:
            axes_list: a list of axes objects, this should be implemented in each subscript

        Returns: None

        """

        # default behaviour just calls the standard plot function that creates a new image everytime it is called
        # for heavier plots such as images implement a function here that updates only the date of the plot
        # but doesn't create a whole new image
        self._plot(axes_list)


    def force_update(self):
        """
        forces the plot to refresh
        Returns:

        """
        self._plot_refresh = True

    def plot(self, figure_list):
        """
        plots the data contained in self.data, which should be a dictionary or a deque of dictionaries
        for the latter use the last entry
        Args:
            figure_list: list of figure objects that are passed to self.get_axes_layout to get axis objects for plotting
        """
        # if there is not data we do not plot anything
        if not self.data:
            return

        # if plot function is called when script is not running we request a plot refresh
        if not self.is_running:
            self._plot_refresh = True

        axes_list = self.get_axes_layout(figure_list)
        if self._plot_refresh is True:
            self._plot(axes_list)
            self._plot_refresh = False
            for figure in figure_list:
                if figure.axes:
                    figure.set_tight_layout(True)
        else:
            self._update_plot(axes_list)

    def get_axes_layout(self, figure_list):
        """
        returns the axes objects the script needs to plot its data
        the default creates a single axes object on each figure
        This can/should be overwritten in a child script if more axes objects are needed
        Args:
            figure_list: a list of figure objects
        Returns:
            axes_list: a list of axes objects

        """
        axes_list = []
        if self._plot_refresh is True:
            for fig in figure_list:
                fig.clf()
                axes_list.append(fig.add_subplot(111))

        else:
            for fig in figure_list:
                axes_list.append(fig.axes[0])

        return axes_list

    def plot_validate(self, figure_list):
        """
        plots the data contained in self.data, which should be a dictionary or a deque of dictionaries
        for the latter use the last entry

        """
        axes_list = self.get_axes_layout_validate(figure_list)
        self._plot_validate(axes_list)

    def _plot_validate(self, axes_list):
        """
        plot some visual output as a result of the validation (see self.validate)
        This will most likely be removed in future version: instead preview function, maybe
        :param axes_list: list of axes objects on which to plot
        """
        pass

    def get_axes_layout_validate(self, figure_list):
        """
        creates the axes layout for the validation plots
        :param figure_list: list of figures
        :return: list of axes objects
        """
        return self.get_axes_layout(figure_list)



if __name__ == '__main__':
    pass


    # # test for get_script_information
    # sinfo = {'info': '\nExample Script that has all different types of parameters (integer, str, fload, point, list of parameters). Plots 1D and 2D data.\n    ',
    #                                          'settings': {'count': 3, 'name': 'this is a counter', 'wait_time': 0.1, 'point2': {'y': 0.1, 'x': 0.1}, 'tag': 'scriptdummy', 'path': '',
    #                                                       'save': False, 'plot_style': 'main'},
    #                                          'class': 'ScriptDummy', 'filepath': '/Users/rettentulla/PycharmProjects/pylabcontrol/pylabcontrol/scripts/script_dummy.py'}
    #
    # info = Script.get_script_information(sinfo, verbose=True)
    #
    # print(info)
    #