# AUTOGENERATED! DO NOT EDIT! File to edit: 10_synchro.io.ipynb (unless otherwise specified).

__all__ = ['atoi', 'natural_keys', 'filter_per_extension', 'print_and_log', 'print_info', 'print_error', 'get_offset',
           'logger', 'DataFile', 'read_header', 'get_bytes_per_data_block', 'read_qstring', 'RHDFile', 'H5File',
           'RawBinaryFile', 'NumpyFile', 'load_all_data', 'load_all_data_adc', 'load_all_data_dig_in',
           'load_all_data_both', 'export_adc_raw', 'export_dig_in_raw', 'export_raw', 'export_both_raw', 'load_adc_raw',
           'load_digin_raw', 'load_sync_raw']

# Cell
import numpy as np
import re, sys, os, logging, struct
import h5py
from colorama import Fore

logger = logging.getLogger(__name__)

def atoi(text):
    return int(text) if text.isdigit() else text

def natural_keys(text):
    '''
    alist.sort(key=natural_keys) sorts in human order
    http://nedbatchelder.com/blog/200712/human_sorting.html
    (See Toothy's implementation in the comments)
    '''
    return [atoi(c) for c in re.split('(\d+)', text) ]

def filter_per_extension(files, extension):
    results = []
    for file in files:
        fn, ext = os.path.splitext(file)
        if ext == extension:
            results += [file]
    return results

def print_and_log(to_print, level='info', logger=None, display=True):
    if display:
        if level == 'default':
            for line in to_print:
                print(Fore.WHITE + line + '\n')
        if level == 'info':
            print_info(to_print)
        elif level == 'error':
            print_error(to_print)
    sys.stdout.flush()

def print_info(lines):
    """Prints informations messages, enhanced graphical aspects."""
    print(Fore.YELLOW + "-------------------------  Informations  -------------------------\n")
    for line in lines:
        print(Fore.YELLOW + "| " + line + '\n')
    print(Fore.YELLOW + "------------------------------------------------------------------\n" + Fore.WHITE)

def print_error(lines):
    """Prints errors messages, enhanced graphical aspects."""
    print(Fore.RED + "----------------------------  Error  -----------------------------\n")
    for line in lines:
        print(Fore.RED + "| " + line + '\n')
    print(Fore.RED + "------------------------------------------------------------------\n" + Fore.WHITE)


def get_offset(data_dtype, dtype_offset):

    if dtype_offset == 'auto':
        if data_dtype in ['uint16', np.uint16]:
            dtype_offset = 32768
        elif data_dtype in ['int16', np.int16]:
            dtype_offset = 0
        elif data_dtype in ['int32', np.int32]:
            dtype_offset = 0
        elif data_dtype in ['int64', np.int64]:
            dtype_offset = 0
        elif data_dtype in ['float32', np.float32]:
            dtype_offset = 0
        elif data_dtype in ['int8', np.int8]:
            dtype_offset = 0
        elif data_dtype in ['uint8', np.uint8]:
            dtype_offset = 127
        elif data_dtype in ['float64', np.float64]:
            dtype_offset = 0
        elif data_dtype==">d":
            dtype_offset = 0
    else:
        try:
            dtype_offset = int(dtype_offset)
        except:
            print("Offset %s is not valid" %dtype_offset)

    return dtype_offset

# Cell
class DataFile(object):

    '''
    A generic class that will represent how the program interacts with the data. Such an abstraction
    layer should allow people to write their own wrappers, for several file formats, with or without
    parallel write, streams, and so on. Note that depending on the complexity of the datastructure,
    this extra layer can slow down the code.
    '''

    description      = "mydatafile"     # Description of the file format
    extension        = [".myextension"] # extensions
    parallel_write   = False            # can be written in parallel (using the comm object)
    is_writable      = False            # can be written
    is_streamable    = ['multi-files']  # If the file format can support streams of data ['multi-files' is a default, but can be something else]
    _shape           = None             # The total shape of the data (nb time steps, nb channels) accross streams if any
    _t_start         = None             # The global t_start of the data
    _t_stop          = None             # The final t_stop of the data, accross all streams if any

    # This is a dictionary of values that need to be provided to the constructor, with the corresponding type
    _required_fields = {}

    # This is a dictionary of values that may have a default value, if not provided to the constructor
    _default_values  = {}

    _params          = {}

    def __init__(self, file_name, params, is_empty=False, stream_mode=None):
        '''
        The constructor that will create the DataFile object. Note that by default, values are read from the header
        of the file. If not found in the header, they are read from the parameter file. If no values are found, the
        code will trigger an error

        What you need to specify at a generic level (for a given file format)
            - parallel_write  : can the file be safely written in parallel ?
            - is_writable     : if the file can be written
            - is_streamable   : if the file format can support streaming data
            - required_fields : what parameter must be specified for the file format, along with the type
            - default_values  : parameters that may have default values if not provided

        What you need to specify at a low level (maybe by getting specific values with _read_from_header)
            - _shape          : the size of the data, should be a tuple (duration in time bins, nb_channels)
            - _t_start        : the time (in time steps) of the recording (0 by default)
        '''

        self.params = {}
        self.params.update(self._params)

        if not is_empty:
            self._check_filename(file_name)

        if stream_mode is not None:
            self.is_stream = True
            if not stream_mode in self.is_streamable:
                if self.is_master:
                    print_and_log(["The file format %s does not support stream mode %s" %(self.description, stream_mode)], 'error', logger)
                sys.exit(1)
            if is_empty:
                sys.exit(1)
        else:
            self.is_stream = False

        self.file_name   = file_name
        self.is_empty    = is_empty
        self.stream_mode = stream_mode

        f_next, extension = os.path.splitext(self.file_name)

        self._check_extension(extension)
        self._fill_from_params(params)

        if not self.is_empty:
            try:
                self._fill_from_header(self._read_from_header())
            except Exception as ex:
                print_and_log(["There is an error in the _read_from_header method of the wrapper\n" + str(ex)], 'error', logger)
        else:
            self._shape = (0, 0)

        if self._shape is None:
            sys.exit(1)

        self.params['dtype_offset'] = get_offset(self.data_dtype, self.dtype_offset)

        if self.stream_mode:
            self._sources = self.set_streams(self.stream_mode)
            self._times   = []
            for source in self._sources:
                self._times += [source.t_start]
            print_and_log(['The file is composed of %d streams' %len(self._sources),
                           'Times are between %d and %d' %(self._sources[0].t_start, self._sources[-1].t_stop)], 'debug',logger)

    ##################################################################################################################
    ##################################################################################################################
    #########                  Methods that need to be overwritten for a given fileformat                      #######
    ##################################################################################################################
    ##################################################################################################################


    def _read_from_header(self):
        '''
            This function is called only if the file is not empty, and should fill the values in the constructor
            such as _shape. It returns a dictionnary, that will be added to self._params based on the constrains given by
            required_fields and default_values
        '''
        raise NotImplementedError('The _read_from_header method needs to be implemented for file format %s' %self.description)


    def _open(self, mode=''):
        '''
            This function should open the file
            - mode can be to read only 'r', or to write 'w'
        '''
        raise NotImplementedError('The open method needs to be implemented for file format %s' %self.description)


    def _close(self):
        '''
            This function closes the file
        '''
        raise NotImplementedError('The close method needs to be implemented for file format %s' %self.description)


    def read_chunk(self, idx, chunk_size, padding=(0, 0), nodes=None):
        '''
        Assuming the analyze function has been called before, this is the main function
        used by the code, in all steps, to get data chunks. More precisely, assuming your
        dataset can be divided in nb_chunks (see analyze) of temporal size (chunk_size),

            - idx is the index of the chunk you want to load
            - chunk_size is the time of those chunks, in time steps
            - if the data loaded are data[idx:idx+1], padding should add some offsets,
                in time steps, such that we can load data[idx+padding[0]:idx+padding[1]]
            - nodes is a list of nodes, between 0 and nb_channels
        '''

        raise NotImplementedError('The read_chunk method needs to be implemented for file format %s' %self.description)

    def read_chunk_adc(self, idx, chunk_size, padding=(0, 0), nodes=None):
        '''
        Same as read_chunk, but for the analog channel of the file.

            - idx is the index of the chunk you want to load
            - chunk_size is the time of those chunks, in time steps
            - if the data loaded are data[idx:idx+1], padding should add some offsets,
                in time steps, such that we can load data[idx+padding[0]:idx+padding[1]]
            - nodes is a list of nodes, between 0 and nb_channels
        '''

        raise NotImplementedError('The read_chunk_adc method needs to be implemented for file format %s' %self.description)


    def write_chunk(self, time, data):
        '''
            This function writes data at a given time.
            - time is expressed in timestep
            - data must be a 2D matrix of size time_length x nb_channels
        '''
        raise NotImplementedError('The set_data method needs to be implemented for file format %s' %self.description)


    def set_streams(self, stream_mode):
        '''
            This function is only used for file format supporting streams, and need to return a list of datafiles, with
            appropriate t_start for each of them. Note that the results will be using the times defined by the streams.
            You can do anything regarding the keyword used for the stream mode, but multi-files is immplemented by default
            This will allow every file format to be streamed from multiple sources, and processed as a single file.
        '''

        if stream_mode == 'multi-files':
            dirname         = os.path.abspath(os.path.dirname(self.file_name))
            fname           = os.path.basename(self.file_name)
            fn, ext         = os.path.splitext(fname)
            all_files       = os.listdir(dirname)
            all_files       = filter_per_extension(all_files, ext)
            all_files.sort(key=natural_keys)

            sources         = []
            to_write        = []
            global_time     = 0
            params          = self.get_description()

            for fname in all_files:
                new_data   = type(self)(os.path.join(os.path.abspath(dirname), fname), params)
                new_data._t_start = global_time
                global_time += new_data.duration
                sources     += [new_data]
                to_write    += ['We found the datafile %s with t_start %s and duration %s' %(new_data.file_name, new_data.t_start, new_data.duration)]
            print_and_log(to_write, 'debug', logger)
            return sources

    ################################## Optional, only if internal names are changed ##################################

    @property
    def sampling_rate(self):
        return self.params['sampling_rate']

    @property
    def data_dtype(self):
        return self.params['data_dtype']

    @property
    def dtype_offset(self):
        return self.params['dtype_offset']

    @property
    def data_offset(self):
        return self.params['data_offset']

    @property
    def nb_channels(self):
        return int(self.params['nb_channels'])

    @property
    def gain(self):
        return self.params['gain']

    ##################################################################################################################
    ##################################################################################################################
    #########           End of methods that need to be overwritten for a given fileformat                      #######
    ##################################################################################################################
    ##################################################################################################################


    def get_file_names(self):
        res = []
        if self.stream_mode == 'multi-files':
            for source in self._sources:
                res += [source.file_name]
        return res

    def _check_filename(self, file_name):
        if not os.path.exists(file_name):
            sys.exit(1)


    def _check_extension(self, extension):
        if len(self.extension) > 0:
            if not extension in self.extension + [item.upper() for item in self.extension]:
                sys.exit(1)


    def _fill_from_params(self, params):

        for key in self._required_fields:
            if key not in params:
                self._check_requirements_(params)
            else:
                self.params[key] = self._required_fields[key](params[key])

        for key in self._default_values:
            if key not in params:
                self.params[key] = self._default_values[key]
            else:
                self.params[key] = type(self._default_values[key])(params[key])

    def _fill_from_header(self, header):

        for key in list(header.keys()):
            self.params[key] = header[key]

    def _check_requirements_(self, params):

        missing = {}

        for key, value in list(self._required_fields.items()):
            if key not in list(params.keys()):
                missing[key] = value

        if len(missing) > 0:
            self._display_requirements_()
            sys.exit(1)


    def _display_requirements_(self):

        to_write  = ['The parameters for %s file format are:' %self.description.upper(), '']
        nb_params = 0

        for key, value in list(self._required_fields.items()):
            mystring  = '-- %s -- %s' %(key, str(value))
            mystring  += ' [** mandatory **]'
            to_write  += [mystring]
            nb_params += 1

        to_write += ['']

        for key, value in list(self._default_values.items()):
            mystring  = '-- %s -- %s' %(key, str(type(value)))
            mystring  += ' [default is %s]' %value
            to_write  += [mystring]
            nb_params += 1

    def _scale_data_to_float32(self, data):
        '''
            This function will convert data from local data dtype into float32, the default format of the algorithm
        '''
        if self.data_dtype != np.float32:
            data  = data.astype(np.float32)

        if self.dtype_offset != 0:
            data  -= self.dtype_offset

        if np.any(self.gain != 1):
            data *= self.gain

        return np.ascontiguousarray(data)


    def _unscale_data_from_float32(self, data):
        '''
            This function will convert data from float32 back to the original format of the file
        '''

        if np.any(self.gain != 1):
            data /= self.gain

        if self.dtype_offset != 0:
            data  += self.dtype_offset

        if (data.dtype != self.data_dtype) and (self.data_dtype != np.float32):
            data = data.astype(self.data_dtype)

        return data


    def _count_chunks(self, chunk_size, duration, strict=False):
        '''
            This function will count how many block of size chunk_size can be found within a certain duration
            This returns the number of blocks, plus the remaining part
        '''
        nb_chunks      = duration // chunk_size
        last_chunk_len = duration - nb_chunks * chunk_size

        if not strict and last_chunk_len > 0:
            nb_chunks += 1

        return nb_chunks, last_chunk_len


    def _get_t_start_t_stop(self, idx, chunk_size, padding=(0,0)):

        t_start     = idx*np.int64(chunk_size)+padding[0]
        t_stop      = (idx+1)*np.int64(chunk_size)+padding[1]

        if t_stop > self.duration:
            t_stop = self.duration

        if t_start < 0:
            t_start = 0

        return t_start, t_stop


    def _get_streams_index_by_time(self, local_time):
        if self.is_stream:
            cidx  = np.searchsorted(self._times, local_time, 'right') - 1
            return cidx

    def is_first_chunk(self, idx, nb_chunks):

        if self.is_stream:
            cidx = np.searchsorted(self._chunks_in_sources, idx, 'right') - 1
            idx -= self._chunks_in_sources[cidx]
            if idx == 0:
                return True
        else:
            if idx == 0:
                return True
        return False

    def is_last_chunk(self, idx, nb_chunks):

        if self.is_stream:
            if (idx > 0) and (idx in self._chunks_in_sources - 1):
                return True
        else:
            if idx == nb_chunks:
                return True
        return False

    def get_snippet(self, global_time, length, nodes=None):
        '''
            This function should return a time snippet of size length x nodes
            - time is in timestep
            - length is in timestep
            - nodes is a list of nodes, between 0 and nb_channels
        '''
        if self.is_stream:
            cidx = self._get_streams_index_by_time(global_time)
            return self._sources[cidx].get_snippet(global_time, length, nodes)
        else:
            local_time = global_time - self.t_start
            return self.get_data(0, chunk_size=length, padding=(local_time, local_time), nodes=nodes)[0]

    def get_snippet_adc(self, global_time, length, nodes=None):
        '''
            This function should return a time snippet of size length x nodes
            - time is in timestep
            - length is in timestep
            - nodes is a list of nodes, between 0 and nb_channels
        '''
        if self.is_stream:
            cidx = self._get_streams_index_by_time(global_time)
            return self._sources[cidx].get_snippet_adc(global_time, length, nodes)
        else:
            local_time = global_time - self.t_start
            return self.get_data_adc(0, chunk_size=length, padding=(local_time, local_time), nodes=nodes)[0]


    def get_data(self, idx, chunk_size, padding=(0, 0), nodes=None):

        if self.is_stream:
            cidx = np.searchsorted(self._chunks_in_sources, idx, 'right') - 1
            idx -= self._chunks_in_sources[cidx]
            return self._sources[cidx].read_chunk(idx, chunk_size, padding, nodes), self._sources[cidx].t_start + idx*chunk_size
        else:
            return self.read_chunk(idx, chunk_size, padding, nodes), self.t_start + idx*chunk_size

    def get_data_adc(self, idx, chunk_size, padding=(0, 0), nodes=None):

        if self.is_stream:
            cidx = np.searchsorted(self._chunks_in_sources, idx, 'right') - 1
            idx -= self._chunks_in_sources[cidx]
            return self._sources[cidx].read_chunk_adc(idx, chunk_size, padding, nodes), self._sources[cidx].t_start + idx*chunk_size
        else:
            return self.read_chunk_adc(idx, chunk_size, padding, nodes), self.t_start + idx*chunk_size

    def get_data_dig_in(self, idx, chunk_size, padding=(0, 0), nodes=None):

        if self.is_stream:
            cidx = np.searchsorted(self._chunks_in_sources, idx, 'right') - 1
            idx -= self._chunks_in_sources[cidx]
            return self._sources[cidx].read_chunk_dig_in(idx, chunk_size, padding, nodes), self._sources[cidx].t_start + idx*chunk_size
        else:
            return self.read_chunk_dig_in(idx, chunk_size, padding, nodes), self.t_start + idx*chunk_size

    def get_data_both(self, idx, chunk_size, padding=(0, 0), nodes=None):

        if self.is_stream:
            cidx = np.searchsorted(self._chunks_in_sources, idx, 'right') - 1
            idx -= self._chunks_in_sources[cidx]
            return (*self._sources[cidx].read_chunk_both(idx, chunk_size, padding, nodes), self._sources[cidx].t_start + idx*chunk_size)
        else:
            return (*self.read_chunk_both(idx, chunk_size, padding, nodes), self.t_start + idx*chunk_size)

    def set_data(self, global_time, data):

        if self.is_stream:
            cidx = self._get_streams_index_by_time(global_time)
            local_time = global_time - self._sources[cidx].t_start
            return self._sources[cidx].write_chunk(local_time, data)
        else:
            local_time = global_time - self.t_start
            return self.write_chunk(local_time, data)


    def analyze(self, chunk_size, strict=False):
        '''
            This function should return two values:
            - the number of temporal chunks of temporal size chunk_size that can be found
            in the data. Note that even if the last chunk is not complete, it has to be
            counted. chunk_size is expressed in time steps
            - the length of the last uncomplete chunk, in time steps
        '''
        if self.is_stream:
            nb_chunks               = 0
            last_chunk_len          = 0
            self._chunks_in_sources = [0]

            for source in self._sources:
                a, b            = self._count_chunks(chunk_size, source.duration, strict)
                nb_chunks      += a
                last_chunk_len += b

                self._chunks_in_sources += [nb_chunks]

            self._chunks_in_sources = np.array(self._chunks_in_sources)

            return nb_chunks, last_chunk_len
        else:
            return self._count_chunks(chunk_size, self.duration, strict)


    def get_description(self):
        result = {}
        for key in ['sampling_rate', 'data_dtype', 'gain', 'nb_channels', 'dtype_offset'] + list(self._default_values.keys()) + list(self._required_fields.keys()):
            result[key] = self.params[key]
        return result


    @property
    def shape(self):
        return (self.duration, int(self.nb_channels))


    @property
    def duration(self):
        if self.is_stream:
            duration = 0
            for source in self._sources:
                duration += source.duration
            return duration
        else:
            return np.int64(self._shape[0])


    @property
    def is_master(self):
        return True#comm.rank == 0


    @property
    def t_start(self):
        if self.is_stream:
            return self._sources[0].t_start
        else:
            if self._t_start is None:
                self._t_start = 0
            return self._t_start


    @property
    def t_stop(self):
        if self.is_stream:
            return self._sources[-1].t_stop
        else:
            if self._t_stop is None:
                self._t_stop = self.t_start + self.duration
            return self._t_stop


    @property
    def nb_streams(self):
        if self.is_stream:
            return len(self._sources)
        else:
            return 1

    def open(self, mode='r'):
        if self.is_stream:
            for source in self._sources:
                source._open(mode)
        else:
            self._open(mode)


    def close(self):
        if self.is_stream:
            for source in self._sources:
                source._close()
        else:
            self._close()

# Cell
def read_header(fid):
    """Reads the Intan File Format header from the given file."""

    # Check 'magic number' at beginning of file to make sure this is an Intan
    # Technologies RHD2000 data file.
    magic_number, = struct.unpack('<I', fid.read(4))
    if magic_number != int('c6912702', 16): raise Exception('Unrecognized file type.')

    header = {}
    # Read version number.
    version = {}
    (version['major'], version['minor']) = struct.unpack('<hh', fid.read(4))
    header['version'] = version

    freq = {}

    # Read information of sampling rate and amplifier frequency settings.
    header['sample_rate'], = struct.unpack('<f', fid.read(4))
    (freq['dsp_enabled'], freq['actual_dsp_cutoff_frequency'], freq['actual_lower_bandwidth'], freq['actual_upper_bandwidth'],
    freq['desired_dsp_cutoff_frequency'], freq['desired_lower_bandwidth'], freq['desired_upper_bandwidth']) = struct.unpack('<hffffff', fid.read(26))


    # This tells us if a software 50/60 Hz notch filter was enabled during
    # the data acquisition.
    notch_filter_mode, = struct.unpack('<h', fid.read(2))
    header['notch_filter_frequency'] = 0
    if notch_filter_mode == 1:
        header['notch_filter_frequency'] = 50
    elif notch_filter_mode == 2:
        header['notch_filter_frequency'] = 60
    freq['notch_filter_frequency'] = header['notch_filter_frequency']

    (freq['desired_impedance_test_frequency'], freq['actual_impedance_test_frequency']) = struct.unpack('<ff', fid.read(8))

    note1 = read_qstring(fid)
    note2 = read_qstring(fid)
    note3 = read_qstring(fid)
    header['notes'] = { 'note1' : note1, 'note2' : note2, 'note3' : note3}

    # If data file is from GUI v1.1 or later, see if temperature sensor data was saved.
    header['num_temp_sensor_channels'] = 0
    if (version['major'] == 1 and version['minor'] >= 1) or (version['major'] > 1) :
        header['num_temp_sensor_channels'], = struct.unpack('<h', fid.read(2))


    # If data file is from GUI v1.3 or later, load eval board mode.
    header['eval_board_mode'] = 0
    if ((version['major'] == 1) and (version['minor'] >= 3)) or (version['major'] > 1) :
        header['eval_board_mode'], = struct.unpack('<h', fid.read(2))

    # Place frequency-related information in data structure. (Note: much of this structure is set above)
    freq['amplifier_sample_rate'] = header['sample_rate']
    freq['aux_input_sample_rate'] = header['sample_rate'] / 4
    freq['supply_voltage_sample_rate'] = header['sample_rate'] / 60
    freq['board_adc_sample_rate'] = header['sample_rate']
    freq['board_dig_in_sample_rate'] = header['sample_rate']

    header['frequency_parameters'] = freq

    # Create structure arrays for each type of data channel.
    header['spike_triggers'] = []
    header['amplifier_channels'] = []
    header['aux_input_channels'] = []
    header['supply_voltage_channels'] = []
    header['board_adc_channels'] = []
    header['board_dig_in_channels'] = []
    header['board_dig_out_channels'] = []

    # Read signal summary from data file header.

    if (header['version']['major'] > 1):
        header['reference_channel'] = read_qstring(fid)

    number_of_signal_groups, = struct.unpack('<h', fid.read(2))

    for signal_group in range(0, number_of_signal_groups):
        signal_group_name = read_qstring(fid)
        signal_group_prefix = read_qstring(fid)
        (signal_group_enabled, signal_group_num_channels, signal_group_num_amp_channels) = struct.unpack('<hhh', fid.read(6))

        if (signal_group_num_channels > 0) and (signal_group_enabled > 0):
            for signal_channel in range(0, signal_group_num_channels):
                new_channel = {'port_name' : signal_group_name, 'port_prefix' : signal_group_prefix, 'port_number' : signal_group}
                new_channel['native_channel_name'] = read_qstring(fid)
                new_channel['custom_channel_name'] = read_qstring(fid)
                (new_channel['native_order'], new_channel['custom_order'], signal_type, channel_enabled, new_channel['chip_channel'], new_channel['board_stream']) = struct.unpack('<hhhhhh', fid.read(12))
                new_trigger_channel = {}
                (new_trigger_channel['voltage_trigger_mode'], new_trigger_channel['voltage_threshold'], new_trigger_channel['digital_trigger_channel'], new_trigger_channel['digital_edge_polarity'])  = struct.unpack('<hhhh', fid.read(8))
                (new_channel['electrode_impedance_magnitude'], new_channel['electrode_impedance_phase']) = struct.unpack('<ff', fid.read(8))

                if channel_enabled:
                    if signal_type == 0:
                        header['amplifier_channels'].append(new_channel)
                        header['spike_triggers'].append(new_trigger_channel)
                    elif signal_type == 1:
                        header['aux_input_channels'].append(new_channel)
                    elif signal_type == 2:
                        header['supply_voltage_channels'].append(new_channel)
                    elif signal_type == 3:
                        header['board_adc_channels'].append(new_channel)
                    elif signal_type == 4:
                        header['board_dig_in_channels'].append(new_channel)
                    elif signal_type == 5:
                        header['board_dig_out_channels'].append(new_channel)
                    else:
                        raise Exception('Unknown channel type.')


    # Summarize contents of data file.
    header['num_amplifier_channels'] = len(header['amplifier_channels'])
    header['num_aux_input_channels'] = len(header['aux_input_channels'])
    header['num_supply_voltage_channels'] = len(header['supply_voltage_channels'])
    header['num_board_adc_channels'] = len(header['board_adc_channels'])
    header['num_board_dig_in_channels'] = len(header['board_dig_in_channels'])
    header['num_board_dig_out_channels'] = len(header['board_dig_out_channels'])

    return header


def get_bytes_per_data_block(header):
    """Calculates the number of bytes in each 60-sample datablock."""

    if (header['version']['major'] == 1):
        num_samples_per_data_block = 60
    else:
        num_samples_per_data_block = 128

    # Each data block contains 60 amplifier samples.
    bytes_per_block = num_samples_per_data_block * 4  # timestamp data
    bytes_per_block = bytes_per_block + num_samples_per_data_block * 2 * header['num_amplifier_channels']

    # Auxiliary inputs are sampled 4x slower than amplifiers
    bytes_per_block = bytes_per_block + (num_samples_per_data_block / 4) * 2 * header['num_aux_input_channels']

    # Supply voltage is sampled 60x slower than amplifiers
    bytes_per_block = bytes_per_block + 1 * 2 * header['num_supply_voltage_channels']

    # Board analog inputs are sampled at same rate as amplifiers
    bytes_per_block = bytes_per_block + num_samples_per_data_block * 2 * header['num_board_adc_channels']

    # Board digital inputs are sampled at same rate as amplifiers
    if header['num_board_dig_in_channels'] > 0:
        bytes_per_block = bytes_per_block + num_samples_per_data_block * 2

    # Board digital outputs are sampled at same rate as amplifiers
    if header['num_board_dig_out_channels'] > 0:
        bytes_per_block = bytes_per_block + num_samples_per_data_block * 2

    # Temp sensor is sampled 60x slower than amplifiers
    if header['num_temp_sensor_channels'] > 0:
        bytes_per_block = bytes_per_block + 1 * 2 * header['num_temp_sensor_channels']

    return bytes_per_block



def read_qstring(fid):
    """Read Qt style QString.

    The first 32-bit unsigned number indicates the length of the string (in bytes).
    If this number equals 0xFFFFFFFF, the string is null.

    Strings are stored as unicode.
    """

    length, = struct.unpack('<I', fid.read(4))
    if length == int('ffffffff', 16): return ""

    if length > (os.fstat(fid.fileno()).st_size - fid.tell() + 1) :
        print(length)
        raise Exception('Length too long.')

    # convert length from bytes to 16-bit Unicode words
    length = int(length / 2)

    data = []
    for i in range(0, length):
        c, = struct.unpack('<H', fid.read(2))
        data.append(c)

    if sys.version_info >= (3,0):
        a = ''.join([chr(c) for c in data])
    else:
        a = ''.join([chr(c) for c in data])

    return a


class RHDFile(DataFile):

    description    = "rhd"
    extension      = [".rhd"]
    parallel_write = True
    is_writable    = True
    is_streamable  = ['multi-files']

    _required_fields = {}
    _default_values  = {}

    _params          = {'dtype_offset' : 'auto',
                        'data_dtype'   : 'uint16',
                        'gain'         : 0.195}

    def _read_from_header(self):

        header = {}

        self.file   = open(self.file_name, 'rb')
        full_header = read_header(self.file)
        self.header = full_header
        header['nb_channels']   = full_header['num_amplifier_channels']
        header['sampling_rate'] = full_header['sample_rate']

        if full_header['version']['major'] == 1:
            self.SAMPLES_PER_RECORD = 60
        else:
            self.SAMPLES_PER_RECORD = 128
        self.nb_channels_adc    = full_header['num_board_adc_channels']
        self.nb_channels_dig_in = full_header['num_board_dig_in_channels']
        header['data_offset']   = self.file.tell()

        data_present          = False
        filesize              = os.path.getsize(self.file_name)
        self.bytes_per_block  = get_bytes_per_data_block(full_header)
        self.block_offset     = self.SAMPLES_PER_RECORD * 4
        self.block_size       = 2 * self.SAMPLES_PER_RECORD * header['nb_channels']
        self.block_offset_adc = (self.block_offset + self.block_size +
                                 (self.SAMPLES_PER_RECORD/4) * full_header['num_aux_input_channels'] * 2 +
                                 2 * full_header['num_supply_voltage_channels'])
        self.block_size_adc      = 2 * self.SAMPLES_PER_RECORD * self.nb_channels_adc
        self.block_offset_dig_in = self.block_offset_adc + self.block_size_adc
        self.block_size_dig_in   = 2 * self.SAMPLES_PER_RECORD
        bytes_remaining          = filesize - self.file.tell()

        self.bytes_per_block_div     = self.bytes_per_block / 2
        self.block_offset_div        = self.block_offset / 2
        self.block_offset_div_adc    = self.block_offset_adc / 2
        self.block_offset_div_dig_in = self.block_offset_dig_in / 2
        self.block_size_div          = self.block_size / 2
        self.block_size_div_adc      = self.block_size_adc / 2
        self.block_size_div_dig_in   = self.block_size_dig_in / 2

        if bytes_remaining > 0:
            data_present = True
        if bytes_remaining % self.bytes_per_block != 0:
            print_and_log(['Something is wrong with file size : should have a whole number of data blocks'], 'error', logger)

        num_data_blocks = int(bytes_remaining / self.bytes_per_block)
        self.num_amplifier_samples = self.SAMPLES_PER_RECORD * num_data_blocks

        self.size        = self.num_amplifier_samples
        self._shape      = (self.size, header['nb_channels'])
        self.file.close()

        return header

    def _get_slice_(self, t_start, t_stop):

        x_beg = np.int64(t_start // self.SAMPLES_PER_RECORD)
        r_beg = np.mod(t_start, self.SAMPLES_PER_RECORD)
        x_end = np.int64(t_stop // self.SAMPLES_PER_RECORD)
        r_end = np.mod(t_stop, self.SAMPLES_PER_RECORD)

        if x_beg == x_end:
            g_offset = x_beg * self.bytes_per_block_div + self.block_offset_div
            data_slice = np.arange(g_offset + r_beg * self.nb_channels, g_offset + r_end * self.nb_channels, dtype=np.int64)
            yield data_slice
        else:
            for count, nb_blocks in enumerate(np.arange(x_beg, x_end + 1, dtype=np.int64)):
                g_offset = nb_blocks * self.bytes_per_block_div + self.block_offset_div
                if count == 0:
                    data_slice = np.arange(g_offset + r_beg * self.nb_channels, g_offset + self.block_size_div, dtype=np.int64)
                elif (count == (x_end - x_beg)):
                    data_slice = np.arange(g_offset, g_offset + r_end * self.nb_channels, dtype=np.int64)
                else:
                    data_slice = np.arange(g_offset, g_offset + self.block_size_div, dtype=np.int64)
                yield data_slice

    def _get_slice_adc_(self, t_start, t_stop):

        x_beg = np.int64(t_start // self.SAMPLES_PER_RECORD)
        r_beg = np.mod(t_start, self.SAMPLES_PER_RECORD)
        x_end = np.int64(t_stop // self.SAMPLES_PER_RECORD)
        r_end = np.mod(t_stop, self.SAMPLES_PER_RECORD)

        if x_beg == x_end:
            g_offset = x_beg * self.bytes_per_block_div + self.block_offset_div_adc
            data_slice = np.arange(g_offset + r_beg * self.nb_channels_adc, g_offset + r_end * self.nb_channels_adc, dtype=np.int64)
            yield data_slice
        else:
            for count, nb_blocks in enumerate(np.arange(x_beg, x_end + 1, dtype=np.int64)):
                g_offset = nb_blocks * self.bytes_per_block_div + self.block_offset_div_adc
                if count == 0:
                    data_slice = np.arange(g_offset + r_beg * self.nb_channels_adc, g_offset + self.block_size_div_adc, dtype=np.int64)
                elif (count == (x_end - x_beg)):
                    data_slice = np.arange(g_offset, g_offset + r_end * self.nb_channels_adc, dtype=np.int64)
                else:
                    data_slice = np.arange(g_offset, g_offset + self.block_size_div_adc, dtype=np.int64)
                yield data_slice

    def _get_slice_dig_in_(self, t_start, t_stop):

        x_beg = np.int64(t_start // self.SAMPLES_PER_RECORD)
        r_beg = np.mod(t_start, self.SAMPLES_PER_RECORD)
        x_end = np.int64(t_stop // self.SAMPLES_PER_RECORD)
        r_end = np.mod(t_stop, self.SAMPLES_PER_RECORD)

        if x_beg == x_end:
            g_offset = x_beg * self.bytes_per_block_div + self.block_offset_div_dig_in
            data_slice = np.arange(g_offset + r_beg, g_offset + r_end, dtype=np.int64)
            yield data_slice
        else:
            for count, nb_blocks in enumerate(np.arange(x_beg, x_end + 1, dtype=np.int64)):
                g_offset = nb_blocks * self.bytes_per_block_div + self.block_offset_div_dig_in
                if count == 0:
                    data_slice = np.arange(g_offset + r_beg, g_offset + self.block_size_div_dig_in, dtype=np.int64)
                elif (count == (x_end - x_beg)):
                    data_slice = np.arange(g_offset, g_offset + r_end, dtype=np.int64)
                else:
                    data_slice = np.arange(g_offset, g_offset + self.block_size_div_dig_in, dtype=np.int64)
                yield data_slice


    def read_chunk(self, idx, chunk_size, padding=(0, 0), nodes=None):

        t_start, t_stop = self._get_t_start_t_stop(idx, chunk_size, padding)
        local_shape     = t_stop - t_start

        local_chunk = np.zeros((self.nb_channels, local_shape), dtype=self.data_dtype)
        data_slice  = self._get_slice_(t_start, t_stop)

        self._open()
        count = 0

        for s in data_slice:
            t_slice = len(s)//self.nb_channels
            local_chunk[:, count:count + t_slice] = self.data[s].reshape(self.nb_channels, len(s)//self.nb_channels)
            count += t_slice

        local_chunk = local_chunk.T
        self._close()

        if nodes is not None:
            if not np.all(nodes == np.arange(self.nb_channels)):
                local_chunk = np.take(local_chunk, nodes, axis=1)

        return self._scale_data_to_float32(local_chunk)

    def read_chunk_adc(self, idx, chunk_size, padding=(0, 0), nodes=None):

        t_start, t_stop = self._get_t_start_t_stop(idx, chunk_size, padding)
        local_shape     = t_stop - t_start

        local_chunk = np.zeros((self.nb_channels_adc, local_shape), dtype=self.data_dtype)
        data_slice  = self._get_slice_adc_(t_start, t_stop)

        self._open()
        count = 0

        for s in data_slice:
            t_slice = len(s)//self.nb_channels_adc
            local_chunk[:, count:count + t_slice] = self.data[s].reshape(self.nb_channels_adc, len(s)//self.nb_channels_adc)
            count += t_slice

        local_chunk = local_chunk.T
        self._close()

        if nodes is not None:
            if not np.all(nodes == np.arange(self.nb_channels_adc)):
                local_chunk = np.take(local_chunk, nodes, axis=1)

        return self._scale_data_to_float32(local_chunk)

    def read_chunk_dig_in(self, idx, chunk_size, padding=(0, 0), nodes=None):

        t_start, t_stop = self._get_t_start_t_stop(idx, chunk_size, padding)
        local_shape     = t_stop - t_start

        tmp_chunk   = np.zeros((local_shape,), dtype=np.uint64)
        local_chunk = np.zeros((self.nb_channels_dig_in, local_shape), dtype=np.uint8)
        data_slice  = self._get_slice_dig_in_(t_start, t_stop)

        self._open()
        count = 0
        for s in data_slice:
            t_slice = len(s)
            tmp_chunk[count:count + t_slice] = self.data[s] #Putting all data in channel 1, then masking
            count += t_slice
        for i in range(self.nb_channels_dig_in):
            chann_bitmask  = 1 << self.header['board_dig_in_channels'][i]['native_order']
            local_chunk[i] = np.not_equal(np.bitwise_and(tmp_chunk,
                                                         chann_bitmask),
                                          0)
        local_chunk = local_chunk.T
        self._close()

        if nodes is not None:
            if not np.all(nodes == np.arange(self.nb_channels_dig_in)):
                local_chunk = np.take(local_chunk, nodes, axis=1)

        return local_chunk

    def read_chunk_both(self, idx, chunk_size, padding=(0, 0), nodes=None):

        t_start, t_stop = self._get_t_start_t_stop(idx, chunk_size, padding)
        local_shape     = t_stop - t_start

        local_chunk_adc = np.zeros((self.nb_channels_adc, local_shape), dtype=self.data_dtype)
        local_chunk     = np.zeros((self.nb_channels, local_shape), dtype=self.data_dtype)
        data_slice_adc  = self._get_slice_adc_(t_start, t_stop)
        data_slice      = self._get_slice_(t_start, t_stop)

        self._open()
        count = 0
        for s in data_slice_adc:
            t_slice = len(s)//self.nb_channels_adc
            local_chunk_adc[:, count:count + t_slice] = self.data[s].reshape(self.nb_channels_adc, len(s)//self.nb_channels_adc)
            count += t_slice

        count = 0
        for s in data_slice:
            t_slice = len(s)//self.nb_channels
            local_chunk[:, count:count + t_slice] = self.data[s].reshape(self.nb_channels, len(s)//self.nb_channels)
            count += t_slice

        local_chunk     = local_chunk.T
        local_chunk_adc = local_chunk_adc.T
        self._close()

        if nodes is not None:
            if not np.all(nodes == np.arange(self.nb_channels_adc)):
                local_chunk_adc = np.take(local_chunk_adc, nodes, axis=1)
            if not np.all(nodes == np.arange(self.nb_channels)):
                local_chunk     = np.take(local_chunk, nodes, axis=1)

        return self._scale_data_to_float32(local_chunk), self._scale_data_to_float32(local_chunk_adc)

    def write_chunk(self, time, data):

        t_start     = time
        t_stop      = time + data.shape[0]

        if t_stop > self.duration:
            t_stop  = self.duration

        data = self._unscale_data_from_float32(data)
        data_slice  = self._get_slice_(t_start, t_stop)

        self._open(mode='r+')
        count = 0
        for s in data_slice:
            t_slice      = len(s)//self.nb_channels
            self.data[s] = data[count:count + t_slice, :].T.ravel()
            count += t_slice

        self._close()

    def _open(self, mode='r'):
        self.data = np.memmap(self.file_name, offset=self.data_offset, dtype=self.data_dtype, mode=mode)

    def _close(self):
        self.data = None

# Cell
class H5File(DataFile):

    description    = "hdf5"
    extension      = [".h5", ".hdf5"]
    parallel_write = h5py.get_config().mpi
    is_writable    = True

    _required_fields = {'h5_key'        : str,
                        'sampling_rate' : float}

    _default_values  = {'dtype_offset'  : 'auto',
                        'h5_key_adc'    : "Data/Recording_0/AnalogStream/Stream_1/ChannelData",
                        'gain'          : 1.,
                        'data_dtype'    : 'uint8',
                        'nb_channels'   : 1}


    def _check_compression(self):
        # HDF5 does not support parallel writes with compression
        if self.compression != '':
            self.parallel_write = False
            if self.is_master:
                print_and_log(['Data are compressed thus parallel writing is disabled'], 'debug', logger)

    def __check_valid_key__(self, key):
        file       = h5py.File(self.file_name, mode='r')
        all_fields = []
        file.visit(all_fields.append)
        if not key in all_fields:
            print_and_log(['The key %s can not be found in the dataset! Keys found are:' %key,
                         ", ".join(all_fields)], 'error', logger)
            sys.exit(1)
        file.close()

    def _read_from_header(self):

        self.__check_valid_key__(self.h5_key)
        self._open()

        header = {}
        header['data_dtype']   = self.my_file.get(self.h5_key).dtype
        self.compression       = self.my_file.get(self.h5_key).compression

        self._check_compression()

        self.size        = self.my_file.get(self.h5_key).shape

        if self.size[0] > self.size[1]:
            self.time_axis = 0
            self._shape = (self.size[0], self.size[1])
        else:
            self.time_axis = 1
            self._shape = (self.size[1], self.size[0])

        header['nb_channels']  = self._shape[1]
        self._close()

        return header

    def read_chunk(self, idx, chunk_size, padding=(0, 0), nodes=None):

        t_start, t_stop = self._get_t_start_t_stop(idx, chunk_size, padding)

        if nodes is None:
            if self.time_axis == 0:
                local_chunk = self.data[t_start:t_stop, :]
            elif self.time_axis == 1:
                local_chunk = self.data[:, t_start:t_stop].T
        else:
            if self.time_axis == 0:
                local_chunk = self.data[t_start:t_stop, nodes]
            elif self.time_axis == 1:
                local_chunk = self.data[nodes, t_start:t_stop].T

        return self._scale_data_to_float32(local_chunk)

    def read_chunk_adc(self, idx, chunk_size, padding=(0, 0), nodes=None):

        t_start, t_stop = self._get_t_start_t_stop(idx, chunk_size, padding)

        if nodes is None:
            local_chunk = self.data_adc[0,t_start:t_stop]
        else:
            local_chunk = self.data_adc[0,t_start:t_stop]

        return self._scale_data_to_float32(local_chunk)

    def write_chunk(self, time, data):

        data = self._unscale_data_from_float32(data)

        if self.time_axis == 0:
            self.data[time:time+data.shape[0], :] = data
        elif self.time_axis == 1:
            self.data[:, time:time+data.shape[0]] = data.T

    def _open(self, mode='r'):
#        if mode in ['r+', 'w'] and self.parallel_write:
#            self.my_file = h5py.File(self.file_name, mode=mode, driver='mpio', comm=comm)
#        else:
        self.my_file = h5py.File(self.file_name, mode=mode)

        self.data = self.my_file.get(self.h5_key)
        self.data_adc = self.my_file.get(self.h5_key_adc)

    def _close(self):
        self.my_file.close()
        del self.data
        del self.data_adc

    @property
    def h5_key(self):
        return self.params['h5_key']

    @property
    def h5_key_adc(self):
        return self.params['h5_key_adc']

# Cell
class RawBinaryFile(DataFile):

    description    = "raw_binary"
    extension      = []
    parallel_write = True
    is_writable    = True

    _required_fields = {'data_dtype'    : str,
                        'sampling_rate' : float,
                        'nb_channels'   : int}

    _default_values  = {'dtype_offset'  : 'auto',
                        'data_offset'   : 0,
                        'gain'          : 1.}

    def _read_from_header(self):
        self._open()
        self.size   = len(self.data)
        self._shape = (self.size//self.nb_channels, int(self.nb_channels))
        self._close()
        return {}

    def allocate(self, shape, data_dtype=None):
        if data_dtype is None:
            data_dtype = self.data_dtype

        if self.is_master:
            self.data = np.memmap(self.file_name, offset=self.data_offset, dtype=data_dtype, mode='w+', shape=shape)
#        comm.Barrier()

        self._read_from_header()
        del self.data

    def read_chunk(self, idx, chunk_size, padding=(0, 0), nodes=None):

        t_start, t_stop = self._get_t_start_t_stop(idx, chunk_size, padding)
        local_shape     = t_stop - t_start

        self._open()
        local_chunk  = self.data[t_start*self.nb_channels:t_stop*self.nb_channels]
        local_chunk  = local_chunk.reshape(local_shape, self.nb_channels)
        self._close()

        if nodes is not None:
            if not np.all(nodes == np.arange(self.nb_channels)):
                local_chunk = np.take(local_chunk, nodes, axis=1)

        return self._scale_data_to_float32(local_chunk)

    def read_chunk_adc(self, idx, chunk_size, padding=(0, 0), nodes=None):
            return self.read_chunk(idx, chunk_size, padding=padding, nodes=nodes)

    def write_chunk(self, time, data):
        self._open(mode='r+')

        data = self._unscale_data_from_float32(data)
        data = data.ravel()
        self.data[self.nb_channels*time:self.nb_channels*time+len(data)] = data
        self._close()


    def _open(self, mode='r'):
        self.data = np.memmap(self.file_name, offset=self.data_offset, dtype=self.data_dtype, mode=mode)

    def _close(self):
        self.data = None


# Cell
from numpy.lib.format import open_memmap

class NumpyFile(RawBinaryFile):

    description    = "numpy"
    extension      = [".npy"]
    parallel_write = True
    is_writable    = True

    _required_fields = {'sampling_rate' : float}

    _default_values  = {'dtype_offset'  : 'auto',
                        'gain'          : 1.}

    def _read_from_header(self):

        header = {}

        self._open()
        self.size = self.data.shape

        if self.size[0] > self.size[1]:
            self.time_axis = 0
            self._shape = (self.size[0], self.size[1])
        else:
            self.time_axis = 1
            self._shape = (self.size[1], self.size[0])

        header['nb_channels'] = self._shape[1]
        header['data_dtype']  = self.data.dtype
        self.size             = len(self.data)
        self._close()

        return header


    def read_chunk(self, idx, chunk_size, padding=(0, 0), nodes=None):

        self._open()

        t_start, t_stop = self._get_t_start_t_stop(idx, chunk_size, padding)

        if self.time_axis == 0:
            local_chunk  = self.data[t_start:t_stop, :].copy()
        elif self.time_axis == 1:
            local_chunk  = self.data[:, t_start:t_stop].copy().T
        self._close()

        if nodes is not None:
            if not np.all(nodes == np.arange(self.nb_channels)):
                local_chunk = np.take(local_chunk, nodes, axis=1)

        return self._scale_data_to_float32(local_chunk)

    def read_chunk_adc(self, idx, chunk_size, padding=(0, 0), nodes=None):
        return self.read_chunk(idx, chunk_size, padding=padding, nodes=nodes)

    def write_chunk(self, time, data):
        self._open(mode='r+')
        data = self._unscale_data_from_float32(data)
        if self.time_axis == 0:
            self.data[time:time+len(data)] = data
        elif self.time_axis == 1:
            self.data[:, time:time+len(data)] = data.T
        self._close()


    def _open(self, mode='r'):
        self.data = open_memmap(self.file_name, mode=mode)


    def _close(self):
        self.data = None

# Cell
def load_all_data(datafile:DataFile):
    """Read all the data contained by a file. For rhd and hdf5, correspond to the ephy channels. To read the ADC
    data, see `load_all_data_adc`"""
    datafile.open()
    if isinstance(datafile, RHDFile):
        chunk_size = 1800960
    else:
        chunk_size =  datafile.duration
    n_chunks, _ = datafile.analyze(chunk_size)
    data = np.zeros((datafile.duration, datafile._shape[1]))
    print("Loading the data... "+str(round(0,2))+"%    ",end='\n',flush=True)
    for idx in range(n_chunks):
        data_tmp, t_offset = datafile.get_data(idx, chunk_size)
        data[t_offset:t_offset+len(data_tmp)] = data_tmp
        print("Loading the data... "+str(round(100*(idx+1)/n_chunks,2))+"%    ",end='\n',flush=True)
    print("Loading the data... "+str(round(100,2))+"%    ",end='\n',flush=True)
    datafile.close()
    return data

def load_all_data_adc(datafile:DataFile, channel_idx=0):
    """Read all the data contained by a file. For rhd and hdf5, correspond to the adc channels. To read the ephy
    data, see `load_all_data`"""
    datafile.open()
    if isinstance(datafile, RHDFile):
        chunk_size = 1800960
    else:
        chunk_size =  datafile.duration
    n_chunks, _ = datafile.analyze(chunk_size)
    data = np.zeros(datafile.duration)
    print("Loading the data... "+str(round(0,2))+"%    ",end='\n',flush=True)
    for idx in range(n_chunks):
        data_tmp, t_offset = datafile.get_data_adc(idx, chunk_size)
        if data_tmp.ndim == 2:
            data_tmp = data_tmp[:,channel_idx]
        data[t_offset:t_offset+len(data_tmp)] = data_tmp
        print("Loading the data... "+str(round(100*(idx+1)/n_chunks,2))+"%    ",end='\n',flush=True)
    print("Loading the data... "+str(round(100,2))+"%    ",end='\n',flush=True)
    datafile.close()
    return data

def load_all_data_dig_in(datafile:DataFile, channel_idx=0):
    """Read all the data contained by a file. For rhd and hdf5, correspond to the adc channels. To read the ephy
    data, see `load_all_data`"""
    datafile.open()
    if isinstance(datafile, RHDFile):
        chunk_size = 1800960
    else:
        chunk_size =  datafile.duration
    n_chunks, _ = datafile.analyze(chunk_size)
    data = np.zeros(datafile.duration, dtype=float)
    print("Loading the data... "+str(round(0,2))+"%    ",end='\r',flush=True)
    for idx in range(n_chunks):
        data_tmp, t_offset = datafile.get_data_dig_in(idx, chunk_size)
        if data_tmp.ndim == 2:
            data_tmp = data_tmp[:,channel_idx]
        data[t_offset:t_offset+len(data_tmp)] = data_tmp
        print("Loading the data... "+str(round(100*(idx+1)/n_chunks,2))+"%    ",end='\n',flush=True)
    print("Loading the data... "+str(round(100,2))+"%    ",end='\n',flush=True)
    datafile.close()
    return data

def load_all_data_both(datafile:DataFile):
    """Read all the data contained by a file. For rhd and hdf5, correspond to the adc channels. To read the ephy
    data, see `load_all_data`"""
    datafile.open()
    if isinstance(datafile, RHDFile):
        chunk_size = 1800960
    else:
        chunk_size =  datafile.duration
    n_chunks, _ = datafile.analyze(chunk_size)
    data_adc = np.zeros(datafile.duration)
    data     = np.zeros((datafile.duration, datafile._shape[1]))
    print("Loading the data... "+str(round(0,2))+"%    ",end='\n',flush=True)
    for idx in range(n_chunks):
        data_tmp, data_tmp_adc, t_offset = datafile.get_data_both(idx, chunk_size)
        data[t_offset:t_offset+len(data_tmp)] = data_tmp
        if data_tmp_adc.ndim == 2:
            data_tmp_adc = data_tmp_adc[:,0]
        data_adc[t_offset:t_offset+len(data_tmp)] = data_tmp_adc
        print("Loading the data... "+str(round(100*(idx+1)/n_chunks,2))+"%    ",end='\n',flush=True)
    print("Loading the data... "+str(round(100,2))+"%    ",end='\n',flush=True)
    datafile.close()
    return data, data_adc

def export_adc_raw(datafile:DataFile, output_fn="", channel_idx=0):
    """Exports a datafile adc channel to a single raw binary file. Useful to reduce disk usage after that
    spike sorting is done."""
    data = load_all_data_adc(datafile, channel_idx=channel_idx)
    if output_fn=="":
        raw_fn = os.path.splitext(datafile.file_name)[0]+".dat"
    else:
        raw_fn = os.path.split(datafile.file_name)[0]+"/"+output_fn
    param_d = {'sampling_rate': datafile.sampling_rate,
               'data_dtype': 'uint16',
               'gain': 0.195,
               'nb_channels': 1,
               'dtype_offset': 32768}
    raw_file = RawBinaryFile(raw_fn, param_d, is_empty=True)
    raw_file.allocate(datafile.shape[0])
    raw_file.set_data(0, data)
    raw_file.close()

def export_dig_in_raw(datafile:DataFile, output_fn="", channel_idx=0):
    """Exports a datafile adc channel to a single raw binary file. Useful to reduce disk usage after that
    spike sorting is done."""
    data = load_all_data_dig_in(datafile, channel_idx=channel_idx)
    if output_fn=="":
        raw_fn = os.path.splitext(datafile.file_name)[0]+".dat"
    else:
        raw_fn = os.path.split(datafile.file_name)[0]+"/"+output_fn
    param_d = {'sampling_rate': datafile.sampling_rate,
               'data_dtype': 'uint8',
               'gain': 1,
               'nb_channels': 1,
               'dtype_offset': 127}
    raw_file = RawBinaryFile(raw_fn, param_d, is_empty=True)
    raw_file.allocate(datafile.shape[0])
    raw_file.set_data(0, data)
    raw_file.close()

def export_raw(datafile:DataFile, output_fn=""):
    """Exports a datafile adc channel to a single raw binary file. Useful to reduce disk usage after that
    spike sorting is done."""
    data = load_all_data(datafile)
    if output_fn=="":
        raw_fn = os.path.splitext(datafile.file_name)[0]+".dat"
    else:
        raw_fn = os.path.split(datafile.file_name)[0]+"/"+output_fn
    param_d = datafile.get_description()
    raw_file = RawBinaryFile(raw_fn, param_d, is_empty=True)
    raw_file.allocate(datafile.shape)
    raw_file.set_data(0, data)
    raw_file.close()

def export_both_raw(datafile:DataFile):
    """Exports a both raw data, adc and ephy."""
    data, data_adc = load_all_data_both(datafile)
    raw_fn = os.path.splitext(datafile.file_name)[0]+".dat"
    param_d = {'sampling_rate': datafile.sampling_rate,
               'data_dtype': 'uint16',
               'gain': 0.195,
               'nb_channels': 1,
               'dtype_offset': 32768}
    raw_file = RawBinaryFile(raw_fn, param_d, is_empty=True)
    raw_file.allocate(datafile.shape[0])
    raw_file.set_data(0, data_adc)
    raw_file.close()

    os.rename(raw_fn, os.path.splitext(datafile.file_name)[0]+".data")

    param_d = datafile.get_description()
    raw_file = RawBinaryFile(raw_fn, param_d, is_empty=True)
    raw_file.allocate(datafile.shape)
    raw_file.set_data(0, data)
    raw_file.close()


def load_adc_raw(filepath, sampling_rate=30000):
    """Loads adc raw data, in the format exported by `export_adc_raw`"""
    param_d = {'sampling_rate': sampling_rate,
               'data_dtype': 'uint16',
               'gain': 0.195,
               'nb_channels': 1,
               'dtype_offset': 32768}
    raw_file = RawBinaryFile(filepath, param_d)
    return load_all_data_adc(raw_file)

def load_digin_raw(filepath, sampling_rate=30000):
    """Loads adc raw data, in the format exported by `export_adc_raw`"""
    param_d = {'sampling_rate': sampling_rate,
               'data_dtype': 'uint8',
               'gain': 1,
               'nb_channels': 1,
               'dtype_offset': 127}
    raw_file = RawBinaryFile(filepath, param_d)
    return load_all_data_adc(raw_file)

def load_sync_raw(filepath, sampling_rate=10000):
    """Loads the sync files made by labview for Asari Lab 2P setup"""
    param_d = {'sampling_rate': sampling_rate,
               'data_dtype': '>d',
               'gain': 1,
               'nb_channels': 1,
               'dtype_offset': 0}
    raw_file = RawBinaryFile(filepath, param_d)
    return load_all_data_adc(raw_file)