from __future__ import print_function
from __future__ import absolute_import

import string
import os
import weakref
import re
import glob
import numpy as np
import pandas as pds
from pysat import data_dir as data_dir

class Files(object):
    """Maintains collection of files for instrument object.
    
    Uses the list_files functions for each specific instrument
    to create an ordered collection of files in time. Used by
    instrument object to load the correct files. Files also
    contains helper methods for determining the presence of
    new files and creating an ordered list of files.
      
    Attributes
    ----------
    base_path : string
        path to .pysat directory in user home
    start_date : datetime
        date of first file, used as default start bound for instrument
        object
    stop_date : datetime
        date of last file, used as default stop bound for instrument
        object
    data_path : string
        path to the directory containing instrument files,
        top_dir/platform/name/tag/
        
    Note
    ----
    User should generally use the interface provided by a pysat.Instrument
    instance. Exceptions are the classmethod from_os, provided to assist
    in generating the appropriate output for an instrument routine.
    
    Examples
    --------
    ::
    
        # convenient file access
        inst = pysat.Instrument(platform=platform, name=name, tag=tag)
        # first file
        inst.files[0]
        
        # files from start up to stop (exclusive on stop)
        start = pysat.datetime(2009,1,1)
        stop = pysat.datetime(2009,1,3)
        print(vefi.files[start:stop])
        
        # files for date
        print(vefi.files[start])
        
        # files by slicing
        print(vefi.files[0:4])
        
        # get a list of new files
        # new files are those that weren't present the last time
        # a given instrument's file list was stored
        new_files = vefi.files.get_new()
        
        # search pysat appropriate directory for instrument files and
        # update Files instance, knowledge not written to disk.
        vefi.files.refresh()
        
        # search pysat appropriate directory for files and store new list
        vefi.files.refresh(store=True)
        # running get_new will now return an empty list until
        # additional files are introduced
        
        
    """
        
    def __init__(self, sat):

        self._sat = weakref.proxy(sat)
        self.base_path = os.path.join(os.getenv('HOME'), '.pysat')
        self.start_date = None
        self.stop_date = None
        self.files = pds.Series(None)
        self.data_path = os.path.join(data_dir, self._sat.platform, 
                                      self._sat.name, self._sat.tag)
        
        if self._sat.platform != '':
            info = self._load()
            if info is not False:
                self._attach_files(info)
            else:
                print("pysat is searching for the requested instrument's files.")
                # couldn't find stored info, load file list and then store
                info = self._sat._list_rtn(tag=self._sat.tag, data_path=self.data_path)
                if not info.empty:
                    info = self._remove_data_dir_path(info)	
                    self._attach_files(info)
                    self._store()

    def _attach_files(self, files_info):
        """Attaches info returned by instrument list_files routine to Instrument object."""

        if (len(files_info.index.unique()) != len(files_info)):
            print('Duplicate datetimes ', files_info.index.get_duplicates())
            raise ValueError('List of files must have unique datetimes.')

        self.files = files_info.sort_index()
        date = files_info.index[0]
        self.start_date = pds.datetime(date.year, date.month, date.day)
        date = files_info.index[-1]
        self.stop_date = pds.datetime(date.year, date.month, date.day)

    def _store(self, dir=None):
        """Store currently loaded filelist for instrument onto filesystem"""
        name = ''.join((self._sat.platform,'_',self._sat.name,'_',self._sat.tag,
                        '_stored_file_info.txt'))
        if dir is None:
            name = os.path.join(self.base_path, name)
        else:
            name = os.path.join(dir, name)
        try:
            self.files.to_csv(name, date_format='%Y-%m-%d %H:%M:%S.%f')
            return True
        except IOError:
            return False	

    def _load(self):
        """Load stored filelist and return as Pandas Series"""
        fname = ''.join((self._sat.platform,'_',self._sat.name,'_',
                        self._sat.tag, '_stored_file_info.txt'))
        fname = os.path.join(self.base_path, fname)
        try:
            data = pds.Series.from_csv(fname, index_col=0)
        except IOError:
            return False
        else:
            return data

    def refresh(self, store=False):
        """Refresh loaded instrument filelist by searching filesystem.
        
        Searches pysat provided path, pysat_data_dir/platform/name/tag/, 
        where pysat_data_dir is set by pysat.utils.set_data_dir(path=path).
        
        Parameters
        ----------
        store : boolean
            set True to store loaded file names into .pysat directory
            
        """
        info = self._sat._list_rtn(tag=self._sat.tag, data_path=self.data_path)
        info = self._remove_data_dir_path(info)
        self._attach_files(info)
        if store:
            self._store()

    def get_new(self):
        """List all new files since last time list was stored.
        
        pysat stores filenames in the user_home/.pysat directory. Returns
        a list of all new fileanmes since the last store. Filenames
        are stored if update_files is True at instrument object level and
        if files.refresh(store=True) is called.
        
        Returns
        -------
            pandas Series of filenames
            False if no filenames
        """
        stored_info = self._load()
        if stored_info is not False:
            new_info = self._sat._list_rtn(tag = self._sat.tag, data_path=self.data_path)
            new_info = self._remove_data_dir_path(new_info)
            new_files = new_info[~new_info.isin(stored_info) ]
            return new_files
        else:
            print('No previously stored files that we may compare to.')
            return False

    def get_index(self, fname):
        """Return index for a given filename. 
        
        Parameters
        ----------
        fname : string
            filename
        
        Note
        ----
        If fname not found in the file information already attached 
        to the instrument.files instance, then a files.refresh() call 
        is made.
        
        """
        idx, = np.where(fname == self.files)
        if len(idx) == 0:
            # filename not in index, try reloading files from disk
            self.refresh()
            print (fname, self.files)
            idx, = np.where(fname == self.files)
            if len(idx) == 0:
                raise IOError('Could not find supplied file on disk')
        return idx

    # convert this to a normal get so files[in:in2] gives the same as requested here
    # support slicing via date and index
    # filename is inclusive slicing, date and index are normal non-inclusive end point

    def __getitem__(self, key):
        if isinstance(key, slice):
            try:
                out = self.files.ix[key]
            except IndexError:
                raise IndexError('Date requested outside file bounds.')
            if isinstance(key.start, pds.datetime):
                # enforce exclusive slicing on datetime
                if len(out) > 1:
                    if out.index[-1] >= key.stop:
                        return out[:-1]
                    else:
                        return out
                elif len(out) == 1:
                    if out.index[0] >= key.stop:
                        return pds.Series([])
                    else:
                        return out
                else:
                    return out
            else:
                # not a datetime
                return out
        else:
            return self.files.ix[key]
            #raise ValueError('Not implemented yet.')         
        #if isinstance(key, tuple):
        #    if len(key) == 2:
        #        start = key[0]
        #        end = key[1]
        #    else:
        #        raise ValueError('Must input 2 and only 2 items/iterables')
                
    def get_file_array(self, start, end):
        """Return a list of filenames between and including start and end.
        
        Parameters
        ----------
            start: array_like or single string 
                filenames for start of returned filelist
            stop: array_like or single string
                filenames inclusive end of list
                
        Returns
        -------
            list of filenames between and including start and end over all
            intervals. 
            
        """
        if hasattr(start, '__iter__') & hasattr(end, '__iter__'):
            files = []
            for (sta,stp) in zip(start, end):
                id1 = self.get_index(sta)
                id2 = self.get_index(stp)
                files.extend(self.files.iloc[id1[0] : id2[0]+1])
        elif hasattr(start, '__iter__') | hasattr(end, '__iter__'):
            raise ValueError('Either both or none of the inputs need to be iterable')
        else:
            id1 = self.get_index(start)
            id2 = self.get_index(end)
            files = self.files[id1[0]:id2[0]+1].to_list()
        return files

    def _remove_data_dir_path(self, inp=None):
        # import string
        """Remove the data directory path from filenames"""
        # need to add a check in here to make sure data_dir path is actually in
        # the filename
        if inp is not None:
            split_str = os.path.join(self.data_path, '')
            return inp.apply(lambda x: x.split(split_str)[-1])
            # match = os.path.join(self.data_path,'')
            # num = len(match)
            # return inp.apply(lambda x: x[num:])
        
    @classmethod    
    def from_os(cls, data_path=None, format_str=None, 
                two_digit_year_break=None):
        """
        Produces a list of files and and formats it for Files class.
        
        Parameters
        ----------
            data_path : string
                Top level directory to search files for. This directory
                is provided by pysat to the instrument_module.list_files
                functions as data_path.
            format_string : string with python format codes
                Provides the naming pattern of the instrument files and the 
                locations of date information so an ordered list may be 
                produced.
            two_digit_year_break : int
                If filenames only store two digits for the year, then
                '1900' will be added for years >= two_digit_year_break,
                and '2000' will be added for years < two_digit_year_break.
                
        Note
        ----
        Does not produce a Files instance, but the proper output
        from instrument_module.list_files method.
        
        """
        import collections
        
        from pysat.utils import create_datetime_index
        
        if format_str is None:
            raise ValueError("Must supply a filename template (format_str).")
        if data_path is None:
            raise ValueError("Must supply instrument directory path (dir_path)")
        
        # parse format string to figure out the search string to use
        # to identify files in the filesystem
        search_str = ''
        form = string.Formatter()
        keys = []
        snips = []
        length = []
        stored = collections.OrderedDict()
        stored['year'] = []; stored['month'] = []; stored['day'] = [];
        stored['hour'] = []; stored['min'] = []; stored['sec'] = [];
        for snip in form.parse(format_str):
            search_str += snip[0]
            snips.append(snip[0])
            if snip[1] is not None:
                keys.append(snip[1])
                search_str += '*'
                # try and determine formatting width
                temp = re.findall(r'\d+', snip[2])
                if temp:
                    # there are items, try and grab width
                    for i in temp:
                        if i != 0:
                            length.append(int(i))
                            break
                else:
                    raise ValueError("Couldn't determine formatting width")

        abs_search_str = os.path.join(data_path, search_str)
        files = glob.glob(abs_search_str)   
        
        # we have a list of files, now we need to extract the date information        
        # code below works, but only if the size of file string 
        # remains unchanged
        
        # determine the loactaions the date information in a filename is stored
        # use these indices to slice out date from loaded filenames
        # test_str = format_str.format(**periods)
        if len(files) > 0:  
            idx = 0
            begin_key = []
            end_key = []
            for i,snip in enumerate(snips):
                idx += len(snip)
                if i < (len(length)):
                    begin_key.append(idx)
                    idx += length[i]
                    end_key.append(idx)
            max_len = idx
            # setting up negative indexing to pick out filenames
            key_str_idx = [np.array(begin_key, dtype=int) - max_len, 
                            np.array(end_key, dtype=int) - max_len]
            # need to parse out dates for datetime index
            for i,temp in enumerate(files):
                for j,key in enumerate(keys):
                    val = temp[key_str_idx[0][j]:key_str_idx[1][j]]
                    stored[key].append(val)
            # convert to numpy arrays
            for key in stored.keys():
                stored[key] = np.array(stored[key]).astype(int)
                if len(stored[key]) == 0:
                    stored[key]=None
            # deal with the possibility of two digit years
            # years above or equal to break are considered to be 1900+
            # years below break are considered to be 2000+
            if two_digit_year_break is not None:
                idx, = np.where(stored['year'] >= two_digit_year_break)
                stored['year'][idx] = stored['year'][idx] + 1900
                idx, = np.where(stored['year'] < two_digit_year_break)
                stored['year'][idx] = stored['year'][idx] + 2000 
            # need to sort the information for things to work
            rec_arr = [stored[key] for key in keys]
            rec_arr.append(files)
            # sort all arrays
            val_keys = keys+['files']
            rec_arr = np.rec.fromarrays(rec_arr, names=val_keys)
            rec_arr.sort(order=val_keys, axis=0)
            # pull out sorted info
            for key in keys:
                stored[key] = rec_arr[key]
            files = rec_arr['files']
            # add hour and minute information to 'sec'
            if stored['sec'] is None:
                stored['sec'] = np.zeros(len(files))                
            if stored['hour'] is not None:
                stored['sec'] += 3600*stored['hour']
            if stored['min'] is not None:
                stored['sec'] += 60*stored['min']
            
            index = create_datetime_index(year=stored['year'], month=stored['month'], 
                                    day=stored['day'], uts=stored['sec'])

            return pds.Series(files, index=index)
        else:
            print ("Unable to find any files. If you have the necessary files "+
                   "please check pysat settings and file locations.")
            return pds.Series(None) 

