'''
This module contains handler classes for DigitalGlobe imagery.  Includes XML and
PVL (IMD, etc) meta data handling, image date retrieval (UTC to local via
tzwhere), and spectral processing built on top of the GeoImage class
capabilities.
'''


import collections
import datetime
import os
import re
import logging

import ephem
import numpy as np
import pytz
import xmltodict
from pkg_resources import resource_filename
from tzwhere import tzwhere

import tinytools as tt
from base import GeoImage
import constants as const

# Module setup
logger = logging.getLogger(__name__)
# To get access to logging statements from the command line:
# import logging
# logging.basicConfig(level=logging.DEBUG) # or your desired level

class DGImage(GeoImage):
    """ Input can be .TIL, .VRT, OR .TIF.  If .TIL or .VRT, checking is done
    for tiles that belong to the virtual dataset.  If .TIF, then an
    XML or IMD file should exist with the exact same name (sans suffix).  No
    checking is done that the meta file actually belongs with the .TIF.  e.g.
    if an XML file belongs to DN data but surface reflectance data is
    passed, several of this classes methods might explode in bad ways.

    meta data is stored as:
    DGImage.meta_dg [populated by data from DigitalGlobe XML or pvl files -
                     i.e. IMD, RPB, etc.]
    DGImage.meta [populated from GeoImage class __init__]
    DGImage.files
            === Populated by GeoImage ===
            -fin  [file passed in - virtual or otherwise]
            -gdal_file_list  [GetFileList returned from gdal]
            -dfile  [Working target data file (i.e. VRT if TIL passed,
                        otherwise it wil be the same as fin]
            -dfile_tiles  [The tiles of the virtual data set - if the data set
                           isn't virtual, this should be = "files.dfile"]

            === Populated by DGImage ===
            -meta [all meta files, first one is used to populate meta_dg]
            -spectral_data_sets  [Dictionary of Radiance, TOA Reflectance,
                                  and DGAComp as True/False is present/not]
            -dgacomp  [link to DGAComp file]
            -dgacomp_tiles  [The tiles of the virtual data set - if the data set
                            isn't virtual, this should be = "files.dgacomp"]
            -dgacomp_aodmap [link to aop map file generated by dgacomp.  Only
                             present if DGAComp in spectral_data_sets is True]
            -rad  [link to radiance file]
            -rad_tiles  [The tiles of the virtual data set - if the data set
                        isn't virtual, this should be = "files.rad"]
            -toa  [link to toa reflectance file]
            -toa_tiles  [The tiles of the virtual data set - if the data set
                        isn't virtual, this should be = "files.toa"]
    """

    def __init__(self, dg_file_in, derived_dir=None):  #,dg_img_directory
        """Figure out what the main dataset (create a vrt if necessary) and
        set the meta data file/list.
        """
        ### Trigger super __init__ and set the GeoImage stuff
        # ... variables created here:
        # See above for list of "Populated by GeoImage"
        super(DGImage,self).__init__(dg_file_in, derived_dir=derived_dir)
        #########  IMPORTANT !!!!  ##############################
        # DO NOT SUBCLASS WITH THIE SELF REFERENCED SUPER FROM BELOW!
        # INFINITE RECURSION WILL RESULT.  I HAVE THIS IN
        # BECAUSE THIS FIXES AUTORELOAD IN IPYTHON FOR DEVELOPMENT
        # Use the super line above for deployed code
        ##########################################################
        #super(self.__class__,self).__init__(dg_file_in, derived_dir=derived_dir)

        # Get the file name, full file directory, and flist
        ifile = self.files.dfile
        fname = os.path.basename(ifile)
        ifile_base = os.path.splitext(ifile)[0]
        dgbase = fname.split("-")[0]
        fdir = os.path.dirname(ifile)
        flist = os.listdir(fdir)
        flist = [os.path.join(fdir, x) for x in flist]

        ## Populate DG metadata files
        self.files.meta = []
        for e in const.DG_META:
            found_mfiles = tt.files.filter(flist,ifile_base+e,
                                                case_sensitive=False)
            self.files.meta = self.files.meta + found_mfiles

        if not self.files.meta:
            raise ValueError("Failed to find any DigitalGlobe metadata "
                             "files when initializing the DGImage Object.")

        ## Read DG metadata files
        if tt.files.filter(self.files.meta[0], '*.XML', case_sensitive=False):
            # There is an xml file, so lets just use that.
            self._read_dg_dir_meta_xml(self.files.meta[0])
        else:
            # Then there was not an xml so we need to assemble the pvl files.
            self._read_dg_dir_meta_pvl(self.files.meta)

        ## Populate the spectral files for this DGImage
        self._set_dg_spectral_files()

        ## Populate the spectral files for this DGImage
        self._set_dg_meta()

    def _read_dg_dir_meta_xml(self,xml_file):
        # Load the XML dg_meta_file into a dictionary
        with open(xml_file) as fd:
            tmp_dict = xmltodict.parse(fd.read())

        # Filter data types to more easily access members
        tt.bunch.guess_dtype_convert_dict(tmp_dict)

        # Cap everything and filter for stupid pvl to xml differences
        # delivered from the factory
        self._update_dict_differences(tmp_dict)

        # Make a OrderedBunch object for easy command line access
        self.meta_dg = tt.bunch.ordered_bunchify(tmp_dict['ISD'])

    def _read_dg_dir_meta_pvl(self,pvl_file_list):
        # Read dg meta data from the IMD file
        tmp_dict = {'isd':{}}
        add_to_loc = tmp_dict['isd']
        for x in pvl_file_list:
            xend = os.path.splitext(x)[1].split('.')[1].upper()
            add_to_loc[xend]=tt.pvl.read_from_pvl(x)

        # Filter for stupid pvl to xml differences from the factory
        self._update_dict_differences(tmp_dict,'IMAGE_1','IMAGE')

        # Cap everything and filter for stupid pvl to xml differences
        # delivered from the factory
        tt.bunch.guess_dtype_convert_dict(tmp_dict)

        # Make a OrderedBunch object for easy command line access
        self.meta_dg = tt.bunch.ordered_bunchify(tmp_dict['ISD'])

    def _update_dict_differences(self,d,input=None,output=None):
        for x in d.keys():
            if x == input:
                d[output.upper()] = d.pop(x)
                x = output.upper()
            else:
                d[x.upper()] = d.pop(x)
                x = x.upper()

            if isinstance(d[x],dict):
                self._update_dict_differences(d[x],input,output)
            elif isinstance(d[x], collections.OrderedDict):
                self._update_dict_differences(d[x],input,output)
            else:
                pass

    def _set_dg_meta(self):

        # If for some reason the meta object wasn't created, create it
        # This can happen when a class overloads that meta creation routines
        # in the super object.
        try:
            tmp = self.meta
        except AttributeError:
            self.meta = tt.bunch.OrderedBunch()

        self.meta.satid = self.meta_dg.IMD.IMAGE.SATID
        self.meta.bandid = self.meta_dg.IMD.BANDID
        self.meta.catid = self.meta_dg.IMD.IMAGE.CATID

        abscalfactor = []
        effbandwidth = []
        tdilevel = []
        lat = []
        lon = []
        hae = []

        for x in self.meta_dg.IMD:
            if 'BAND_' in x:
                abscalfactor.append(float(self.meta_dg.IMD[x]['ABSCALFACTOR']))
                effbandwidth.append(float(self.meta_dg.IMD[x]['EFFECTIVEBANDWIDTH']))
                tdilevel.append(float(self.meta_dg.IMD[x]['TDILEVEL']))
                lat.append(float(self.meta_dg.IMD[x]['ULLAT']))
                lon.append(float(self.meta_dg.IMD[x]['ULLON']))
                hae.append(float(self.meta_dg.IMD[x]['ULHAE']))

        # Set band data collected from IMD/XML
        self.meta.abscalfactor = abscalfactor
        self.meta.effbandwidth = effbandwidth
        self.meta.tdilevel = tdilevel

        # Set data from constants file
        sat_index = self.meta.satid.upper() + "_" + \
                    self.meta.bandid.upper()
        self.meta.band_names = const.DG_BAND_NAMES[sat_index]
        self.meta.band_centers = const.DG_WEIGHTED_BAND_CENTERS[sat_index]

        # Longitude will be +/-180, Latitude will be +/-90, both with
        # precision of eight decimal places
        # HAE is in meters above WGS84 ellipsoid
        #ToDo - move reported point to the center of the image
        self.meta.latlonhae = (lat[0], lon[0], hae[0])

        # Date/time of collection from EarliestAcqTime
        # Don't use "map_projected_product" time for this to work on 1b files
        # dtstr = self.meta_dg.IMD.MAP_PROJECTED_PRODUCT.EARLIESTACQTIME
        dtstr = self.meta_dg.IMD.IMAGE.FIRSTLINETIME
        time_vars = parse_dg_time_str(dtstr)
        tzinfo_utc = pytz.timezone('UTC')
        self.meta.img_datetime_obj_utc = datetime.datetime(
            *time_vars, tzinfo=tzinfo_utc)

    def _set_dg_spectral_files(self,path=None):
        # Get the search directory
        data_file = self.files.dfile

        if (not path) & hasattr(self,'derived_dir'):
            path=self.derived_dir

        (self.files.rad,self.files.rad_tiles) = \
            self._get_dg_spectral_files(data_file,
                                        const.DG_SPEC['RAD_IMGS'],
                                        path=path)

        (self.files.toa,self.files.toa_tiles) = \
            self._get_dg_spectral_files(data_file,
                                        const.DG_SPEC['TOA_IMGS'],
                                        path=path)

        (self.files.dgacomp,self.files.dgacomp_tiles) = \
            self._get_dg_spectral_files(data_file,
                                        const.DG_SPEC['DGACOMP_IMGS'],
                                        path=path)

        (self.files.dgacomp_aodmap,tmp_dump) = \
            self._get_dg_spectral_files(data_file,
                                        const.DG_SPEC['DGACOMP_AOD'],
                                        path=path)

        ### ToDo - Add DGACOMP_OTHER
        # Find all DGAComp files not in dgacomp or dgacomp_aodmap
        if self.files.dgacomp is not None:
            search_dirs = []
            if path:
                search_dirs.append(path)
            else:
                search_dirs.append(os.path.dirname(data_file))

            for search_dir in search_dirs:
                tmp_files = tt.files.search(search_dir, ['*DG-AComp*', '*DGAComp*'],
                                            depth=2, case_sensitive=False)

            dgacomp_tracked = []
            dgacomp_tracked.append(self.files.dgacomp)
            dgacomp_tracked.extend(self.files.dgacomp_tiles)
            dgacomp_tracked.append(self.files.dgacomp_aodmap)

            [tmp_files.remove(x) for x in dgacomp_tracked]

            self.files.dgacomp_other = tmp_files
        else:
            self.files.dgacomp_other = None


    def _get_dg_spectral_files(self, dfile, search_endings, path=None):

        sestar = [os.path.splitext(os.path.basename(dfile))[0] + x + '.' + '*'
                  for x in search_endings]

        search_dirs = []
        if path:
            search_dirs.append(path)
        else:
            search_dirs.append(os.path.dirname(dfile))

        for search_dir in search_dirs:
            tmp_files = tt.files.search(search_dir, sestar, depth=2)

        if not tmp_files:
            return (None,[])

        if not (len(tmp_files) <= 2):
            raise ValueError("There should be no more than two files "
                             "returned for the supported file types (TIL, "
                             "VRT, TIF, ENVI).")
        if len(tmp_files) == 2:
            tif_file = tt.files.filter(tmp_files, '*.TIF')
            if tif_file:
                tmp_files.remove(tif_file[0])

        return self._get_file_and_tiles(tmp_files[0])

    def populate_img_datetime_obj_local(self,w=None):
        ## Get the time/date at the image collect location
        logger.debug('')
        logger.debug('Adding self.img_datetime_obj_local to obj data members.')
        logger.debug('')
        try:
            self.img_tz = w.tzNameAt(self.meta.latlonhae[0],
                                     self.meta.latlonhae[1])
        except:
            ###
            # Loading a tzwhere intstance from scratch...
            # This could be theoretically changed by including a '
            # timezone database or a module that wraps one instead of '
            # the json file.'
            ###
            #logger.debug('')
            logger.debug('Opening a tzwhere instance for access to the json file.  '
                  'This is slow!  You can pass in a pregenerated instance '
                  'of the json database to the variable w in order to speed '
                  'up the processing of multiple images.  This is done by '
                  'importing tzwhere using "from tzwhere import tzwhere" and '
                  'then creating the instance using "w=tzwhere.tzwhere()"  '
                  'See notes in code for possible development to improve '
                  'speed if needed.')
            logger.debug('')
            w = tzwhere.tzwhere()
            self.meta.img_tz = w.tzNameAt(
                                            self.meta.latlonhae[0],
                                            self.meta.latlonhae[1])
            if not self.meta.img_tz:
                logger.debug('')
                logger.debug('No time zone code returned, this location may be in '
                      'the middle of the ocean.  Defaulting to a simple '
                      'latitude based time zone calculation.')
                offset = self._calc_gmtoffset(self.meta.latlonhae[1])
                #print offset
                self.meta.img_tz = 'Etc/GMT' + str(offset)

        tzinfo_img = pytz.timezone(self.meta.img_tz)
        self.meta.img_datetime_obj_local = \
            self.meta.img_datetime_obj_utc.astimezone(tzinfo_img)

    def _calc_gmtoffset(self,l):
        """Calculate the general time zone offset based on latitude alone."""
        offset=int(((l+180+7.5)//15)-12)
        # There is no such thing as a -12 time zone offset.  By the math,
        # the difference offsets go from -12.5 to 12.5, but by convention,
        # the fractional offsets are convereted to 12.  So, the offset range
        # is [-11,12] inclusive.  There are areas that use timezones of +13
        # and +14 in the pacific, but these should be taken care of by the
        # tzwhere call above.
        if offset==-12:
            return 12
        else:
            return offset

    # ToDo - change to passing **kwargs to super.  Before that happens, shoudl
    # probably figure out how to merge the docstring from super with below.
    def get_data(self, component = None,
                       bands = None,
                       window = None,
                       buffer = None,
                       geom=None,
                       mask = False,
                       mask_all_touched=False,
                       virtual = False,
                       stype = None):
        """Get image data with ability to output a data frame or request
        keyword arguments to the parent get_data function.  These include
        requesting certain bands and specific components.  This function will
        hijack any bands request and look to see if the request matches
        band aliases defined in const before passing a converted list of
        ints to super."""

        # Set spectral retrival if requested
        if stype:
            raise NotImplementedError("stype handling in get_data is not " \
                                      "implemented yet.  Please use the " \
                                      "dedicated methods.")

        # Set satelite index to query const dictionaries
        # Set initial band names that might be updated below
        sat_index = self.meta.satid.upper() + "_" + \
                    self.meta.bandid.upper()
        bnames = const.DG_BAND_NAMES[sat_index]

        # handle options request for individual bands
        if bands:
            band_nums = get_alias_band_numbers(sat_index,bands)
            band_nums = [x for x in band_nums if x is not None]
            bnames = [bnames[x-1] for x in band_nums if x is not None]
        else:
            band_nums = get_alias_band_numbers(sat_index,bnames)

        if len(band_nums) is 0:
            raise ValueError("No band values were found in the requested " \
                             "alias.")

        # Call super with keywords passed in and/or convereted above
        data = super(self.__class__,self).get_data(component = component,
                                           bands = band_nums,
                                           window = window,
                                           buffer = buffer,
                                           geom=geom,
                                           mask = mask,
                                           mask_all_touched=mask_all_touched,
                                           virtual = virtual)

        return data

    def get_data_as_at_sensor_rad(self,component=None):
        """Read data from sensor as at sensor radiance.  The returned values
        are in W/(m^2*sr*nm).  The values are calculated with known
        gain/offset values pull from the geoio.constants file as provided by
        the DigitalGlobe calibration team.  The absolute calibration factor
        and effective bandwidths are pull from the IMD files for each image."""

        # Set satellite index to look up cal factors
        sat_index = self.meta.satid.upper() + "_" + \
                    self.meta.bandid.upper()

        # Set scale for at sensor radiance
        # Eq is:
        # L = DN * (ACF/EBW) * (2 - Gain) - Offset
        # Above equation is retired as of Jan. 2016, the equation to use with
        # the most recent cal coefficients is:
        # L = GAIN * DN * (ACF/EBW) + Offset
        # ACF abscal factor from meta data
        # EBW effectiveBandwidth from meta data
        # Gain provided by abscal from const
        # Offset provided by abscal from const
        num = np.asarray(self.meta.abscalfactor) # Should be nbands length
        den = np.asarray(self.meta.effbandwidth)  # Should be nbands length
        gain = np.asarray(const.DG_ABSCAL_GAIN[sat_index])
        scale = (num/den)*(gain)

        # Shape for easy multiple
        scale = scale[:, np.newaxis, np.newaxis]
        offset = np.asarray(const.DG_ABSCAL_OFFSET[sat_index])
        offset = offset[:, np.newaxis, np.newaxis]

        # Pull raw data
        if component is not None:
            data = self.get_data(component=component)
        else:
            data = self.get_data()

        # Return scaled data
        out = (data*scale.astype('float32'))+offset.astype('float32')

        return out

    def get_data_as_toa_ref(self,component=None):
        """Get data in a numpy array as top-of-atmosphere reflectance.
        Output is in scaled reflectance 0-10,000.
        Input:  data (numpy array in bands,lines,samples)
                absCalFactor (int or list of nbands)
                effBandWidth (int or list of nbands)
                l_band (list of nbands)
                e_sun (list of nbands)
                d_es (int/float)
                theta_s (int/float)
        Output: image data in TOA reflectance
        """

        # Set satellite index to look up cal factors
        sat_index = self.meta.satid.upper() + "_" + \
                    self.meta.bandid.upper()

        # Set scale for at sensor radiance
        # Eq is:
        # L = DN * (ACF/EBW) * (2 - Gain) - Offset
        # Above equation is retired as of Jan. 2016, the equation to use with
        # the most recent cal coefficients is:
        # L = GAIN * DN * (ACF/EBW) + Offset
        # ACF abscal factor from meta data
        # EBW effectiveBandwidth from meta data
        # Gain provided by abscal from const
        # Offset provided by abscal from const
        num = np.asarray(self.meta.abscalfactor) # Should be nbands length
        den = np.asarray(self.meta.effbandwidth)  # Should be nbands length
        gain = np.asarray(const.DG_ABSCAL_GAIN[sat_index])
        scale = (num/den)*(gain)

        # Shape for easy multiple
        scale = scale[:, np.newaxis, np.newaxis]
        offset = np.asarray(const.DG_ABSCAL_OFFSET[sat_index])
        offset = offset[:, np.newaxis, np.newaxis]

        ## Read Esun for the bands of this satellite from the const file
        e_sun_index = self.meta.satid.upper() + "_" + \
                      self.meta.bandid.upper()
        e_sun = const.DG_ESUN[e_sun_index]

        ## Calculate the generic earth_sun distance
        # For a test image over Longmont, the code below produces:
        # 1.0161819458007812
        # sun = ephem.Sun()
        # sun.compute(self.img_datetime_obj)
        # d_es = sun.earth_distance

        ## Calculate the earth_sun distance for the image location
        # This is probably overkill, but... whatever, it was easy.  =)
        # For a test image over Longmont, the code below produces:
        # 1.0161415338516235
        sun = ephem.Sun()
        img_obs = ephem.Observer()
        img_obs.lon = self.meta.latlonhae[1]
        img_obs.lat = self.meta.latlonhae[0]
        img_obs.elevation = self.meta.latlonhae[2]
        img_obs.date = self.meta.img_datetime_obj_utc
        sun.compute(img_obs)
        d_es = sun.earth_distance

        ## Pull sun elevation from the image metadata
        #theta_s can be zenith or elevation - the calc below will us either
        # a cos or s in respectively
        #theta_s = float(self.meta_dg.IMD.IMAGE.MEANSUNEL)
        theta_s = 90-float(self.meta_dg.IMD.IMAGE.MEANSUNEL)

        # Set up the solar and geometry parameters
        d = np.repeat(d_es, self.shape[0]) # earth-sun distance
        e = np.asarray(e_sun)          # e_sun for each band
        te = np.repeat(theta_s, self.shape[0])  # sun elevation

        # Perform the TOA reflectance calculation
        #scale2 = (d**2*np.pi)/(e*np.sin(np.deg2rad(te)))
        scale2 = (d ** 2 * np.pi) / (e * np.cos(np.deg2rad(te)))

        # Shape for easy multiple
        scale2 = scale2[:, np.newaxis, np.newaxis]

        # Pull raw data
        if component is not None:
            data = self.get_data(component=component)
        else:
            data = self.get_data()

        # Set data types for output
        # scale = scale.astype('float32')
        # offset = offset.astype('float32')
        # scale2 = scale2.astype('float32')

        # Set data types for output
        scale = scale.astype('float16')
        offset = offset.astype('float16')
        scale2 = scale2.astype('float16')

        # Return scaled data
        return ((((data*scale)+offset)*scale2)*10000).astype('int16')

    def create_at_sensor_rad_files(self,path=None,components=True):
        if hasattr(self,'derived_dir'):
            path=self.derived_dir
        #Convert to files, each component if requested.
        if not components:
            # Create single file from image data
            fl = list(os.path.splitext(self.files.dfile))
            if path:
                bn = os.path.basename(fl[0])
                fl[0] = os.path.join(path, bn)
            new_fname = fl[0] + const.DG_SPEC['RAD_IMGS'][0] + fl[1]
            data = self.get_data_as_at_sensor_rad()
            self.write_img_like_this(new_fname,data)
            # Add to dg_img_rad
            self.dg_img_rad = new_fname
        elif components:
            if (self.files.dfile_tiles[0] == self.files.dfile):
                logger.debug("This data set does not appear to have "
                      "componenets, reverting to the main file.")
                self.create_at_sensor_rad_files(components=False,path=path)
            else:
                # Create each component file
                flist_for_vrt = []
                for yi,yv in enumerate(self.files.dfile_tiles):
                    x = GeoImage(yv)
                    fl = list(os.path.splitext(x.files.dfile))
                    if path:
                        bn = os.path.basename(fl[0])
                        fl[0] = os.path.join(path, bn)
                    new_fname = fl[0] + const.DG_SPEC['RAD_IMGS'][0] + fl[1]
                    flist_for_vrt.append(new_fname)
                    data = self.get_data_as_at_sensor_rad(component=yi+1)
                    x.write_img_like_this(new_fname,data)
                    x=None
                # Create the vrt
                tmp = list(os.path.splitext(self.files.dfile))
                if path:
                    bn = os.path.basename(tmp[0])
                    tmp[0] = os.path.join(path, bn)
                #Until geoio can write a .TIL file this is commented
                #vrt_name = tmp[0]+const.DG_SPEC['RAD_IMGS'][0]+tmp[1]
                vrt_name = tmp[0] + const.DG_SPEC['RAD_IMGS'][0] + '.VRT'
                cmd = []
                cmd.append("gdalbuildvrt")
                cmd.append(vrt_name)
                for i in flist_for_vrt: cmd.append(i)
                tt.cmd_line.exec_cmd(cmd)
                if not os.path.isfile(vrt_name):
                   raise StandardError("Creation of file "+vrt_name+" "
                                       "failed. This could possibly be a "
                                       "write access problem?")

        # Update the DG spectral file meta data
        self._set_dg_spectral_files()

    def create_toa_ref_files(self,path=None,components=True):
        if hasattr(self,'derived_dir'):
            path=self.derived_dir
        #Convert to files, each component if requested.
        if not components:
            # Create single file from image data
            fl = list(os.path.splitext(self.files.dfile))
            if path:
                bn = os.path.basename(fl[0])
                fl[0] = os.path.join(path, bn)
            new_fname = fl[0] + const.DG_SPEC['TOA_IMGS'][0] + fl[1]
            data = self.get_data_as_toa_ref()
            self.write_img_like_this(new_fname,data)
            # Add to dg_img_rad
            self.dg_img_rad = new_fname
        elif components:
            if (self.files.dfile_tiles[0] == self.files.dfile):
                logger.debug("This data set does not appear to have "
                      "componenets, reverting to the main file.")
                self.create_toa_ref_files(components=False,path=path)
            else:
                # Create each component file
                flist_for_vrt = []
                for yi,yv in enumerate(self.files.dfile_tiles):
                    x = GeoImage(yv)
                    fl = list(os.path.splitext(x.files.dfile))
                    if path:
                        bn = os.path.basename(fl[0])
                        fl[0] = os.path.join(path, bn)
                    new_fname = fl[0] + const.DG_SPEC['TOA_IMGS'][0] + fl[1]
                    flist_for_vrt.append(new_fname)
                    data = self.get_data_as_toa_ref(component=yi+1)
                    x.write_img_like_this(new_fname,data)
                    x=None
                # Create the vrt
                tmp = list(os.path.splitext(self.files.dfile))
                if path:
                    bn = os.path.basename(tmp[0])
                    tmp[0] = os.path.join(path, bn)
                #Until geoio can write a .TIL file this is commented
                #vrt_name = tmp[0]+const.DG_SPEC['TOA_IMGS'][0]+tmp[1]
                vrt_name = tmp[0] + const.DG_SPEC['TOA_IMGS'][0] + '.VRT'
                cmd = []
                cmd.append("gdalbuildvrt")
                cmd.append(vrt_name)
                for i in flist_for_vrt: cmd.append(i)
                tt.cmd_line.exec_cmd(cmd)
                if not os.path.isfile(vrt_name):
                   raise StandardError("Creation of file "+vrt_name+" "
                                       "failed. This could possibly be a "
                                       "write access problem?")

        # Update the DG spectral file meta data
        self._set_dg_spectral_files()

    def create_dgacomp_ref_files(self,path=None,ms_aod_map=None,
                                 force_create=False):
        """Create surface reflectance files using DGAComp.  This currently
        uses the IDL code.

        ms_aod_map must be pass if this is a SWIR or a PAN image.
        """

        if force_create:
            pass
        elif self.files.dgacomp:
            logger.debug("DGAcomp files alearead exist... skipping recreation.")
            return

        # Check that the request is not a .VRT file - I don't think dgacomp
        # can handle that
        if not ((os.path.splitext(self.files.dfile)[-1] != '.VRT') and \
                (os.path.splitext(self.files.dfile)[-1] != '.vrt')):
            raise ValueError("DGAComp doesn't handle VRT files.  Can you "
                             "pass in the TIL file?")

        if (not path) & hasattr(self,'derived_dir'):
            path=self.derived_dir

        # Catch the approriate call for each spectral type
        if self.meta_dg.IMD.BANDID == const.DG_BANDID['IMDXML'][0]: #PAN
            if not ms_aod_map: raise ValueError("An ms_aod_map must be passed "
                                                "for PAN data.")
            inFile = self.files.dfile
            tmp = os.path.splitext(self.files.dfile)
            if path:
                x = os.path.join(path, os.path.basename(tmp[0]))
            else:
                x = tmp[0]
            outFile = x + const.DG_SPEC['DGACOMP_IMGS'][0]
            aodFile = ms_aod_map
        elif self.meta_dg.IMD.BANDID == const.DG_BANDID['IMDXML'][1]: #MS
            inFile = self.files.dfile
            tmp = os.path.splitext(self.files.dfile)
            if path:
                x = os.path.join(path, os.path.basename(tmp[0]))
            else:
                x = tmp[0]
            outFile = x + const.DG_SPEC['DGACOMP_IMGS'][0]
        elif self.meta_dg.IMD.BANDID == const.DG_BANDID['IMDXML'][2]: #SWIR
            if not ms_aod_map: raise ValueError("An ms_aod_map must be "
                                                "passed for SWIR data.")
            inFile = self.files.dfile
            tmp = os.path.splitext(self.files.dfile)
            if path:
                x = os.path.join(path, os.path.basename(tmp[0]))
            else:
                x = tmp[0]
            outFile = x + const.DG_SPEC['DGACOMP_IMGS'][0]
            aodFile = ms_aod_map

        # This works but uses hard coded file names which is evidentally
        # fragile under certain python deployment circumstances.
        # dgacomp_wrapper = os.path.join(os.path.dirname(__file__),
        #                                'nwl_dgacomp_batch.pro')
        # Running the above with setuptools' pkg_resources
        dgacomp_wrapper = resource_filename(__name__, 'nwl_dgacomp_batch.pro')
        os.path.isfile(dgacomp_wrapper)
        logger.debug(dgacomp_wrapper)
        if not os.path.isfile(dgacomp_wrapper):
            raise ValueError("The DGAComp wrapper files isn't present "
                             "in the module.")

        # Create command line idl call like:
        # idl -e ".run nwl_batch.pro" -args '/path/to/in_file'
        #                                   '/path/to/out_file'
        #                                   '/path/to/ms_aod_map'
        # pass ms_aod_map if in_file is swir or pan

        cmd = []
        cmd.append('idl -e ".run '+dgacomp_wrapper+'" -args')
        cmd.append(const.DGACOMP['TARGET'][const.DGACOMP_INDEX])
        cmd.append(inFile)
        cmd.append(outFile)
        try:
            cmd.append(aodFile)
        except:
            pass

        # Hack subprocess to get aliases into the shell
        # http://stackoverflow.com/questions/25099895/from-python-start-a-shell-that-can-interpret-functions-and-aliases
        # http://stackoverflow.com/questions/12060863/python-subprocess-call-a-bash-alias
        tt.cmd_line.exec_cmd([os.getenv('SHELL'), '-i', '-c', ':;' + ' '.join(cmd)])

        # Add to dg_img_dgacomp
        # Since outFile doesn't have the ending on, I'm search for the file
        # to make sure I find a file that really exists with the correct
        # ending.
        # if os.path.isfile(outFile+'.TIL'):
        #     self.files.dgacomp = outFile+'.TIL'
        # elif os.path.isfile(outfile):
        #     self.files.dgacomp = outFile
        #
        # if os.path.isfile(outFile+const.DG_SPEC['DGACOMP_AOD'][0]):
        #     self.files.dgacomp_aodmap = outFile+const.DG_SPEC['DGACOMP_AOD'][0]

        # Update the DG spectral file meta data
        self._set_dg_spectral_files()

    def delete_rad_files(self,test_only=True):
        """Delete from disk the radiance files that this class can generate.
        If test_only is set to true (default), this will only simulated the
        removal until you pass test_only=False.
        """

        # Empty line for readability
        logger.info('')
        if test_only:
            logger.info("*** NO RADIANCE FILE OPERATIONS WILL BE PERFORMED ***")
            logger.info("*** test_only is set to True (default) ***")
            logger.info("")

        # Find spectral files
        if self.files.rad in self.files.rad_tiles:
            rall = self.files.rad_tiles
        else:
            rall = [self.files.rad]+self.files.rad_tiles

        # Kick out if no files of this spectral type are present
        if rall[0] == None:
            logger.info("No radiance files to be deleted.")
            return

        # Annouce the files to be removed
        logger.info("Deleting radiance files:")
        for f in rall:  logger.info(f)

        if test_only:
            return

        # Remove the files
        [os.remove(f) for f in rall]

        # Redo the spectral file dict and bunch
        self._set_dg_spectral_files()

    def delete_toa_ref_files(self,test_only=True):
        """Delete from disk the toa_reflectance files that this class can
        generate.  If test_only is set to true (default), this will only
        simulated the removal until you pass test_only=False.
        """

        # Empty line for readability
        logger.info('')
        if test_only:
            logger.info("*** NO TOA REFLECTANCE FILE OPERATIONS WILL BE PERFORMED ***")
            logger.info("*** test_only is set to True (default) ***")
            logger.info("")

        # Find spectral files
        if self.files.toa in self.files.toa_tiles:
            tall = self.files.toa_tiles
        else:
            tall = [self.files.toa]+self.files.toa_tiles

        # Kick out if no files of this spectral type are present
        if tall[0] == None:
            logger.info("No TOA files to be deleted.")
            return

        # Print files to be operated on
        logger.info("Deleting TOA reflectance files:")
        for f in tall:  logger.info(f)

        if test_only:
            return

        # Remove the files
        [os.remove(f) for f in tall]

        # Redo the spectral file dict and bunch
        self._set_dg_spectral_files()

    def delete_dgacomp_files(self,test_only=True):
        """Delete from disk the DGAComp files that this class can generate.
        If test_only is set to true (default), this will only simulated the
        removal until you pass test_only=False.
        """

        # Empty line for readability
        logger.info('')
        if test_only:
            logger.info("*** NO DGACOMP FILE OPERATIONS WILL BE PERFORMED ***")
            logger.info("*** test_only is set to True (default) ***")
            logger.info("")

        # Find spectral files
        dall = []
        if self.files.dgacomp in self.files.dgacomp_tiles:
            if self.files.dgacomp_tiles: dall=dall+self.files.dgacomp_tiles
            if self.files.dgacomp_aodmap: dall=dall+[self.files.dgacomp_aodmap]
            if self.files.dgacomp_other: dall=dall+self.files.dgacomp_other
        else:
            if self.files.dgacomp: dall=dall+[self.files.dgacomp]
            if self.files.dgacomp_tiles: dall=dall+self.files.dgacomp_tiles
            if self.files.dgacomp_aodmap: dall=dall+[self.files.dgacomp_aodmap]
            if self.files.dgacomp_other: dall=dall+self.files.dgacomp_other

        # Kick out if no files of this spectral type are present
        if not dall:
            logger.info("No DGAComp files to be deleted.")
            return

        # Print files to be operated on
        logger.info("Deleting DGAComp files:")
        for f in dall:  logger.info(f)

        if test_only:
            return

        # Remove the files
        [os.remove(f) for f in dall]

        # Redo the spectral file dict and bunch
        self._set_dg_spectral_files()

    def delete_all_spectral_files(self,test_only=True):
        """Delete from disk all spectrally derived files that this class can
        generate.  If test_only is set to true (default), this will only
        simulated the removal until you pass test_only=False.
        """
        self.delete_rad_files(test_only=test_only)
        self.delete_toa_ref_files(test_only=test_only)
        self.delete_dgacomp_files(test_only=test_only)

    def get_data_as_surf_ref(self):
        """Get data from DGAcomp files, if they exist."""
        if not self.files.dgacomp:
            raise ValueError('DGAComp files must be precomputed to use ' \
                             'this method.')
        tmp = GeoImage(self.files.dgacomp)
        data = tmp.get_data()
        del tmp

        return data

def parse_dg_time_str(dtstr):
    if not dtstr.endswith('Z'):
        logger.debug(dtstr)
        raise ValueError('The date/time string retrieved from the DG '
                         'meta data file is not of the format expected.')
    # Example:  '2013-02-16T18:28:30.183342Z'
    tmp = re.split("-|T|:|\.|Z", dtstr)
    year = int(tmp[0])
    month = int(tmp[1])
    day = int(tmp[2])
    hour = int(tmp[3])
    min = int(tmp[4])
    sec = int(tmp[5])
    usec = int(tmp[6]) #Fraction second to msecs

    if len(tmp[6]) != 6: raise ValueError("This routine expects there to be "
                                          "six digits in the usec field.")

    return [year,month,day,hour,min,sec,usec]


class DGImageSet(object):
    """This class is designed to encapsulate the imagery from a DigitalGlobe
    delivery directory.
    """

    def __init__(self,list_of_dg_imgs):
        # Initialize the DGImageDir Objects for this image
        self._set_delivery_dir_components(list_of_dg_imgs)

    def _set_delivery_dir_components(self,paths):
        counts = [0,0,0,0]
        self.obj_list = []
        for x in paths:
            y = DGImage(x)
            if y.bandid == 'P':
                self.pan = DGImage(x)
                self.obj_list.append(self.pan)
                counts[0] += 1
            if y.bandid == 'Multi':
                self.mul = DGImage(x)
                self.obj_list.append(self.mul)
                counts[1] += 1
            if y.bandid == 'All-S':
                self.swr = DGImage(x)
                self.obj_list.append(self.swr)
                counts[2] += 1
            if y.bandid == 'All-C':
                self.cavis = DGImage(x)
                self.obj_list.append(self.cavis)
                counts[3] += 1
            y = None

        for x in counts:
            if x>1:
                raise ValueError("More than one of a single image type was "
                                 "passed into DGDelivDir.")

    def __repr__(self):
        """overload repr to print somethign a bit more useful."""
        sss = ''
        sss+=str(self.__class__.__name__)

        # Get components present and file_names
        components = []
        cdirs = []
        try:
            cdirs.append(self.pan.meta.file_name)
            components.append("Pan")
        except: pass
        try:
            cdirs.append(self.mul.meta.file_name)
            components.append("VNIR")
        except: pass
        try:
            cdirs.append(self.swr.meta.file_name)
            components.append("SWIR")
        except: pass
        try:
            cdirs.append(self.cavis.meta.file_name)
            components.append("CAVIS")
        except: pass

        # Add components to string
        sss+=' ('+', '.join(components)+')'

        # Add most common directory to string
        sss+=" [" + os.path.commonprefix(cdirs).split(os.sep)[-2] + "]"

        return sss

    def build_supercube(self):
        logger.error("not implemented yet")
        pass

    def run_dg_acomp(self):
        logger.error("not implemented yet")
        pass


def create_DGDelivDir_objs(spath,stupid_search=True):
    """This function scans for multiple DigitalGlobe image
    delivery directories and attempts to reconstruct them into
    DGDelivDir objects with the correct components (MUL, SWIR, etc.).

    stupid_search is there to help find SWIR and CAVIS orders that don't
    have the same product id.  This is complicated because as of now (Jan.
    3rd, 2015) the factory is not providing CAVIS and SWIR imagery in a
    single order.  So this function has to go search for them based on the
    catid and do manual checks against the image properties (projection,
    etc.) to guess if they belong to together.
    """
    img_exts = []
    for x in list_bandid_dir:
        img_exts.append('*'+x+'*')

    # Find all directories that match the image filter strings above
    all_img_dirs = tt.files.search(spath, img_exts, ret_files=False,
                                   ret_dirs=True, depth=0)

    ## Put together dir and img value lists
    # If the directory does load, it isn't a DG image directory.
    # Also populated info to filter with.
    img_dirs = []   #Final list of image dirs to use
    img_pid = []    #Image product id
    #dir_pid = []   #Directory product id
    img_cid = []    #Image catalog id
    img_bid = []    #Image band id (PAN, MUL, etc)
    img_outf = []   #Image output foramt
    img_plvl = []   #Image product level
    img_scnd = []   #Scan direction
    img_satid = []  #Satellite ID
    img_rker = []   #Resampling kernel
    img_gentm = []  #Product generation time
    for x in all_img_dirs:
        try:
            y = DGImageDir(x)
        except:
            # If an error from above, then this isn't a DG image dir.
            continue
        img_dirs.append(x)
        img_pid.append(y.meta_dg.IMD.PRODUCTORDERID)
        #dir_pid.append((os.path.basename(x).rsplit('_',1))[0])
        img_cid.append(y.meta_dg.IMD.IMAGE.CATID)
        img_bid.append(y.meta_dg.IMD.BANDID)
        img_outf.append(y.meta_dg.IMD.OUTPUTFORMAT)
        img_plvl.append(y.meta_dg.IMD.PRODUCTLEVEL)
        img_scnd.append(y.meta_dg.IMD.IMAGE.SCANDIRECTION)
        img_satid.append(y.meta_dg.IMD.IMAGE.SATID)
        img_rker.append(y.meta_dg.IMD.IMAGE.RESAMPLINGKERNEL)
        img_gentm.append(y.meta_dg.IMD.GENERATIONTIME)
        y = None

    ## Put together image sets
    if not stupid_search:
        # If I don't need to do anything fancy, pass based on product id
        (set_pid,index_pid) = np.unique(img_pid, return_inverse=True)
        index_master = index_pid
    elif stupid_search:
        # Since stupid search is turned on (until the factory starts
        # delivering everything together), start putting the directories
        # together based on related information and a little bit of luck.
        # Checking catids and other values that indicate image directories
        # below to the same product.
        img_cid2 = [x[:3]+x[4:] for x in img_cid]
        (set_cid2,index_cid2) = np.unique(img_cid2, return_inverse=True)
        index_master = index_cid2

        checks=[]
        for x in set(index_master):
            check_bool = np.asarray(index_master == x)
            checks.append(set(np.asarray(img_outf)[check_bool]))
            checks.append(set(np.asarray(img_plvl)[check_bool]))
            checks.append(set(np.asarray(img_scnd)[check_bool]))
            checks.append(set(np.asarray(img_satid)[check_bool]))
            checks.append(set(np.asarray(img_rker)[check_bool]))

        #(set_outf,index_outf) = np.unique(img_outf,return_inverse=True)
        #(set_plvl,index_plvl) = np.unique(img_plvl,return_inverse=True)
        #(set_scnd,index_scnd) = np.unique(img_scnd,return_inverse=True)
        #(set_satid,index_satid) = np.unique(img_satid,return_inverse=True)
        #(set_rker,index_rker) = np.unique(img_rker,return_inverse=True)
        index_master = _split_master(img_outf,index_master)
        index_master = _split_master(img_plvl,index_master)
        index_master = _split_master(img_scnd,index_master)
        index_master = _split_master(img_satid,index_master)
        index_master = _split_master(img_rker,index_master)

        (set_gentm,index_gentm) = np.unique(img_gentm, return_inverse=True)
        logger.debug(img_dirs)

        # TODO --- actually code up this time check
        # Check that the image generation time is +/- one hour
        # Not the best check because gen time is not guarantee
        time_vars = parse_dg_time_str(img_gentm[0])
        tzinfo_utc = pytz.timezone('UTC')
        gentime = datetime.datetime(*time_vars, tzinfo=tzinfo_utc)
        range = datetime.timedelta(hours=1)
        gentime-range<=gentime<=gentime+range

    # Check that there isn't more than one of each band type in the master list
    # TODO --- code up this band array check
    (set_bid,index_bid) = np.unique(img_bid, return_inverse=True)

    ### Generate the DGDelivDir objects
    list_of_objs = []
    for x in set(index_master):
        pass_list=np.asarray(img_dirs)[index_master == x]
        logger.debug(pass_list)
        logger.debug('')
        list_of_objs.append(DGDelivDir(pass_list))

    return list_of_objs


def _split_master(p,master):
    # For each item in master, check if those items are split in p.  If they
    # are, split master by the p split.
    mbool = [True]*len(master)
    for x in set(master):
        ind = [yi for yi,yv in enumerate(master) if yv == x]
        mbool = [False for qi,qv in enumerate(mbool) if master[qi]==x]
        #ind = np.where(master==x)
        #(sm,im) = np.unique(master[ind],return_inverse=True)
        p_atind = [yv for yi,yv in enumerate(p) if yi in ind]
        p_atind_set = set(p_atind)
        if len(p_atind_set) > 1:
            # Need to split index_master
            #(sp,ip) = np.unique(p_atind,return_inverse=True)
            skip = True
            imax = max(master)
            for y in p_atind_set:
                if skip==True:
                    skip=False
                    continue

                indy = [zi for zi,zv in enumerate(p_atind) if zv == y]
                #master = [imax if ]
                master = [iup for ai,av in enumerate(master) if ai]
                indy = [z for z in p[ind] if z == y]
                imax += 1

    return master


def get_alias_band_numbers(dg_band_name,requested_alias):
    """Get the band numbers, for a specific DG image sensor, that correspond to
    a spectral region alias.  If a band is requested by an alias that isn't
    present in the requested band names, a None value is returned.  The
    returned values are base 1 so that they can be passed directly to
    gdal GetRasterBand.
    """

    # Look up band names given the provided dg_band string
    names = const.DG_BAND_NAMES[dg_band_name]

    try:
        alias = const.DG_BAND_ALIASES[requested_alias]
    except TypeError:
        alias = requested_alias
    except KeyError:
        if requested_alias in names:
            alias = requested_alias
        else:
            raise

    # Return the band numbers for the requested alias
    out=[None]*len(alias)
    for i,v in enumerate(alias):
        try:
            if v.upper() in names:
                out[i] = names.index(v.upper())+1
        except:
            out[i] = v
    # The above code replaces the following list comprehension so that the
    # function will pass through integers
    #out=[names.index(v.upper()) if v.upper() in names else None for v in alias]


    return out
