"""
IO operations for Feature Classes
"""
import os
import sys
import shutil
import datetime

import numpy as np
import pandas as pd

try:
    import arcpy
    from arcpy import da
    HASARCPY = True
except:
    HASARCPY = False

try:
    import fiona
    HASFIONA = True
except:
    HASFIONA = False

try:
    import shapefile
    HASPYSHP = True
except:
    HASPYSHP = False
#--------------------------------------------------------------------------
def _infer_type(df, col):
    """
    internal function used to get the datatypes for the feature class if
    the dataframe's _field_reference is NULL or there is a column that does
    not have a dtype assigned to it.

    Input:
     dataframe - spatialdataframe object
    Ouput:
      field type name
    """
    import six
    nn = df[col].notnull()
    nn = list(df[nn].index)
    if len(nn) > 0:
        val = df[col][nn[0]]
        if isinstance(val, six.string_types):
            return "TEXT"
        elif isinstance(val, tuple(list(six.integer_types) + [np.int32])):
            return "INTEGER"
        elif isinstance(val, (float, np.int64 )):
            return "FLOAT"
        elif isinstance(val, datetime):
            return "DATE"
    return "TEXT"
#--------------------------------------------------------------------------
def _geojson_to_esrijson(geojson):
    """converts the geojson spec to esri json spec"""
    if geojson['type'] in ['Polygon', 'MultiPolygon']:
        return {
            'rings' : geojson['coordinates'],
            'spatialReference' : {'wkid' : 4326}
        }
    elif geojson['type'] == "Point":
        return {
            "x" : geojson['coordinates'][0],
            "y" : geojson['coordinates'][1],
            'spatialReference' : {'wkid' : 4326}
        }
    elif geojson['type'] == "MultiPoint":
        return {
            "points" : geojson['coordinates'],
            'spatialReference' : {'wkid' : 4326}
        }
    elif geojson['type'] in ['LineString', 'MultiLineString']:
        return {
            "paths" : geojson['coordinates'],
            'spatialReference' : {'wkid' : 4326}
        }
    return geojson
#--------------------------------------------------------------------------
def _geometry_to_geojson(geom):
    """converts the esri json spec to geojson"""
    if 'rings' in geom and \
       len(geom['rings']) == 1:
        return {
            'type' : "Polygon",
            "coordinates" : geom['rings']
        }
    elif 'rings' in geom and \
       len(geom['rings']) > 1:
        return {
            'type' : "MultiPolygon",
            "coordinates" : geom['rings']
        }
    elif geom['type'] == "Point":
        return {
            "coordinates" : [geom['x'], geom['y']],
            "type" : "Point"
        }
    elif geom['type'] == "MultiPoint":
        return {
            "coordinates" : geom['points'],
            'type' : "MultiPoint"
        }
    elif geom['type'].lower() == "polyline" and \
         len(geom['paths']) <= 1:
        return {
            "coordinates" : geom['paths'],
            'type' : "LineString"
        }
    elif geom['type'].lower() == "polyline" and \
         len(geom['paths']) > 1:
        return {
            "coordinates" : geom['paths'],
            'type' : "MultiLineString"
        }
    return geom
#--------------------------------------------------------------------------
def _from_xy(df, x_column, y_column, sr=None):
    """
    Takes an X/Y Column and Creates a Point Geometry from it.
    """
    from arcgis.geometry import SpatialReference, Geometry
    if sr is None:
        sr = SpatialReference({'wkid' : 4326})
    if not isinstance(sr, SpatialReference):
        if isinstance(sr, dict):
            sr = SpatialReference(sr)
        elif isinstance(sr, int):
            sr = SpatialReference({'wkid' : sr})
        elif isinstance(sr, str):
            sr = SpatialReference({'wkt' : sr})
    geoms = []
    for idx, row in df.iterrows():
        geoms.append(
            Geometry({'x' : row[x_column], 'y' : row[y_column],
             'spatialReference' : sr})
        )
    df['SHAPE'] = geoms
    df.spatial.set_geometry('SHAPE')
    return df

#--------------------------------------------------------------------------
def from_table(filename, **kwargs):
    """
    Allows a user to read from a non-spatial table

    **Note: ArcPy is Required for this method**

    ===============     ====================================================
    **Argument**        **Description**
    ---------------     ----------------------------------------------------
    filename            Required string. The path to the table.
    ===============     ====================================================

    **Keyword Arguments**

    ===============     ====================================================
    **Argument**        **Description**
    ---------------     ----------------------------------------------------
    fields              Optional List/Tuple. A list (or tuple) of field
                        names. For a single field, you can use a string
                        instead of a list of strings.

                        Use an asterisk (*) instead of a list of fields if
                        you want to access all fields from the input table
                        (raster and BLOB fields are excluded). However, for
                        faster performance and reliable field order, it is
                        recommended that the list of fields be narrowed to
                        only those that are actually needed.

                        Geometry, raster, and BLOB fields are not supported.

    ---------------     ----------------------------------------------------
    where               Optional String. An optional expression that limits
                        the records returned.
    ---------------     ----------------------------------------------------
    skip_nulls          Optional Boolean. This controls whether records
                        using nulls are skipped.
    ---------------     ----------------------------------------------------
    null_value          Optional String/Integer/Float. Replaces null values
                        from the input with a new value.
    ===============     ====================================================

    :returns: pd.DataFrame

    """
    if HASARCPY:
        where = kwargs.pop("where", None)
        fields = kwargs.pop('fields', "*")
        skip_nulls = kwargs.pop('skip_nulls', True)
        null_value = kwargs.pop("null_value", None)
        return pd.DataFrame(da.TableToNumPyArray(in_table=filename,
                                                 field_names=fields,
                                                 where_clause=where,
                                                 skip_nulls=skip_nulls,
                                                 null_value=null_value))
    elif filename.lower().find('.csv') > -1:
        return pd.read_csv(filename)

    return
#--------------------------------------------------------------------------
def to_table(geo, location, overwrite=True):
    """
    Exports a geo enabled dataframe to a table.

    ===========================     ====================================================================
    **Argument**                    **Description**
    ---------------------------     --------------------------------------------------------------------
    location                        Required string. The output of the table.
    ---------------------------     --------------------------------------------------------------------
    overwrite                       Optional Boolean.  If True and if the table exists, it will be
                                    deleted and overwritten.  This is default.  If False, the table and
                                    the table exists, and exception will be raised.
    ===========================     ====================================================================

    :returns: String
    """
    out_location= os.path.dirname(location)
    fc_name = os.path.basename(location)
    df = geo._data
    if location.lower().find('.csv') > -1:
        geo._df.to_csv(location)
        return location
    elif HASARCPY:
        columns = df.columns.tolist()
        join_dummy = "AEIOUYAJC81Z"
        try:
            columns.pop(columns.index(df.spatial.name))
        except:
            pass
        dtypes = [(join_dummy, np.int64)]
        if overwrite and arcpy.Exists(location):
            arcpy.Delete_management(location)
        elif overwrite == False and arcpy.Exists(location):
            raise ValueError(('overwrite set to False, Cannot '
                              'overwrite the table. '))
        fc = arcpy.CreateTable_management(out_path=out_location,
                                          out_name=fc_name)[0]
        # 2. Add the Fields and Data Types
        #
        oidfld = da.Describe(fc)['OIDFieldName']
        for col in columns[:]:
            if col.lower() in ['fid', 'oid', 'objectid']:
                dtypes.append((col, np.int32))
            elif df[col].dtype.name == 'datetime64[ns]':
                dtypes.append((col, '<M8[us]'))
            elif df[col].dtype.name == 'object':
                try:
                    u = type(df[col][df[col].first_valid_index()])
                except:
                    u = pd.unique(df[col].apply(type)).tolist()[0]
                if issubclass(u, str):
                    mlen = df[col].str.len().max()
                    dtypes.append((col, '<U%s' % int(mlen)))
                else:
                    try:
                        dtypes.append((col, type(df[col][0])))
                    except:
                        dtypes.append((col, '<U254'))
            elif df[col].dtype.name == 'int64':
                dtypes.append((col, np.int64))
            elif df[col].dtype.name == 'bool':
                dtypes.append((col, np.int32))
            else:
                dtypes.append((col, df[col].dtype.type))

        array = np.array([],
                        np.dtype(dtypes))
        arcpy.da.ExtendTable(fc,
                             oidfld, array,
                             join_dummy, append_only=False)
        # 3. Insert the Data
        #
        fields = arcpy.ListFields(fc)
        icols = [fld.name for fld in fields \
                 if fld.type not in ['OID', 'Geometry'] and \
                 fld.name in df.columns]
        dfcols = [fld.name for fld in fields \
                  if fld.type not in ['OID', 'Geometry'] and\
                  fld.name in df.columns]
        with da.InsertCursor(fc, icols) as irows:
            for idx, row in df[dfcols].iterrows():
                try:
                    irows.insertRow(row.tolist())
                except:
                    print("row %s could not be inserted." % idx)
        return fc

    return
#--------------------------------------------------------------------------
def from_featureclass(filename, **kwargs):
    """
    Returns a GeoDataFrame from a feature class.
    Inputs:
     filename: full path to the feature class
    Optional Parameters:
     sql_clause: sql clause to parse data down
     where_clause: where statement
     sr: spatial reference object
     fields: list of fields to extract from the table
    """
    from arcgis.geometry import _types
    import json
    if HASARCPY:
        sql_clause = kwargs.pop('sql_clause', (None,None))
        where_clause = kwargs.pop('where_clause', None)
        fields = kwargs.pop('fields', None)
        sr = kwargs.pop('sr', None)
        try:
            desc = arcpy.da.Describe(filename)
            area_field = desc.pop('areaFieldName', None)
            length_field = desc.pop('lengthFieldName', None)
        except: # for older versions of arcpy
            desc = arcpy.Describe(filename)
            desc = {
                'fields' : desc.fields,
                'shapeType' : desc.shapeType
            }
            area_field = getattr(desc, 'areaFieldName', None)
            length_field = getattr(desc, 'lengthFieldName', None)
        shape_name = desc['shapeType']
        if fields is None:
            fields = [fld.name for fld in desc['fields'] \
                      if fld.type not in ['Geometry'] and \
                      fld.name not in [area_field, length_field]]
            cursor_fields = fields + ['SHAPE@JSON']
            df_fields = fields + ['SHAPE']
        count = 0
        dfs = []
        shape_field_idx = cursor_fields.index("SHAPE@JSON")
        with da.SearchCursor(filename,
                             field_names=cursor_fields,
                             where_clause=where_clause,
                             sql_clause=sql_clause,
                             spatial_reference=sr) as rows:
            srows = []
            for row in rows:
                srows.append(row)
                if len(srows) == 25000:
                    dfs.append( pd.DataFrame(srows,
                                             columns=df_fields))
                    srows = []
            if len(srows):
                dfs.append( pd.DataFrame(srows,
                                         columns=df_fields))
                srows = []
            del srows
        if len(dfs) > 0:
            df = pd.concat(dfs)
            df = df.reset_index(drop=True)
        elif len(dfs) == 1:
            df = dfs[0]
        else:
            df = pd.DataFrame([],
                              columns=df_fields)
        q = df.SHAPE.notnull()
        gt = desc['shapeType'].lower()
        geoms = {
            "point" : _types.Point,
            "polygon" : _types.Polygon,
            "polyline" : _types.Polyline,
            "multipoint" : _types.MultiPoint,
            "envelope" : _types.Envelope,
            "geometry" : _types.Geometry
        }
        df.SHAPE = (
           df.SHAPE[q]
           .apply(pd.io.json.loads)
           .apply(geoms[gt])
        )
        df.spatial.set_geometry("SHAPE")
        return df
    elif HASARCPY == False and \
         HASPYSHP == True and\
         filename.lower().find('.shp') > -1:
        geoms = []
        records = []
        reader = shapefile.Reader(filename)
        fields = [field[0] for field in reader.fields if field[0] != 'DeletionFlag']
        for r in reader.shapeRecords():
            atr = dict(zip(fields, r.record))
            g = r.shape.__geo_interface__
            g = _geojson_to_esrijson(g)
            geom = _types.Geometry(g)
            atr['SHAPE'] = geom
            records.append(atr)
            del atr
            del r, g
            del geom
        sdf = pd.DataFrame(records)
        sdf.spatial.set_geometry('SHAPE')
        sdf.reset_index(inplace=True)
        return sdf
    elif HASARCPY == False and \
         HASFIONA == True and \
         (filename.lower().find('.shp') > -1 or \
          os.path.dirname(filename).lower().find('.gdb') > -1):
        is_gdb = os.path.dirname(filename).lower().find('.gdb') > -1
        if is_gdb:
            with fiona.drivers():
                from arcgis.geometry import _types
                fp = os.path.dirname(filename)
                fn = os.path.basename(filename)
                geoms = []
                atts = []
                with fiona.open(path=fp, layer=fn) as source:
                    meta = source.meta
                    cols = list(source.schema['properties'].keys())
                    for idx, row in source.items():
                        geoms.append(_types.Geometry(row['geometry']))
                        atts.append(list(row['properties'].values()))
                        del idx, row
                    df = pd.DataFrame(data=atts, columns=cols)
                    df.spatial.set_geometry(geoms)
                    return df
        else:
            with fiona.drivers():
                from arcgis.geometry import _types
                geoms = []
                atts = []
                with fiona.open(path=filename) as source:
                    meta = source.meta
                    cols = list(source.schema['properties'].keys())
                    for idx, row in source.items():
                        geoms.append(_types.Geometry(row['geometry']))
                        atts.append(list(row['properties'].values()))
                        del idx, row
                    df = pd.DataFrame(data=atts, columns=cols)
                    df.spatial.set_geometry(geoms)
                    return df
    return
#--------------------------------------------------------------------------
def to_featureclass(geo,
                    location,
                    overwrite=True,
                    validate=False):
    """
    Exports the DataFrame to a Feature class.

    ===============     ====================================================
    **Argument**        **Description**
    ---------------     ----------------------------------------------------
    location            Required string. This is the output location for the
                        feature class. This should be the path and feature
                        class name.
    ---------------     ----------------------------------------------------
    overwrite           Optional Boolean. If overwrite is true, existing
                        data will be deleted and replaced with the spatial
                        dataframe.
    ===============     ====================================================


    :returns: string

    """


    out_location= os.path.dirname(location)
    fc_name = os.path.basename(location)
    df = geo._data
    if geo.name is None:
        raise ValueError("DataFrame must have geometry set.")
    if validate and \
       geo.validate(strict=True) == False:
        raise ValueError(("Mixed geometry types detected, "
                         "cannot export to feature class."))
    if HASARCPY:
        # 1. Create the Save Feature Class
        #
        columns = df.columns.tolist()
        join_dummy = "AEIOUYAJC81Z"
        columns.pop(columns.index(df.spatial.name))
        dtypes = [(join_dummy, np.int64)]
        if overwrite and arcpy.Exists(location):
            arcpy.Delete_management(location)
        elif overwrite == False and arcpy.Exists(location):
            raise ValueError(('overwrite set to False, Cannot '
                              'overwrite the table. '))

        notnull = geo._data[geo._name].notnull()
        idx = geo._data[geo._name][notnull].first_valid_index()
        sr = sr = geo._data[geo._name][idx]['spatialReference']
        gt = geo._data[geo._name][idx].geometry_type.upper()
        null_geom = {
            'point': pd.io.json.dumps({'x' : None, 'y': None, 'spatialReference' : sr}),
            'polyline' : pd.io.json.dumps({'paths' : [], 'spatialReference' : sr}),
            'polygon' : pd.io.json.dumps({'rings' : [], 'spatialReference' : sr}),
            'multipoint' : pd.io.json.dumps({'points' : [], 'spatialReference' : sr})
        }
        sr = geo._data[geo._name][idx].spatial_reference
        null_geom = null_geom[gt.lower()]
        fc = arcpy.CreateFeatureclass_management(out_location,
                                                 spatial_reference=sr,
                                                 geometry_type=gt,
                                                 out_name=fc_name,
                                                )[0]
        # 2. Add the Fields and Data Types
        #
        oidfld = da.Describe(fc)['OIDFieldName']
        for col in columns[:]:
            if col.lower() in ['fid', 'oid', 'objectid']:
                dtypes.append((col, np.int32))
            elif df[col].dtype.name == 'datetime64[ns]':
                dtypes.append((col, '<M8[us]'))
            elif df[col].dtype.name == 'object':
                try:
                    u = type(df[col][df[col].first_valid_index()])
                except:
                    u = pd.unique(df[col].apply(type)).tolist()[0]
                if issubclass(u, str):
                    mlen = df[col].str.len().max()
                    dtypes.append((col, '<U%s' % int(mlen)))
                else:
                    try:
                        dtypes.append((col, type(df[col][idx])))
                    except:
                        dtypes.append((col, '<U254'))
            elif df[col].dtype.name == 'int64':
                dtypes.append((col, np.int64))
            elif df[col].dtype.name == 'bool':
                dtypes.append((col, np.int32))
            else:
                dtypes.append((col, df[col].dtype.type))

        array = np.array([],
                        np.dtype(dtypes))
        arcpy.da.ExtendTable(fc,
                             oidfld, array,
                             join_dummy, append_only=False)
        # 3. Insert the Data
        #
        fields = arcpy.ListFields(fc)
        icols = [fld.name for fld in fields \
                 if fld.type not in ['OID', 'Geometry'] and \
                 fld.name in df.columns] + ['SHAPE@JSON']
        dfcols = [fld.name for fld in fields \
                  if fld.type not in ['OID', 'Geometry'] and\
                  fld.name in df.columns] + [df.spatial.name]
        with da.InsertCursor(fc, icols) as irows:
            def _insert_row(row):
                row[-1] = pd.io.json.dumps(row[-1])
                irows.insertRow(row)
            q = df[geo._name].isna()
            df.loc[q, 'SHAPE'] = null_geom # set null values to proper JSON
            np.apply_along_axis(_insert_row, 1, df[dfcols].values)
            df.loc[q, 'SHAPE'] = None # reset null values
        return fc
    elif HASPYSHP:
        if fc_name.endswith('.shp') == False:
            fc_name = "%s.shp" % fc_name
        return _pyshp_to_shapefile(df=df,
                            out_path=out_location,
                            out_name=fc_name)
    elif HASARCPY == False and HASPYSHP == False:
        raise Exception(("Cannot Export the data without ArcPy or PyShp modules."
                        " Please install them and try again."))
    else:
        return None
#--------------------------------------------------------------------------
def _pyshp_to_shapefile(df, out_path, out_name):
    """
    Saves a SpatialDataFrame to a Shapefile using pyshp

    :Parameters:
     :df: spatial dataframe
     :out_path: folder location to save the data
     :out_name: name of the shapefile
    :Output:
     path to the shapefile or None if pyshp isn't installed or
     spatial dataframe does not have a geometry column.
    """
    from arcgis.geometry._types import Geometry
    if HASPYSHP:
        GEOMTYPELOOKUP = {
            "Polygon" : shapefile.POLYGON,
            "Point" : shapefile.POINT,
            "Polyline" : shapefile.POLYLINE,
            'null' : shapefile.NULL
        }
        if os.path.isdir(out_path) == False:
            os.makedirs(out_path)
        out_fc = os.path.join(out_path, out_name)
        if out_fc.lower().endswith('.shp') == False:
            out_fc += ".shp"
        geom_field = df.spatial._name
        if geom_field is None:
            return
        geom_type = "null"
        idx = df[geom_field].first_valid_index()
        if idx > -1:
            geom_type = df.loc[idx][geom_field].type
        shpfile = shapefile.Writer(GEOMTYPELOOKUP[geom_type])
        shpfile.autoBalance = 1
        dfields = []
        cfields = []
        for c in df.columns:
            idx = df[c].first_valid_index()
            if idx > -1:
                if isinstance(df[c].loc[idx],
                              Geometry):
                    geom_field = (c, "GEOMETRY")
                else:
                    cfields.append(c)
                    if isinstance(df[c].loc[idx], (str)):
                        shpfile.field(name=c, size=255)
                    elif isinstance(df[c].loc[idx], (int)):
                        shpfile.field(name=c, fieldType="N", size=5)
                    elif isinstance(df[c].loc[idx], (np.int, np.int32, np.int64)):
                        shpfile.field(name=c, fieldType="N", size=10)
                    elif isinstance(df[c].loc[idx], (np.float, np.float64)):
                        shpfile.field(name=c, fieldType="F", size=19, decimal=11)
                    elif isinstance(df[c].loc[idx], (datetime.datetime, np.datetime64)) or \
                         df[c].dtype.name == 'datetime64[ns]':
                        shpfile.field(name=c, fieldType="D", size=8)
                        dfields.append(c)
                    elif isinstance(df[c].loc[idx], (bool, np.bool)):
                        shpfile.field(name=c, fieldType="L", size=1)
            del c
            del idx
        for idx, row in df.iterrows():
            geom = row[df.spatial._name]
            if geom.type == "Polygon":
                shpfile.poly(geom['rings'])
            elif geom.type == "Polyline":
                shpfile.line(geom['paths'])
            elif geom.type == "Point":
                shpfile.point(x=geom.x, y=geom.y)
            else:
                shpfile.null()
            row = row[cfields].tolist()
            for fld in dfields:
                idx = df[cfields].columns.tolist().index(fld)
                if row[idx]:
                    row[idx] = row[idx].to_pydatetime()
            shpfile.record(*row)
            del idx
            del row
            del geom
        shpfile.save(out_fc)


        # create the PRJ file
        try:
            from urllib import request
            wkid = df.spatial.sr['wkid']

            prj_filename = out_fc.replace('.shp', '.prj')

            url = 'http://epsg.io/{}.esriwkt'.format(wkid)

            opener = request.build_opener()
            opener.addheaders = [('User-Agent', 'geosaurus')]
            resp = opener.open(url)

            wkt = resp.read().decode('utf-8')
            if len(wkt) > 0:
                prj = open(prj_filename, "w")
                prj.write(wkt)
                prj.close()
        except:
            # Unable to write PRJ file.
            pass

        del shpfile
        return out_fc
    return None