"""
Feature Layers and Tables provide the primary interface for working with features in a GIS.

Users create, import, export, analyze, edit, and visualize features, i.e. entities in space as feature layers.

A FeatureLayerCollection is a collection of feature layers and tables, with the associated relationships among the entities.
"""
import json
import os
from re import search
import time

import six
from arcgis._impl.common import _utils
from arcgis._impl.common._filters import StatisticFilter, TimeFilter, GeometryFilter
from arcgis._impl.common._mixins import PropertyMap
from arcgis._impl.common._utils import _date_handler, chunks

from .managers import AttachmentManager, SyncManager, FeatureLayerCollectionManager, FeatureLayerManager
from .feature import Feature, FeatureSet
from arcgis.geometry import SpatialReference
from arcgis.gis import Layer, _GISResource


class FeatureLayer(Layer):
    """
    The feature layer is the primary concept for working with features in a GIS.

    Users create, import, export, analyze, edit, and visualize features, i.e. entities in space as feature layers.

    Feature layers can be added to and visualized using maps. They act as inputs to and outputs from feature analysis
    tools.

    Feature layers are created by publishing feature data to a GIS, and are exposed as a broader resource (Item) in the
    GIS. Feature layer objects can be obtained through the layers attribute on feature layer Items in the GIS.
    """
    _metadatamanager = None

    def __init__(self, url, gis=None, container=None, dynamic_layer=None):
        """
        Constructs a feature layer given a feature layer URL
        :param url: feature layer url
        :param gis: optional, the GIS that this layer belongs to. Required for secure feature layers.
        :param container: optional, the feature layer collection to which this layer belongs
        :param dynamic_layer: optional dictionary. If the layer is given a dynamic layer definition, this will be added to functions.
        """
        if gis is None:
            import arcgis
            gis = arcgis.env.active_gis
        super(FeatureLayer, self).__init__(url, gis)
        self._storage = container
        self._dynamic_layer = dynamic_layer
        self.attachments = AttachmentManager(self)


    @classmethod
    def fromitem(cls, item, layer_id=0):
        """
        Creates a feature layer from a GIS Item.
        The type of item should be a 'Feature Service' that represents a FeatureLayerCollection.
        The layer_id is the id of the layer in feature layer collection (feature service).
        """
        return FeatureLayerCollection.fromitem(item).layers[layer_id]

    @property
    def manager(self):
        """
        Helper object to manage the feature layer, update it's definition, etc
        """
        url = self._url
        res = search("/rest/", url).span()
        add_text = "admin/"
        part1 = url[:res[1]]
        part2 = url[res[1]:]
        admin_url = "%s%s%s" % (part1, add_text, part2)

        res = FeatureLayerManager(admin_url, self._gis)
        return res

    @property
    def metadata(self):
        """
        The `metadata` property allows for the setting and downloading of the
        Feature Layer's metadata.  If metadata is disabled on the GIS or the
        layer does not support metdata, None value will be returned.

        =================     ====================================================================
        **Argument**          **Description**
        -----------------     --------------------------------------------------------------------
        value                 Required String. (SET) Path to the metadata file.
        =================     ====================================================================

        :returns: String (GET)

        """
        if 'hasMetadata' in self.properties:
            try:
                return self._download_metadata()
            except:
                return None
        return None
    #----------------------------------------------------------------------
    def _download_metadata(self, save_folder=None):
        """
        Downloads the metadata.xml to local disk

        =================     ====================================================================
        **Argument**          **Description**
        -----------------     --------------------------------------------------------------------
        save_folder           Optional String. A save location to download the metadata XML file.
        =================     ====================================================================

        :returns: String
        """
        import tempfile
        if save_folder is None:
            save_folder = tempfile.gettempdir()
        url = "%s%s" % (self.url, "/metadata")
        params = {'f' : 'json',
                  'format' : 'default'
                  }

        return self._con.get(url, params,
                                    out_folder=save_folder,
                                    file_name='metadata.xml')
    #----------------------------------------------------------------------
    def update_metadata(self, file_path):
        """
        Updates a Layer's metadata from an xml file.

        =================     ====================================================================
        **Argument**          **Description**
        -----------------     --------------------------------------------------------------------
        file_path             Required String.  The path to the .xml file that contains the metadata.
        =================     ====================================================================

        :returns: boolean
        """
        if 'hasMetadata' not in self.properties:
            return None

        if os.path.isfile(file_path) == False or \
           os.path.splitext(file_path)[1].lower() != '.xml':
            raise ValueError("file_path must be a XML file.")

        url = "%s%s" % (self.url, "/metadata/update")
        with open(file_path, 'r') as reader:
            text = reader.read()

            params = {
                'f' : 'json',
                'metadata' : text,
                "metadataUploadId": "",
                "metadataItemId": "",
                "metadataUploadFormat": "xml"
            }
            res = self._con.post(url, params)
            if 'statusUrl' in res:
                return self._status_metadata(res['statusUrl'])
        return False
    #----------------------------------------------------------------------
    def _status_metadata(self, url):
        """checks the update status"""
        res = self._con.get(url, {'f':'json'})
        if res['status'].lower() == 'completed':
            return True
        while res['status'].lower() != 'completed':
            time.sleep(1)
            res = self._con.get(url, {'f':'json'})
            if res['status'].lower() == 'completed':
                return True
            elif res['status'].lower() == 'failed':
                return False
        return False

    @property
    def container(self):
        """
        The feature layer collection to which this layer belongs.
        """
        return self._storage

    @container.setter
    def container(self, value):
        """
        The feature layer collection to which this layer belongs.
        """
        self._storage = value

    #----------------------------------------------------------------------
    def generate_renderer(self, definition, where=None):
        """
        This operation groups data using the supplied definition
        (classification definition) and an optional where clause. The
        result is a renderer object. Use baseSymbol and colorRamp to define
        the symbols assigned to each class. If the operation is performed
        on a table, the result is a renderer object containing the data
        classes and no symbols.

        =================     ====================================================================
        **Argument**          **Description**
        -----------------     --------------------------------------------------------------------
        definition            required dict. The definition using the renderer that is generated.
                              Use either class breaks or unique value classification definitions.
                              See: https://resources.arcgis.com/en/help/rest/apiref/ms_classification.html
        -----------------     --------------------------------------------------------------------
        where                 optional string. A where clause for which the data needs to be
                              classified. Any legal SQL where clause operating on the fields in
                              the dynamic layer/table is allowed.
        =================     ====================================================================

        :returns: dictionary

        """
        if self._dynamic_layer:
            url = "%s/generateRenderer" % self._url.split('?')[0]
        else:
            url = "%s/generateRenderer" % self._url
        params = {'f' : 'json',
                  'classificationDef' : definition
                  }
        if where:
            params['where'] = where
        if self._dynamic_layer is not None:
            params['layer'] = self._dynamic_layer
        return self._con.post(path=url, postdata=params)

    def _add_attachment(self, oid, file_path):
        """
        Adds an attachment to a feature service

        =================     ====================================================================
        **Argument**          **Description**
        -----------------     --------------------------------------------------------------------
        oid                   Required string/integer. OBJECTID value to add attachment to.
        -----------------     --------------------------------------------------------------------
        file_path             Required string. Location of the file to attach.
        =================     ====================================================================

        :returns: dictionary

        """
        if (os.path.getsize(file_path) >> 20) <= 9:
            params = {'f': 'json'}
            if self._dynamic_layer:
                attach_url = self._url.split('?')[0] + "/%s/addAttachment" % oid
                params['layer'] = self._dynamic_layer
            else:
                attach_url = self._url + "/%s/addAttachment" % oid
            files = {'attachment': file_path}
            res = self._con.post(path=attach_url,
                                 postdata=params,
                                 files=files, token=self._token)
            return res
        else:
            params = {'f': 'json'}
            container = self.container
            itemid = container.upload(file_path)
            if self._dynamic_layer:
                attach_url = self._url.split('?')[0] + "/%s/addAttachment" % oid
                params['layer'] = self._dynamic_layer
            else:
                attach_url = self._url + "/%s/addAttachment" % oid
            params['uploadId'] = itemid
            res = self._con.post(attach_url,
                                 params)
            if res['addAttachmentResult']['success'] == True:
                container._delete_upload(itemid)
            return res
    # ----------------------------------------------------------------------
    def _delete_attachment(self, oid, attachment_id):
        """
        Removes an attachment from a feature service feature

        =================     ====================================================================
        **Argument**          **Description**
        -----------------     --------------------------------------------------------------------
        oid                   Required string/integer. OBJECTID value to add attachment to.
        -----------------     --------------------------------------------------------------------
        attachment_id         Required integer. Id of the attachment to erase.
        =================     ====================================================================

        :returns: dictionary
        """
        params = {
            "f": "json",
            "attachmentIds": "%s" % attachment_id
        }
        if self._dynamic_layer:
            url = self._url.split('?')[0] + "/%s/deleteAttachments" % oid
            params['layer'] = self._dynamic_layer
        else:
            url = self._url + "/%s/deleteAttachments" % oid
        return self._con.post(url, params, token=self._token)

    # ----------------------------------------------------------------------
    def _update_attachment(self, oid, attachment_id, file_path):
        """
        Updates an existing attachment with a new file

        =================     ====================================================================
        **Argument**          **Description**
        -----------------     --------------------------------------------------------------------
        oid                   Required string/integer. OBJECTID value to add attachment to.
        -----------------     --------------------------------------------------------------------
        attachment_id         Required integer. Id of the attachment to erase.
        -----------------     --------------------------------------------------------------------
        file_path             Required string. Path to new attachment
        =================     ====================================================================

        :returns: dictionary

        """
        params = {
            "f": "json",
            "attachmentId": "%s" % attachment_id
        }
        files = {'file': file_path}
        if self._dynamic_layer is not None:
            url = self.url.split('?')[0] + "/%s/attachments" % oid
            params['layer'] = self._dynamic_layer
        else:
            url = self._url + "/%s/attachments" % oid
        res = self._con.post(path=url,
                             postdata=params,
                             files=files, token=self._token)
        return res

    # ----------------------------------------------------------------------
    def _list_attachments(self, oid):
        """ list attachments for a given OBJECT ID """

        params = {
            "f": "json"
        }
        if self._dynamic_layer is not None:
            url = self.url.split('?')[0] + "/%s/attachments" % oid
            params['layer'] = self._dynamic_layer
        else:
            url = self._url + "/%s/attachments" % oid
        return self._con.get(path=url, params=params, token=self._token)

    # ----------------------------------------------------------------------
    def get_unique_values(self, attribute, query_string='1=1'):
        """Return a list of unique values for a given attribute

        ===============================     ====================================================================
        **Argument**                        **Description**
        -------------------------------     --------------------------------------------------------------------
        attribute                           Required string. The feature layer attribute to query.
        -------------------------------     --------------------------------------------------------------------
        query_string                        Optional string. SQL Query that will be used to filter attributes
                                            before unique values are returned.
                                            ex. "name_2 like '%K%'"
        ===============================     ====================================================================
        """

        result = self.query(query_string, return_geometry=False, out_fields=attribute,return_distinct_values=True)
        return [feature.attributes[attribute] for feature in result.features]

    # ----------------------------------------------------------------------
    def query(self,
              where="1=1",
              out_fields="*",
              time_filter=None,
              geometry_filter=None,
              return_geometry=True,
              return_count_only=False,
              return_ids_only=False,
              return_distinct_values=False,
              return_extent_only=False,
              group_by_fields_for_statistics=None,
              statistic_filter=None,
              result_offset=None,
              result_record_count=None,
              object_ids=None,
              distance=None,
              units=None,
              max_allowable_offset=None,
              out_sr=None,
              geometry_precision=None,
              gdb_version=None,
              order_by_fields=None,
              out_statistics=None,
              return_z=False,
              return_m=False,
              multipatch_option=None,
              quantization_parameters=None,
              return_centroid=False,
              return_all_records=True,
              result_type=None,
              historic_moment=None,
              sql_format=None,
              return_true_curves=False,
              return_exceeded_limit_features=None,
              as_df=False,
              **kwargs):
        """
        Queries a feature layer based on a sql statement

        ===============================     ====================================================================
        **Argument**                        **Description**
        -------------------------------     --------------------------------------------------------------------
        where                               Optional string. The default is 1=1. The selection sql statement.
        -------------------------------     --------------------------------------------------------------------
        out_fields                          Optional List of field names to return. Field names can be specified
                                            either as a List of field names or as a comma separated string.
                                            The default is "*", which returns all the fields.
        -------------------------------     --------------------------------------------------------------------
        object_ids                          Optional string. The object IDs of this layer or table to be queried.
                                            The object ID values should be a comma-separated string.
        -------------------------------     --------------------------------------------------------------------
        distance                            Optional integer. The buffer distance for the input geometries.
                                            The distance unit is specified by units. For example, if the
                                            distance is 100, the query geometry is a point, units is set to
                                            meters, and all points within 100 meters of the point are returned.
        -------------------------------     --------------------------------------------------------------------
        units                               Optional string. The unit for calculating the buffer distance. If
                                            unit is not specified, the unit is derived from the geometry spatial
                                            reference. If the geometry spatial reference is not specified, the
                                            unit is derived from the feature service data spatial reference.
                                            This parameter only applies if supportsQueryWithDistance is true.
                                            Values: esriSRUnit_Meter | esriSRUnit_StatuteMile |
                                                    esriSRUnit_Foot | esriSRUnit_Kilometer |
                                                    esriSRUnit_NauticalMile | esriSRUnit_USNauticalMile
        -------------------------------     --------------------------------------------------------------------
        time_filter                         Optional list. The format is of [<startTime>, <endTime>] using
                                            datetime.date, datetime.datetime or timestamp in milliseconds.
                                            Syntax: time_filter=[<startTime>, <endTime>] ; specified as
                                                    datetime.date, datetime.datetime or timestamp in
                                                    milliseconds
        -------------------------------     --------------------------------------------------------------------
        geometry_filter                     Optional from arcgis.geometry.filter. Allows for the information to
                                            be filtered on spatial relationship with another geometry.
        -------------------------------     --------------------------------------------------------------------
        max_allowable_offset                Optional float. This option can be used to specify the
                                            max_allowable_offset to be used for generalizing geometries returned
                                            by the query operation.
                                            The max_allowable_offset is in the units of out_sr. If out_sr is not
                                            specified, max_allowable_offset is assumed to be in the unit of the
                                            spatial reference of the layer.
        -------------------------------     --------------------------------------------------------------------
        out_sr                              Optional Integer. The WKID for the spatial reference of the returned
                                            geometry.
        -------------------------------     --------------------------------------------------------------------
        geometry_precision                  Optional Integer. This option can be used to specify the number of
                                            decimal places in the response geometries returned by the query
                                            operation.
                                            This applies to X and Y values only (not m or z-values).
        -------------------------------     --------------------------------------------------------------------
        gdb_version                         Optional string. The geodatabase version to query. This parameter
                                            applies only if the isDataVersioned property of the layer is true.
                                            If this is not specified, the query will apply to the published
                                            map's version.
        -------------------------------     --------------------------------------------------------------------
        return_geometry                     Optional boolean. If true, geometry is returned with the query.
                                            Default is true.
        -------------------------------     --------------------------------------------------------------------
        return_distinct_values              Optional boolean.  If true, it returns distinct values based on the
                                            fields specified in out_fields. This parameter applies only if the
                                            supportsAdvancedQueries property of the layer is true.
        -------------------------------     --------------------------------------------------------------------
        return_ids_only                     Optional boolean. Default is False.  If true, the response only
                                            includes an array of object IDs. Otherwise, the response is a
                                            feature set.
        -------------------------------     --------------------------------------------------------------------
        return_count_only                   Optional boolean. If true, the response only includes the count
                                            (number of features/records) that would be returned by a query.
                                            Otherwise, the response is a feature set. The default is false. This
                                            option supersedes the returnIdsOnly parameter. If
                                            returnCountOnly = true, the response will return both the count and
                                            the extent.
        -------------------------------     --------------------------------------------------------------------
        return_extent_only                  Optional boolean. If true, the response only includes the extent of
                                            the features that would be returned by the query. If
                                            returnCountOnly=true, the response will return both the count and
                                            the extent.
                                            The default is false. This parameter applies only if the
                                            supportsReturningQueryExtent property of the layer is true.
        -------------------------------     --------------------------------------------------------------------
        order_by_fields                     Optional string. One or more field names on which the
                                            features/records need to be ordered. Use ASC or DESC for ascending
                                            or descending, respectively, following every field to control the
                                            ordering.
                                            example: STATE_NAME ASC, RACE DESC, GENDER
        -------------------------------     --------------------------------------------------------------------
        group_by_fields_for_statistics      Optional string. One or more field names on which the values need to
                                            be grouped for calculating the statistics.
                                            example: STATE_NAME, GENDER
        -------------------------------     --------------------------------------------------------------------
        out_statistics                      Optional string. The definitions for one or more field-based
                                            statistics to be calculated.

                                            Syntax:

                                            [
                                                {
                                                  "statisticType": "<count | sum | min | max | avg | stddev | var>",
                                                  "onStatisticField": "Field1",
                                                  "outStatisticFieldName": "Out_Field_Name1"
                                                },
                                                {
                                                  "statisticType": "<count | sum | min | max | avg | stddev | var>",
                                                  "onStatisticField": "Field2",
                                                  "outStatisticFieldName": "Out_Field_Name2"
                                                }
                                            ]
        -------------------------------     --------------------------------------------------------------------
        return_z                            Optional boolean. If true, Z values are included in the results if
                                            the features have Z values. Otherwise, Z values are not returned.
                                            The default is False.
        -------------------------------     --------------------------------------------------------------------
        return_m                            Optional boolean. If true, M values are included in the results if
                                            the features have M values. Otherwise, M values are not returned.
                                            The default is false.
        -------------------------------     --------------------------------------------------------------------
        multipatch_option                   Optional x/y footprint. This option dictates how the geometry of
                                            a multipatch feature will be returned.
        -------------------------------     --------------------------------------------------------------------
        result_offset                       Optional integer. This option can be used for fetching query results
                                            by skipping the specified number of records and starting from the
                                            next record (that is, resultOffset + 1th). This option is ignored
                                            if return_all_records is True (i.e. by default).
        -------------------------------     --------------------------------------------------------------------
        result_record_count                 Optional integer. This option can be used for fetching query results
                                            up to the result_record_count specified. When result_offset is
                                            specified but this parameter is not, the map service defaults it to
                                            max_record_count. The maximum value for this parameter is the value
                                            of the layer's max_record_count property. This option is ignored if
                                            return_all_records is True (i.e. by default).
        -------------------------------     --------------------------------------------------------------------
        quantization_parameters             Optional dict. Used to project the geometry onto a virtual grid,
                                            likely representing pixels on the screen.
        -------------------------------     --------------------------------------------------------------------
        return_centroid                     Optional boolean. Used to return the geometry centroid associated
                                            with each feature returned. If true, the result includes the geometry
                                            centroid. The default is false.
        -------------------------------     --------------------------------------------------------------------
        return_all_records                  Optional boolean. When True, the query operation will call the
                                            service until all records that satisfy the where_clause are
                                            returned. Note: result_offset and result_record_count will be
                                            ignored if return_all_records is True. Also, if return_count_only,
                                            return_ids_only, or return_extent_only are True, this parameter
                                            will be ignored.
        -------------------------------     --------------------------------------------------------------------
        result_type                         Optional string. The result_type parameter can be used to control
                                            the number of features returned by the query operation.
                                            Values: None | standard | tile
        -------------------------------     --------------------------------------------------------------------
        historic_moment                     Optional integer. The historic moment to query. This parameter
                                            applies only if the layer is archiving enabled and the
                                            supportsQueryWithHistoricMoment property is set to true. This
                                            property is provided in the layer resource.

                                            If historic_moment is not specified, the query will apply to the
                                            current features.
        -------------------------------     --------------------------------------------------------------------
        sql_format                          Optional string.  The sql_format parameter can be either standard
                                            SQL92 standard or it can use the native SQL of the underlying
                                            datastore native. The default is none which means the sql_format
                                            depends on useStandardizedQuery parameter.
                                            Values: none | standard | native
        -------------------------------     --------------------------------------------------------------------
        return_true_curves                  Optional boolean. When set to true, returns true curves in output
                                            geometries. When set to false, curves are converted to densified
                                            polylines or polygons.
        -------------------------------     --------------------------------------------------------------------
        return_exceeded_limit_features      Optional boolean. Optional parameter which is true by default. When
                                            set to true, features are returned even when the results include
                                            'exceededTransferLimit': True.

                                            When set to false and querying with resultType = tile features are
                                            not returned when the results include 'exceededTransferLimit': True.
                                            This allows a client to find the resolution in which the transfer
                                            limit is no longer exceeded without making multiple calls.
        -------------------------------     --------------------------------------------------------------------
        as_df                               Optional boolean.  If True, the results are returned as a DataFrame
                                            instead of a FeatureSet.
        -------------------------------     --------------------------------------------------------------------
        kwargs                              Optional dict. Optional parameters that can be passed to the Query
                                            function.  This will allow users to pass additional parameters not
                                            explicitly implemented on the function. A complete list of functions
                                            available is documented on the Query REST API.
        ===============================     ====================================================================

        :returns: A FeatureSet containing the features matching the query unless another return type is specified, such as count
        """
        as_raw = as_df
        if self._dynamic_layer is None:
            url = self._url + "/query"
        else:
            url = "%s/query" % self._url.split('?')[0]

        params = {"f": "json"}
        if self._dynamic_layer is not None:
            params['layer'] = self._dynamic_layer
        if result_type is not None:
            params['resultType'] = result_type
        if historic_moment is not None:
            params['historicMoment'] = historic_moment
        if sql_format is not None:
            params['sqlFormat'] = sql_format
        if return_true_curves is not None:
            params['returnTrueCurves'] = return_true_curves
        if return_exceeded_limit_features is not None:
            params['returnExceededLimitFeatures'] = return_exceeded_limit_features
        params['where'] = where
        params['returnGeometry'] = return_geometry
        params['returnDistinctValues'] = return_distinct_values
        params['returnCentroid'] = return_centroid
        params['returnCountOnly'] = return_count_only
        params['returnExtentOnly'] = return_extent_only
        params['returnIdsOnly'] = return_ids_only
        params['returnZ'] = return_z
        params['returnM'] = return_m

        # convert out_fields to a comma separated string
        if isinstance(out_fields, (list, tuple)):
            out_fields = ','.join(out_fields)

        if out_fields != '*' and not return_distinct_values:
            try:
                # Check if object id field is in out_fields.
                # If it isn't, add it
                object_id_field = [x.name for x in self.properties.fields if x.type == "esriFieldTypeOID"][0]
                if object_id_field not in out_fields.split(','):
                    out_fields = object_id_field + "," + out_fields
            except (IndexError, AttributeError):
                pass
        params['outFields'] = out_fields
        if return_count_only or return_extent_only or return_ids_only:
            return_all_records = False
        if result_record_count and not return_all_records:
            params['resultRecordCount'] = result_record_count
        if result_offset and not return_all_records:
            params['resultOffset'] = result_offset
        if quantization_parameters:
            params['quantizationParameters'] = quantization_parameters
        if multipatch_option:
            params['multipatchOption'] = multipatch_option
        if order_by_fields:
            params['orderByFields'] = order_by_fields
        if group_by_fields_for_statistics:
            params['groupByFieldsForStatistics'] = group_by_fields_for_statistics
        if statistic_filter and \
                isinstance(statistic_filter, StatisticFilter):
            params['outStatistics'] = statistic_filter.filter
        if out_statistics:
            params['outStatistics'] = out_statistics
        if out_sr:
            params['outSR'] = out_sr
        if max_allowable_offset:
            params['maxAllowableOffset'] = max_allowable_offset
        if gdb_version:
            params['gdbVersion'] = gdb_version
        if geometry_precision:
            params['geometryPrecision'] = geometry_precision
        if object_ids:
            params['objectIds'] = object_ids
        if distance:
            params['distance'] = distance
        if units:
            params['units'] = units

        if time_filter is not None:
            if type(time_filter) is list:
                starttime = _date_handler(time_filter[0])
                endtime = _date_handler(time_filter[1])
                if starttime is None:
                    starttime = 'null'
                if endtime is None:
                    endtime = 'null'
                params['time'] = "%s,%s" % (starttime, endtime)
            elif isinstance(time_filter, dict):
                for key, val in time_filter.items():
                    params[key] = val
            else:
                params['time'] = _date_handler(time_filter)

        if geometry_filter and \
                isinstance(geometry_filter, GeometryFilter):
            for key, val in geometry_filter.filter:
                params[key] = val
        elif geometry_filter and \
                isinstance(geometry_filter, dict):
            for key, val in geometry_filter.items():
                params[key] = val
        if len(kwargs) > 0:
            for key, val in kwargs.items():
                if key in ('returnCountOnly','returnExtentOnly','returnIdsOnly') and val:
                    # If these keys are passed in as kwargs instead of parameters, set return_all_records
                    return_all_records = False
                params[key] = val
                del key, val

        if not return_all_records:
            return self._query(url, params, raw=as_raw)

        params['returnCountOnly'] = True
        record_count = self._query(url, params, raw=as_raw)
        if 'maxRecordCount' in self.properties:
            max_records = self.properties['maxRecordCount']
        else:
            max_records = 1000

        params['returnCountOnly'] = False
        if record_count <= max_records:
            if as_df:
                import pandas as pd
                df = self._query_df(url, params)
                dt_fields = [fld['name'] for fld in self.properties.fields \
                             if fld['type'] == 'esriFieldTypeDate']
                if 'SHAPE' in df.columns:
                    df.spatial.set_geometry('SHAPE')
                for fld in dt_fields:
                    try:
                        if fld in df.columns:
                            df[fld] = pd.to_datetime(df[fld]/1000,
                                                 infer_datetime_format=True,
                                                 unit='s')
                    except:
                        if fld in df.columns:
                            df[fld] = pd.to_datetime(df[fld], infer_datetime_format=True)
                return df

            return self._query(url, params, raw=as_raw)

        result = None
        if 'advancedQueryCapabilities' not in self.properties or \
                'supportsPagination' not in self.properties['advancedQueryCapabilities'] or \
                not self.properties['advancedQueryCapabilities']['supportsPagination']:
            params['returnIdsOnly'] = True
            oid_info = self._query(url, params, raw=as_raw)
            params['returnIdsOnly'] = False
            for ids in chunks(oid_info['objectIds'], max_records):
                ids = [str(i) for i in ids]
                sql = "%s in (%s)" % (oid_info['objectIdFieldName'], ",".join(ids))
                params['where'] = sql
                records = self._query(url, params, raw=as_raw)
                if result:
                    result.features.extend(records.features)
                else:
                    result = records
        else:
            i = 0
            count = 0
            df = None
            params['resultRecordCount'] = max_records
            dfs = []
            while True:
                params['resultOffset'] = max_records * i
                if as_df == False:
                    records = self._query(url, params, raw=as_raw)

                    if result:
                        result.features.extend(records.features)
                    else:
                        result = records

                    if len(records.features) < max_records:
                        break
                else:

                    df = self._query_df(url, params)
                    count += len(df)
                    dfs.append(df)
                    if count == record_count:
                        break
                i += 1
        if as_df:
            import pandas as pd
            dt_fields = [fld['name'] for fld in self.properties.fields \
                         if fld['type'] == 'esriFieldTypeDate']
            if len(dfs) == 1:
                df = dfs[0]
            else:
                df = pd.concat(dfs)
                df.reset_index(drop=True, inplace=True)
            if 'SHAPE' in df.columns:
                df.spatial.set_geometry('SHAPE')
            for fld in dt_fields:
                try:
                    df[fld] = pd.to_datetime(df[fld]/1000,
                                             infer_datetime_format=True,
                                             unit='s')
                except:
                    df[fld] = pd.to_datetime(df[fld], infer_datetime_format=True)
            return df
        return result
    # ----------------------------------------------------------------------
    def validate_sql(self, sql, sql_type="where"):
        """
        The validate_sql operation validates an SQL-92 expression or WHERE
        clause.
        The validate_sql operation ensures that an SQL-92 expression, such
        as one written by a user through a user interface, is correct
        before performing another operation that uses the expression. For
        example, validateSQL can be used to validate information that is
        subsequently passed in as part of the where parameter of the
        calculate operation.
        validate_sql also prevents SQL injection. In addition, all table
        and field names used in the SQL expression or WHERE clause are
        validated to ensure they are valid tables and fields.


        ===============================     ====================================================================
        **Argument**                        **Description**
        -------------------------------     --------------------------------------------------------------------
        sql                                 Required String. The SQL expression of WHERE clause to validate.
                                            Example: "Population > 300000"
        -------------------------------     --------------------------------------------------------------------
        sql_type                            Optional String. Three SQL types are supported in validate_sql
                                                - where (default) - Represents the custom WHERE clause the user
                                                  can compose when querying a layer or using calculate.
                                                - expression - Represents an SQL-92 expression. Currently,
                                                  expression is used as a default value expression when adding a
                                                  new field or using the calculate API.
                                                - statement - Represents the full SQL-92 statement that can be
                                                  passed directly to the database. No current ArcGIS REST API
                                                  resource or operation supports using the full SQL-92 SELECT
                                                  statement directly. It has been added to the validateSQL for
                                                  completeness.
                                                  Values: where | expression | statement
        ===============================     ====================================================================

        :returns: dict
        """
        params = {
            "f" : "json"
        }
        if not isinstance(sql, six.string_types):
            raise ValueError("sql must be a string")
        else:
            params['sql'] = sql
        if sql_type.lower() not in ['where', 'expression', 'statement']:
            raise ValueError("sql_type must have value of: where, expression or statement")
        else:
            params['sqlType'] = sql_type
        sql_type = sql_type.lower()
        url = self._url + "/validateSQL"
        return self._con.post(path=url,
                              postdata=params,
                              token=self._token)
    # ----------------------------------------------------------------------
    def query_related_records(self,
                              object_ids,
                              relationship_id,
                              out_fields="*",
                              definition_expression=None,
                              return_geometry=True,
                              max_allowable_offset=None,
                              geometry_precision=None,
                              out_wkid=None,
                              gdb_version=None,
                              return_z=False,
                              return_m=False):
        """
        The Query operation is performed on a feature service layer
        resource. The result of this operation are feature sets grouped
        by source layer/table object IDs. Each feature set contains
        Feature objects including the values for the fields requested by
        the user. For related layers, if you request geometry
        information, the geometry of each feature is also returned in
        the feature set. For related tables, the feature set does not
        include geometries.

        ======================     ====================================================================
        **Argument**               **Description**
        ----------------------     --------------------------------------------------------------------
        object_ids                 Required string. The object IDs of the table/layer to be queried
        ----------------------     --------------------------------------------------------------------
        relationship_id            Required string. The ID of the relationship to be queried.
        ----------------------     --------------------------------------------------------------------
        out_fields                 Required string. the list of fields from the related table/layer
                                   to be included in the returned feature set. This list is a comma
                                   delimited list of field names. If you specify the shape field in the
                                   list of return fields, it is ignored. To request geometry, set
                                   return_geometry to true. You can also specify the wildcard "*" as
                                   the value of this parameter. In this case, the results will include
                                   all the field values.
        ----------------------     --------------------------------------------------------------------
        definition_expression      Optional string. The definition expression to be applied to the
                                   related table/layer. From the list of objectIds, only those records
                                   that conform to this expression are queried for related records.
        ----------------------     --------------------------------------------------------------------
        return_geometry            Optional boolean. If true, the feature set includes the geometry
                                   associated with each feature. The default is true.
        ----------------------     --------------------------------------------------------------------
        max_allowable_offset       Optional float. This option can be used to specify the
                                   max_allowable_offset to be used for generalizing geometries returned
                                   by the query operation. The max_allowable_offset is in the units of
                                   the outSR. If out_wkid is not specified, then max_allowable_offset
                                   is assumed to be in the unit of the spatial reference of the map.
        ----------------------     --------------------------------------------------------------------
        geometry_precision         Optional integer. This option can be used to specify the number of
                                   decimal places in the response geometries.
        ----------------------     --------------------------------------------------------------------
        out_wkid                   Optional Integer. The spatial reference of the returned geometry.
        ----------------------     --------------------------------------------------------------------
        gdb_version                Optional string. The geodatabase version to query. This parameter
                                   applies only if the isDataVersioned property of the layer queried is
                                   true.
        ----------------------     --------------------------------------------------------------------
        return_z                   Optional boolean. If true, Z values are included in the results if
                                   the features have Z values. Otherwise, Z values are not returned.
                                   The default is false.
        ----------------------     --------------------------------------------------------------------
        return_m                   Optional boolean. If true, M values are included in the results if
                                   the features have M values. Otherwise, M values are not returned.
                                   The default is false.
        ======================     ====================================================================


        :return: dict


        """
        params = {
            "f": "json",
            "objectIds": object_ids,
            "relationshipId": relationship_id,
            "outFields": out_fields,
            "returnGeometry": return_geometry,
            "returnM": return_m,
            "returnZ": return_z
        }
        if self._dynamic_layer is not None:
            params['layer'] = self._dynamic_layer
        if gdb_version is not None:
            params['gdbVersion'] = gdb_version
        if definition_expression is not None:
            params['definitionExpression'] = definition_expression
        if out_wkid is not None and \
                isinstance(out_wkid, SpatialReference):
            params['outSR'] = out_wkid
        elif out_wkid is not None and \
                isinstance(out_wkid, dict):
            params['outSR'] = out_wkid
        if max_allowable_offset is not None:
            params['maxAllowableOffset'] = max_allowable_offset
        if geometry_precision is not None:
            params['geometryPrecision'] = geometry_precision
        if self._dynamic_layer is None:
            qrr_url = self._url + "/queryRelatedRecords"
        else:
            qrr_url = "%s/queryRelatedRecords" % self._url.split('?')[0]

        return self._con.post(path=qrr_url, postdata=params, token=self._token)

    # ----------------------------------------------------------------------
    def get_html_popup(self, oid):
        """
        The htmlPopup resource provides details about the HTML pop-up
        authored by the user using ArcGIS for Desktop.

        ===============     ====================================================================
        **Argument**        **Description**
        ---------------     --------------------------------------------------------------------
        oid                 Optional string. Object id of the feature to get the HTML popup.
        ===============     ====================================================================


        :return: string

        """
        if self.properties.htmlPopupType != "esriServerHTMLPopupTypeNone":
            pop_url = self._url + "/%s/htmlPopup" % oid
            params = {
                'f': "json"
            }

            return self._con.get(path=pop_url, params=params, token=self._token)
        return ""
    # ----------------------------------------------------------------------
    def append(self,
               item_id=None,
               upload_format="featureCollection",
               source_table_name=None,
               field_mappings=None,
               edits=None,
               source_info=None,
               upsert=True,
               skip_updates=False,
               use_globalids=False,
               update_geometry=True,
               append_fields=None,
               rollback=False,
               skip_inserts=None,
               upsert_matching_field=None
               ):
        """
        Only available in ArcGIS Online

        Update an existing hosted feature layer using append.

        ========================   ====================================================================
        **Argument**               **Description**
        ------------------------   --------------------------------------------------------------------
        source_table_name          optional string. Required only when the source data contains more
                                   than one tables, e.g., for file geodatabase.
                                   Example: source_table_name=  "Building"
        ------------------------   --------------------------------------------------------------------
        item_id                    optional string. The ID for the Portal item that contains the source
                                   file.
                                   Used in conjunction with editsUploadFormat.
        ------------------------   --------------------------------------------------------------------
        field_mappings             optional list. Used to map source data to a destination layer.
                                   Syntax: fieldMappings=[{"name" : <"targerName">,
                                                           "sourceName" : < "sourceName">}, ...]
                                   Examples: fieldMappings=[{"name" : "CountyID",
                                                             "sourceName" : "GEOID10"}]
        ------------------------   --------------------------------------------------------------------
        edits                      optional string. Only feature collection json is supported. Append
                                   supports all format through the upload_id or item_id.
        ------------------------   --------------------------------------------------------------------
        source_info                optional dictionary. This is only needed when appending data from
                                   excel or csv. The appendSourceInfo can be the publishing parameter
                                   returned from analyze the csv or excel file.
        ------------------------   --------------------------------------------------------------------
        upsert                     optional boolean. Optional parameter specifying whether the edits
                                   needs to be applied as updates if the feature already exists.
                                   Default is false.
        ------------------------   --------------------------------------------------------------------
        skip_updates               Optional boolean. Parameter is used only when upsert is true.
        ------------------------   --------------------------------------------------------------------
        use_globalids              Optional boolean. Specifying whether upsert needs to use GlobalId
                                   when matching features.
        ------------------------   --------------------------------------------------------------------
        update_geometry            Optional boolean. The parameter is used only when upsert is true.
                                   Skip updating the geometry and update only the attributes for
                                   existing features if they match source features by objectId or
                                   globalId.(as specified by useGlobalIds parameter).
        ------------------------   --------------------------------------------------------------------
        append_fields              Optional list. The list of destination fields to append to. This is
                                   supported when upsert=true or false.
                                   Values:  ["fieldName1", "fieldName2",....]
        ------------------------   --------------------------------------------------------------------
        upload_format              required string. The source append data format. The default is
                                   featureCollection format.
                                   Values: sqlite | shapefile | filegdb | featureCollection |
                                   geojson | csv | excel
        ------------------------   --------------------------------------------------------------------
        rollback                   Optional boolean. Optional parameter specifying whether the upsert
                                   edits needs to be rolled back in case of failure. Default is false.
        ------------------------   --------------------------------------------------------------------
        skip_inserts               Used only when upsert is true. Used to skip inserts if the value is
                                   true. The default value is false.
        ------------------------   --------------------------------------------------------------------
        upsert_matching_field      Optional string. The layer field to be used when matching features
                                   with upsert. ObjectId, GlobalId, and any other field that has a
                                   unique index can be used with upsert.
                                   This parameter overrides use_globalids; e.g., specifying
                                   upsert_matching_field will be used even if you specify
                                   use_globalids = True.
                                   Example: upsert_matching_field="MyfieldWithUniqueIndex"
        ========================   ====================================================================


        :return: boolean

        """
        import copy
        upload_id = None
        if ((hasattr(self._gis, '_portal') and self._gis._portal.is_logged_in == False) or \
           (hasattr(self._gis, 'is_logged_in') and self._gis.is_logged_in == False)):
            raise Exception("Authentication required to perform append.")
        if self._gis._portal.is_arcgisonline == False:
            raise Exception("Append only available on ArcGIS Online.")
        if self.properties.supportsAppend == False:
            raise Exception("Append is not supported on this layer, please " +\
                            "update service definition capabilities.")
        params = {
            'f' : 'json',
            'sourceTableName' : source_table_name,
            'fieldMappings' : field_mappings,
            'edits' : edits,
            'appendSourceInfo' : source_info,
            'upsert' : upsert,
            'skipUpdates' : skip_updates,
            'useGlobalIds' : use_globalids,
            'updateGeometry' : update_geometry,
            'appendFields' : append_fields,
            'appendUploadId' : upload_id,
            'appendItemId' : item_id,
            'appendUploadFormat' : upload_format,
            'rollbackOnFailure' : rollback
        }
        if not upsert_matching_field is None:
            params['upsertMatchingField'] =  upsert_matching_field
        if not skip_inserts is None:
            params['skipInserts'] =  skip_inserts
        upload_formats = """sqlite,shapefile,filegdb,featureCollection,geojson,csv,excel""".split(',')
        if upload_format not in upload_formats:
            raise ValueError("Invalid upload format: %s." % upload_format)
        cparams = copy.copy(params)
        for k,v in cparams.items():
            if v is None:
                params.pop(k)
            del k, v
        url = self._url + "/append"
        del cparams
        res = self._con.post(path=url,
                             postdata=params)
        if 'statusUrl' in res:
            surl = res['statusUrl']
            sres = self._con.get(path=surl, params={'f' : 'json'})
            while sres['status'].lower() != "completed":
                sres = self._con.get(path=surl, params={'f' : 'json'})
                if sres['status'].lower() in "failed":
                    break
            return True
        return res
    # ----------------------------------------------------------------------
    def delete_features(self,
                        deletes=None,
                        where=None,
                        geometry_filter=None,
                        gdb_version=None,
                        rollback_on_failure=True):
        """
        This operation deletes features in a feature layer or table

        ====================     ====================================================================
        **Argument**             **Description**
        --------------------     --------------------------------------------------------------------
        deletes                  Optional string. A comma seperated string of OIDs to remove from the
                                 service.
        --------------------     --------------------------------------------------------------------
        where                    Optional string.  A where clause for the query filter. Any legal SQL
                                 where clause operating on the fields in the layer is allowed.
                                 Features conforming to the specified where clause will be deleted.
        --------------------     --------------------------------------------------------------------
        geometry_filter          Optional SpatialFilter. A spatial filter from
                                 arcgis.geometry.filters module to filter results by a spatial
                                 relationship with another geometry.
        --------------------     --------------------------------------------------------------------
        gdb_version              Optional string. A Geodatabase version to apply the edits.
        --------------------     --------------------------------------------------------------------
        rollback_on_failure      Optional boolean. Optional parameter to specify if the edits should
                                 be applied only if all submitted edits succeed. If false, the server
                                 will apply the edits that succeed even if some of the submitted
                                 edits fail. If true, the server will apply the edits only if all
                                 edits succeed. The default value is true.
        ====================     ====================================================================


        :return: dict


        """
        delete_url = self._url + "/deleteFeatures"
        params = {
            "f": "json",
            "rollbackOnFailure": rollback_on_failure
        }
        if gdb_version is not None:
            params['gdbVersion'] = gdb_version

        if deletes is not None and \
                isinstance(deletes, str):
            params['objectIds'] = deletes
        elif deletes is not None and \
                isinstance(deletes, PropertyMap):
            print('pass in delete, unable to convert PropertyMap to string list of OIDs')

        elif deletes is not None and \
                isinstance(deletes, FeatureSet):
            params['objectIds'] = ",".join(
                [str(feat.get_value(field_name=deletes.object_id_field_name)) for feat in deletes.features])

        if where is not None:
            params['where'] = where

        if geometry_filter is not None and \
                isinstance(geometry_filter, GeometryFilter):
            for key, val in geometry_filter.filter:
                params[key] = val
        elif geometry_filter is not None and \
                isinstance(geometry_filter, dict):
            for key, val in geometry_filter.items():
                params[key] = val

        if 'objectIds' not in params and 'where' not in params and 'geometry' not in params:
            print("Parameters not valid for delete_features")
            return None
        return self._con.post(path=delete_url, postdata=params, token=self._token)

    # ----------------------------------------------------------------------
    def edit_features(self,
                      adds=None,
                      updates=None,
                      deletes=None,
                      gdb_version=None,
                      use_global_ids=False,
                      rollback_on_failure=True):
        """
           This operation adds, updates, and deletes features to the
           associated feature layer or table in a single call.

           =====================   ===========================================
           **Inputs**              **Description**
           ---------------------   -------------------------------------------
           adds                    Optional FeatureSet/List. The array of
                                   features to be added.
           ---------------------   -------------------------------------------
           updates                 Optional FeatureSet/List. The array of
                                   features to be updated.
           ---------------------   -------------------------------------------
           deletes                 Optional FeatureSet/List. string of OIDs to
                                   remove from service
           ---------------------   -------------------------------------------
           use_global_ids          Optional boolean. Instead of referencing
                                   the default Object ID field, the service
                                   will look at a GUID field to track changes.
                                   This means the GUIDs will be passed instead
                                   of OIDs for delete, update or add features.
           ---------------------   -------------------------------------------
           gdb_version             Optional boolean. Geodatabase version to
                                   apply the edits.
           ---------------------   -------------------------------------------
           rollback_on_failure     Optional boolean. Optional parameter to
                                   specify if the edits should be applied only
                                   if all submitted edits succeed. If false, the
                                   server will apply the edits that succeed
                                   even if some of the submitted edits fail.
                                   If true, the server will apply the edits
                                   only if all edits succeed. The default
                                   value is true.
           =====================   ===========================================

           Output: dictionary
        """
        if adds is None:
            adds = []
        if updates is None:
            updates = []
        edit_url = self._url + "/applyEdits"
        params = {
            "f": "json",
            "useGlobalIds": use_global_ids,
            "rollbackOnFailure": rollback_on_failure
        }
        if gdb_version is not None:
            params['gdbVersion'] = gdb_version
        if isinstance(adds, FeatureSet):
            params['adds'] = json.dumps([f.as_dict for f in adds.features],
                                        default=_date_handler)
        elif len(adds) > 0:
            if isinstance(adds[0], dict):
                params['adds'] = json.dumps([f for f in adds],
                                            default=_date_handler)
            elif isinstance(adds[0], PropertyMap):
                params['adds'] = json.dumps([dict(f) for f in adds],
                                            default=_date_handler)
            elif isinstance(adds[0], Feature):
                params['adds'] = json.dumps([f.as_dict for f in adds],
                                               default=_date_handler)
            else:
                print('pass in features as list of Features, dicts or PropertyMap')
        if isinstance(updates, FeatureSet):
            params['updates'] = json.dumps([f.as_dict for f in updates.features],
                                           default=_date_handler)
        elif len(updates) > 0:
            if isinstance(updates[0], dict):
                params['updates'] = json.dumps([f for f in updates],
                                               default=_date_handler)
            elif isinstance(updates[0], PropertyMap):
                params['updates'] = json.dumps([dict(f) for f in updates],
                                               default=_date_handler)
            elif isinstance(updates[0], Feature):
                params['updates'] = json.dumps([f.as_dict for f in updates],
                                               default=_date_handler)
            else:
                print('pass in features as list of Features, dicts or PropertyMap')
        if deletes is not None and \
                isinstance(deletes, str):
            params['deletes'] = deletes
        elif deletes is not None and \
                isinstance(deletes, PropertyMap):
            print('pass in delete, unable to convert PropertyMap to string list of OIDs')

        elif deletes is not None and \
                isinstance(deletes, FeatureSet):

            field_name = None
            if deletes.object_id_field_name:
                field_name = deletes.object_id_field_name
            elif self.properties.objectIdField in deletes.fields:
                field_name = self.properties.objectIdField
            else:
                print('deletes FeatureSet must have object_id_field_name parameter set')

            if field_name:
                params['deletes'] = ",".join([str(feat.get_value(field_name=field_name)) for feat in deletes.features])

        if 'deletes' not in params and 'updates' not in params and 'adds' not in params:
            print("Parameters not valid for edit_features")
            return None
        return self._con.post(path=edit_url, postdata=params, token=self._token)

    # ----------------------------------------------------------------------
    def calculate(self, where, calc_expression, sql_format="standard"):
        """
        The calculate operation is performed on a feature layer
        resource. It updates the values of one or more fields in an
        existing feature service layer based on SQL expressions or scalar
        values. The calculate operation can only be used if the
        supportsCalculate property of the layer is true.
        Neither the Shape field nor system fields can be updated using
        calculate. System fields include ObjectId and GlobalId.
        See Calculate a field for more information on supported expressions

           =====================   ===========================================
           **Inputs**              **Description**
           ---------------------   -------------------------------------------
           where                   A where clause can be used to limit the updated records.
                                   Any legal SQL where clause operating on the fields in
                                   the layer is allowed.
           ---------------------   -------------------------------------------
           calc_expression         The array of field/value info objects that
                                   contain the field or fields to update and their
                                   scalar values or SQL expression.  Allowed types
                                   are dictionary and list.  List must be a list
                                   of dictionary objects.
                                   Calculation Format is as follows:
                                   {"field" : "<field name>",  "value" : "<value>"}
           ---------------------   -------------------------------------------
           sql_format              The SQL format for the calcExpression. It can be
                                   either standard SQL92 (standard) or native SQL
                                   (native). The default is standard.
                                   Values: standard, native
           =====================   ===========================================

        .. code-block:: python

            # Usage Example 1:

            print(fl.calculate(where="OBJECTID < 2",
                               calc_expression={"field": "ZONE", "value" : "R1"}))

        .. code-block:: python

            # Usage Example 2:

            print(fl.calculate(where="OBJECTID < 2001",
                               calc_expression={"field": "A",  "sqlExpression" : "B*3"}))

        Output: dictionary with format {'updatedFeatureCount': 1, 'success': True}
        """
        url = self._url + "/calculate"
        params = {
            "f": "json",
            "where": where,

        }
        if isinstance(calc_expression, dict):
            params["calcExpression"] = json.dumps([calc_expression],
                                                  default=_date_handler)
        elif isinstance(calc_expression, list):
            params["calcExpression"] = json.dumps(calc_expression,
                                                  default=_date_handler)
        if sql_format.lower() in ['native', 'standard']:
            params['sqlFormat'] = sql_format.lower()
        else:
            params['sqlFormat'] = "standard"
        return self._con.post(path=url,
                              postdata=params, token=self._token)

    # ----------------------------------------------------------------------
    def _query(self, url, params, raw=False):
        """ returns results of query """
        result = self._con.post(path=url,
                                postdata=params, token=self._token)
        if 'error' in result:
            raise ValueError(result)
        if  params['returnCountOnly']:
            return result['count']
        elif params['returnIdsOnly']:
            return result
        elif 'extent' in result:
            return result
        elif raw:
            return result
        else:
            return FeatureSet.from_dict(result)
    def _query_df(self, url, params):
        """ returns results of a query as a pd.DataFrame"""
        import pandas as pd
        from arcgis.features import GeoAccessor, GeoSeriesAccessor
        import numpy as np
        _fld_lu = {
            "esriFieldTypeSmallInteger" : np.int32,
            "esriFieldTypeInteger" : np.int64,
            "esriFieldTypeSingle" : np.int32,
            "esriFieldTypeDouble" : float,
            "esriFieldTypeString" : str,
            "esriFieldTypeDate" : pd.datetime,
            "esriFieldTypeOID" : np.int64,
            "esriFieldTypeGeometry" : object,
            "esriFieldTypeBlob" : object,
            "esriFieldTypeRaster" : object,
            "esriFieldTypeGUID" : str,
            "esriFieldTypeGlobalID" : str,
            "esriFieldTypeXML" : object
        }
        def feature_to_row(feature, sr):
            """:return: a feature from a dict"""
            from arcgis.geometry import Geometry
            geom = feature['geometry'] if 'geometry' in feature else None
            attribs = feature['attributes'] if 'attributes' in feature else {}
            if 'centroid' in feature:
                if attribs is None:
                    attribs = {'centroid' : feature['centroid']}
                elif 'centroid' in attribs:
                    import uuid
                    fld = "centroid_" + uuid.uuid4().hex[:2]
                    attribs[fld] = feature['centroid']
                else:
                    attribs['centroid'] = feature['centroid']
            if geom:
                if "spatialReference" not in geom:
                    geom["spatialReference"] = sr
                attribs['SHAPE'] = Geometry(geom)
            return attribs
        #------------------------------------------------------------------
        featureset_dict = self._con.post(url, params)
        sr = featureset_dict['spatialReference']
        df = None
        dtypes = None
        geom = None
        names = None


        rows = [feature_to_row(row, sr) \
                for row in featureset_dict['features']]
        if len(rows) == 0:
            return None
        df = pd.DataFrame.from_records(data=rows)
        if 'fields' in featureset_dict:
            dtypes = {}
            names = []
            fields = featureset_dict['fields']
            for fld in fields:
                if fld['type'] != "esriFieldTypeGeometry":
                    dtypes[fld['name']] = _fld_lu[fld['type']]
                    names.append(fld['name'])
                if fld['type'] in {"esriFieldTypeSmallInteger",
                                   "esriFieldTypeInteger",
                                   "esriFieldTypeSingle",
                                   "esriFieldTypeDouble"}:
                    q = df[fld['name']].isnull()
                    df.loc[q, fld['name']] = 0
        df = df.astype(dtypes, False)
        if 'geometryType' in featureset_dict:
            df.spatial.set_geometry('SHAPE')

        return df


class Table(FeatureLayer):
    """
    Tables represent entity classes with uniform properties. In addition to working with "entities with location" as
    features, the GIS can also work with non-spatial entities as rows in tables.

    Working with tables is similar to working with feature layers, except that the rows (Features) in a table do not
    have a geometry, and tables ignore any geometry related operation.
    """
    pass


class FeatureLayerCollection(_GISResource):
    """
    A FeatureLayerCollection is a collection of feature layers and tables, with the associated relationships among the entities.

    In a web GIS, a feature layer collection is exposed as a feature service with multiple feature layers.

    Instances of FeatureDatasets can be obtained from feature service Items in the GIS using
    `FeatureLayerCollection.fromitem(item)`, from feature service endpoints using the constructor, or by accessing the `dataset`
    attribute of feature layer objects.

    FeatureDatasets can be configured and managed using their `manager` helper object.

    If the dataset supports the sync operation, the `replicas` helper object allows management and synchronization of
    replicas for disconnected editing of the feature layer collection.

    Note: You can use the `layers` and `tables` property to get to the individual layers and tables in this
    feature layer collection.
    """
    _vermgr = None

    def __init__(self, url, gis=None):
        super(FeatureLayerCollection, self).__init__(url, gis)

        try:
            if self.properties.syncEnabled:
                self.replicas = SyncManager(self)
        except AttributeError:
            pass

        self._populate_layers()
        self._admin = None
        try:
            from arcgis.gis.server._service._adminfactory import AdminServiceGen
            self.service = AdminServiceGen(service=self, gis=gis)
        except: pass

    def _populate_layers(self):
        """
        populates the layers and tables for this feature service
        """
        layers = []
        tables = []

        for lyr in self.properties.layers:
            lyr = FeatureLayer(self.url + '/' + str(lyr.id), self._gis, self)
            layers.append(lyr)

        for lyr in self.properties.tables:
            lyr = Table(self.url + '/' + str(lyr.id), self._gis, self)
            tables.append(lyr)

        # fsurl = self.url + '/layers'
        # params = { "f" : "json" }
        # allayers = self._con.post(fsurl, params, token=self._token)

        # for layer in allayers['layers']:
        #    layers.append(FeatureLayer(self.url + '/' + str(layer['id']), self._gis))

        # for table in allayers['tables']:
        #    tables.append(FeatureLayer(self.url + '/' + str(table['id']), self._gis))

        self.layers = layers
        self.tables = tables

    @property
    def manager(self):
        """ helper object to manage the feature layer collection, update it's definition, etc """
        if self._admin is None:
            url = self._url
            res = search("/rest/", url).span()
            add_text = "admin/"
            part1 = url[:res[1]]
            part2 = url[res[1]:]
            admin_url = "%s%s%s" % (part1, add_text, part2)

            self._admin = FeatureLayerCollectionManager(admin_url, self._gis, self)
        return self._admin

    @property
    def versions(self):
        """
        Returns a `VersionManager` to create, update and use versions on a `FeatureLayerCollection`.
        If versioning is not enabled on the service, None is returned.
        """
        if "hasVersionedData" in self.properties and \
           self.properties.hasVersionedData == True:
            if self._vermgr is None:
                from ._version import VersionManager
                import os
                url = os.path.dirname(self.url) + "/VersionManagementServer"
                self._vermgr = VersionManager(url=url, gis=self._gis)
            return self._vermgr
        return None
    # ----------------------------------------------------------------------
    def extract_changes(self,
                        layers,
                        servergen,
                        queries=None,
                        geometry=None,
                        geometry_type=None,
                        version=None,
                        return_inserts=False,
                        return_updates=False,
                        return_deletes=False,
                        return_ids_only=False,
                        return_extent_only=False,
                        return_attachments=False,
                        attachments_by_url=False,
                        data_format="json",
                        change_extent_grid_cell=None):
        """
        Feature service change tracking is an efficient change tracking
        mechanism for applications. Applications can use change tracking to
        query changes that have been made to the layers and tables in the
        service. For enterprise geodatabase based feature services published
        from ArcGIS Pro 2.2 or higher, the ChangeTracking capability
        requires all layers and tables to be either archive enabled or
        branch versioned and have globalid columns. Change tracking can also
        be enabled for ArcGIS Online hosted feature services. If all layers
        and tables in the service have the ChangeTracking capability, the
        `extract_changes` operation can be used to get changes.


        ================================     ====================================================================
        **Argument**                         **Description**
        --------------------------------     --------------------------------------------------------------------
        layers                               Required List.  The list of layers and tables to include in the
                                             replica.
        --------------------------------     --------------------------------------------------------------------
        servergen                            Required List.   The servergens allows a client to specify the last
                                             layer generation numbers for the changes received from the server.

                                                + minServerGen: It is the min generation of the server data changes.
                                                  clients with layerServerGens that is less than minServerGen cannot
                                                  extract changes and would need to make a full server/layers query
                                                  instead of extracting changes.
                                                + serverGen: It is the current server generation number of the
                                                  changes. Every changed feature has a version or a generation number
                                                  that is changed every time the feature is updated.

                                             Syntax: servergen = [{"id": <layerId1>, "serverGen": <genNum1>},
                                                                 {"id": <layerId2>, "serverGen": <genNum2>}]

                                             Example: servergen= [{"id": 0, "serverGen": 10500},
                                                                 {"id": 1, "serverGen": 1100},
                                                                 {"id": 2, "serverGen": 1200}]
        --------------------------------     --------------------------------------------------------------------
        queries                              Optional Dictionary. In addition to the layers and geometry
                                             parameters, the `queries` parameter can be used to further define
                                             what changes to return. This parameter allows you to set query
                                             properties on a per-layer or per-table basis. If a layer's ID is
                                             present in the layers parameter and missing from layer `queries`,
                                             it's changed features that intersect with the filter geometry are
                                             returned.

                                             The properties include the following:

                                                + where - Defines an attribute query for a layer or table. The
                                                  default is no where clause.
                                                + useGeometry - Determines whether or not to apply the geometry
                                                  for the layer. The default is true. If set to false, features
                                                  from the layer that intersect the geometry are not added.
                                                + includeRelated - Determines whether or not to add related
                                                  rows. The default is true. The value true is honored only
                                                  for queryOption=none. This is only applicable if your data
                                                  has relationship classes. Relationships are only processed
                                                  in a forward direction from origin to destination.
                                                + queryOption - Defines whether or how filters will be applied
                                                  to a layer. The queryOption was added in 10.2. See the
                                                  Compatibility notes topic for more information.

                                             Values: None, useFilter, or all

                                             When is value is none, no feature are returned based on where and
                                             filter geometry. if includeRelated is false, no features are
                                             returned. If includeRelated is true, features in this layer (that
                                             are related to the features in other layers in the replica) are
                                             returned.

                                             When the value is useFilter, features that satisfy filtering based
                                             on geometry and where are returned. The value of includeRelated is
                                             ignored.

                                             Syntax: queries={Layer_or_tableID1:{"where":"attribute query",
                                             "useGeometry": true | false, "includeRelated": true | false},
                                             Layer_or_tableID2: {.}}
        --------------------------------     --------------------------------------------------------------------
        geometry                             Option Geometry/Extent. The geometry to apply as the spatial filter
                                             for the changes. All the changed features in layers intersecting
                                             this geometry will be returned. The structure of the geometry is the
                                             same as the structure of the JSON geometry objects returned by the
                                             ArcGIS REST API. In addition to the JSON structures, for envelopes
                                             and points, you can specify the geometry with a simpler
                                             comma-separated syntax.
        --------------------------------     --------------------------------------------------------------------
        geometry_type                        Optional String. The type of geometry specified by the geometry
                                             parameter. The geometry type can be an envelope, point, line or
                                             polygon. The default geometry type is an envelope.

                                             Values: esriGeometryPoint, esriGeometryMultipoint,
                                                     esriGeometryPolyline, esriGeometryPolygon,
                                                     esriGeometryEnvelope
        --------------------------------     --------------------------------------------------------------------
        in_sr                                Optional Integer. The spatial reference of the input geometry.
        --------------------------------     --------------------------------------------------------------------
        out_sr                               Optional Integer/String. The output spatial reference of the
                                             returned changes.
        --------------------------------     --------------------------------------------------------------------
        version                              Optional String. If branch versioning is enabled, a user can specify
                                             the branch version name to extract changes from.
        --------------------------------     --------------------------------------------------------------------
        return_inserts                       Optional Boolean.  If true, newly inserted features will be
                                             returned. The default is false.
        --------------------------------     --------------------------------------------------------------------
        return_updates                       Optional Boolean. If true, updated features will be returned. The
                                             default is false.
        --------------------------------     --------------------------------------------------------------------
        return_deletes                       Optional Boolean. If true, deleted features will be returned. The
                                             default is false.
        --------------------------------     --------------------------------------------------------------------
        return_ids_only                      Optional Boolean. If true, the response includes an array of object
                                             IDs only. The default is false.
        --------------------------------     --------------------------------------------------------------------
        return_attachments                  Optional Boolean.  If true, attachments changes are returned in the
                                             response. Otherwise, attachments are not included. The default is
                                             false. This parameter is only applicable if the feature service has
                                             attachments.
        --------------------------------     --------------------------------------------------------------------
        attachments_by_url                   Optional Boolean.  If true, a reference to a URL will be provided
                                             for each attachment returned. Otherwise, attachments are embedded in
                                             the response. The default is true.
        --------------------------------     --------------------------------------------------------------------
        data_format                          Optional String. The format of the changes returned in the response.
                                             The default is json. Values: sqllite or json
        --------------------------------     --------------------------------------------------------------------
        change_extent_grid_cell              Optional String. To optimize localizing changes extent, the value
                                             medium is an 8x8 grid that bound the changes extent. Used only when
                                             `return_extent_only` is true. The default is none.
                                             Values: None, large, medium, or small
        ================================     ====================================================================


        """
        url = "%s/extractChanges"  % self._url
        params = {
            "f" : "json",
            "layerQueries" : queries,
            "layers" : layers,
            "geometry" : geometry,
            "geometryType" : geometry_type,
            "inSR" : in_sr,
            "gdbVersion" : version,
            "returnInserts" : return_inserts,
            "returnUpdates" : return_updates,
            "returnDeletes" : return_deletes,
            "returnIdsOnly" : return_ids_only,
            "returnExtentOnly" : return_extent_only,
            "returnAttachments" : return_attachments,
            "returnAttachmentsDatabyURL" : attachments_by_url,
            "dataFormat" : data_format,
            "layerServerGens" : servergen,
            "changesExtentGridCell" : change_extent_grid_cell
        }

        res = self._con.post(url, params)
        if 'statusUrl' in res:
            surl = res['statusUrl']
            params = {'f' : 'json'}
            res = self._con.get(surl, params)
            while res['status'].lower() != "completed":
                res = self._con.get(surl, params)
                status = res['status']
                if status.lower() == 'completed':
                    return self._con.get(res['resultUrl'])
                elif  status.lower() == 'failed':
                    return None
                else:
                    time.sleep(1)
        return None

    def query(self,
              layer_defs_filter=None,
              geometry_filter=None,
              time_filter=None,
              return_geometry=True,
              return_ids_only=False,
              return_count_only=False,
              return_z=False,
              return_m=False,
              out_sr=None):
        """
           queries the feature layer collection
        """
        qurl = self._url + "/query"
        params = {"f": "json",
                  "returnGeometry": return_geometry,
                  "returnIdsOnly": return_ids_only,
                  "returnCountOnly": return_count_only,
                  "returnZ": return_z,
                  "returnM": return_m}
        if layer_defs_filter is not None and \
                isinstance(layer_defs_filter, dict):
            params['layerDefs'] = layer_defs_filter
        elif layer_defs_filter is not None and \
                isinstance(layer_defs_filter, dict):
            pass
        if geometry_filter is not None and \
                isinstance(geometry_filter, dict):
            params['geometryType'] = geometry_filter['geometryType']
            params['spatialRel'] = geometry_filter['spatialRel']
            params['geometry'] = geometry_filter['geometry']
            if 'inSR' in geometry_filter:
                params['inSR'] = geometry_filter['inSR']

        if out_sr is not None and \
                isinstance(out_sr, SpatialReference):
            params['outSR'] = out_sr
        elif out_sr is not None and \
                isinstance(out_sr, dict):
            params['outSR'] = out_sr
        if time_filter is not None and \
                isinstance(time_filter, dict):
            params['time'] = time_filter
        results = self._con.get(path=qurl,
                                params=params, token=self._token)
        if 'error' in results:
            raise ValueError(results)
        if not return_count_only and not return_ids_only:
            return results
            # if returnFeatureClass == True:
            # json_text = json.dumps(results)
            # return results
            # df = json_normalize(results['features'])
            # df.columns = df.columns.str.replace('attributes.', '')
            # return df
            # else:
            #    return results
            # df = json_normalize(results['features'])
            # df.columns = df.columns.str.replace('attributes.', '')
            # return df
        else:
            return FeatureSet.from_dict(results)

    # ----------------------------------------------------------------------
    def query_related_records(self,
                              object_ids,
                              relationship_id,
                              out_fields="*",
                              definition_expression=None,
                              return_geometry=True,
                              max_allowable_offset=None,
                              geometry_precision=None,
                              out_wkid=None,
                              gdb_version=None,
                              return_z=False,
                              return_m=False):
        """
        The Query operation is performed on a feature service layer
        resource. The result of this operation are feature sets grouped
        by source layer/table object IDs. Each feature set contains
        Feature objects including the values for the fields requested by
        the user. For related layers, if you request geometry
        information, the geometry of each feature is also returned in
        the feature set. For related tables, the feature set does not
        include geometries.

        ======================     ====================================================================
        **Argument**               **Description**
        ----------------------     --------------------------------------------------------------------
        object_ids                 Optional string. the object IDs of the table/layer to be queried.
        ----------------------     --------------------------------------------------------------------
        relationship_id            Optional string. The ID of the relationship to be queried.
        ----------------------     --------------------------------------------------------------------
        out_fields                 Optional string.the list of fields from the related table/layer
                                   to be included in the returned feature set. This list is a comma
                                   delimited list of field names. If you specify the shape field in the
                                   list of return fields, it is ignored. To request geometry, set
                                   return_geometry to true. You can also specify the wildcard "*" as the
                                   value of this parameter. In this case, the results will include all
                                   the field values.
        ----------------------     --------------------------------------------------------------------
        definition_expression      Optional string. The definition expression to be applied to the
                                   related table/layer. From the list of objectIds, only those records
                                   that conform to this expression are queried for related records.
        ----------------------     --------------------------------------------------------------------
        return_geometry            Optional boolean. If true, the feature set includes the geometry
                                   associated with each feature. The default is true.
        ----------------------     --------------------------------------------------------------------
        max_allowable_offset       Optional float. This option can be used to specify the
                                   max_allowable_offset to be used for generalizing geometries returned
                                   by the query operation. The max_allowable_offset is in the units of
                                   the outSR. If outSR is not specified, then max_allowable_offset is
                                   assumed to be in the unit of the spatial reference of the map.
        ----------------------     --------------------------------------------------------------------
        geometry_precision         Optional integer. This option can be used to specify the number of
                                   decimal places in the response geometries.
        ----------------------     --------------------------------------------------------------------
        out_wkid                   Optional integer. The spatial reference of the returned geometry.
        ----------------------     --------------------------------------------------------------------
        gdb_version                Optional string. The geodatabase version to query. This parameter
                                   applies only if the isDataVersioned property of the layer queried is
                                   true.
        ----------------------     --------------------------------------------------------------------
        return_z                   Optional boolean. If true, Z values are included in the results if
                                   the features have Z values. Otherwise, Z values are not returned.
                                   The default is false.
        ----------------------     --------------------------------------------------------------------
        return_m                   Optional boolean. If true, M values are included in the results if
                                   the features have M values. Otherwise, M values are not returned.
                                   The default is false.
        ======================     ====================================================================


        :return: dict
        """
        params = {
            "f": "json",
            "objectIds": object_ids,
            "relationshipId": relationship_id,
            "outFields": out_fields,
            "returnGeometry": return_geometry,
            "returnM": return_m,
            "returnZ": return_z
        }
        if gdb_version is not None:
            params['gdbVersion'] = gdb_version
        if definition_expression is not None:
            params['definitionExpression'] = definition_expression
        if out_wkid is not None and \
                isinstance(out_wkid, SpatialReference):
            params['outSR'] = out_wkid
        elif out_wkid is not None and \
                isinstance(out_wkid, dict):
            params['outSR'] = out_wkid
        if max_allowable_offset is not None:
            params['maxAllowableOffset'] = max_allowable_offset
        if geometry_precision is not None:
            params['geometryPrecision'] = geometry_precision
        qrr_url = self._url + "/queryRelatedRecords"
        res = self._con.get(path=qrr_url, params=params, token=self._token)
        return res

    # ----------------------------------------------------------------------
    @property
    def _replicas(self):
        """ returns all the replicas for a feature service """
        params = {
            "f": "json",

        }
        url = self._url + "/replicas"
        return self._con.get(path=url, params=params, token=self._token)

    # ----------------------------------------------------------------------
    def _unregister_replica(self, replica_id):
        """
        Removes a replica from a feature service

        ===============     ====================================================================
        **Argument**        **Description**
        ---------------     --------------------------------------------------------------------
        replica_id          Optional string. The replica_id returned by the feature service when
                            the replica was created.
        ===============     ====================================================================


        :return: boolean

        """
        params = {
            "f": "json",
            "replicaID": replica_id
        }
        url = self._url + "/unRegisterReplica"
        return self._con.post(path=url, postdata=params, token=self._token)

    # ----------------------------------------------------------------------
    def _replica_info(self, replica_id):
        """
        The replica info resources lists replica metadata for a specific replica.


        ===============     ====================================================================
        **Argument**        **Description**
        ---------------     --------------------------------------------------------------------
        replica_id          Optional string. The replica_id returned by the feature service when
                            the replica was created.
        ===============     ====================================================================

        :returns: dict

        """
        params = {
            "f": "json"
        }
        url = self._url + "/replicas/" + replica_id
        return self._con.get(path=url, params=params, token=self._token)

    # ----------------------------------------------------------------------
    def _create_replica(self,
                        replica_name,
                        layers,
                        layer_queries=None,
                        geometry_filter=None,
                        replica_sr=None,
                        transport_type="esriTransportTypeUrl",
                        return_attachments=False,
                        return_attachments_data_by_url=False,
                        asynchronous=False,
                        sync_direction=None,
                        target_type="client",
                        attachments_sync_direction="none",
                        sync_model="none",
                        data_format="json",
                        replica_options=None,
                        wait=False,
                        out_path=None):
        """
        The createReplica operation is performed on a feature service
        resource. This operation creates the replica between the feature
        service and a client based on a client-supplied replica definition.
        It requires the Sync capability. See Sync overview for more
        information on sync. The response for createReplica includes
        replicaID, server generation number, and data similar to the
        response from the feature service query operation.
        The createReplica operation returns a response of type
        esriReplicaResponseTypeData, as the response has data for the
        layers in the replica. If the operation is called to register
        existing data by using replicaOptions, the response type will be
        esriReplicaResponseTypeInfo, and the response will not contain data
        for the layers in the replica.

        Inputs:
           replicaName - name of the replica
           layers - layers to export
           layerQueries - In addition to the layers and geometry parameters, the layerQueries
            parameter can be used to further define what is replicated. This
            parameter allows you to set properties on a per layer or per table
            basis. Only the properties for the layers and tables that you want
            changed from the default are required.
            Example:
             layerQueries = {"0":{"queryOption": "useFilter", "useGeometry": true,
             "where": "requires_inspection = Yes"}}
           geometry_filter - spatial filter from arcgis.geometry.filters module to filter results by a
                             spatial relationship with another geometry. Only intersections are currently supported.
           returnAttachments - If true, attachments are added to the replica and returned in the
            response. Otherwise, attachments are not included.
           returnAttachmentDatabyURL -  If true, a reference to a URL will be provided for each
            attachment returned from createReplica. Otherwise,
            attachments are embedded in the response.
           replicaSR - the spatial reference of the replica geometry.
           transportType -  The transportType represents the response format. If the
            transportType is esriTransportTypeUrl, the JSON response is contained in a file,
            and the URL link to the file is returned. Otherwise, the JSON object is returned
            directly. The default is esriTransportTypeUrl.
            If async is true, the results will always be returned as if transportType is
            esriTransportTypeUrl. If dataFormat is sqlite, the transportFormat will always be
            esriTransportTypeUrl regardless of how the parameter is set.
            Values: esriTransportTypeUrl | esriTransportTypeEmbedded
           returnAttachments - If true, attachments are added to the replica and returned in
            the response. Otherwise, attachments are not included. The default is false. This
            parameter is only applicable if the feature service has attachments.
           returnAttachmentsDatabyURL -  If true, a reference to a URL will be provided for
            each attachment returned from createReplica. Otherwise, attachments are embedded
            in the response. The default is true. This parameter is only applicable if the
            feature service has attachments and if returnAttachments is true.
           attachmentsSyncDirection - Client can specify the attachmentsSyncDirection when
            creating a replica. AttachmentsSyncDirection is currently a createReplica property
            and cannot be overridden during sync.
            Values: none, upload, bidirectional
           asynchronous - If true, the request is processed as an asynchronous job, and a URL is
            returned that a client can visit to check the status of the job. See the topic on
            asynchronous usage for more information. The default is false.
           syncModel - Client can specify the attachmentsSyncDirection when creating a replica.
            AttachmentsSyncDirection is currently a createReplica property and cannot be
            overridden during sync.
           dataFormat - The format of the replica geodatabase returned in the response. The
            default is json.
            Values: filegdb, json, sqlite, shapefile
           target_type - This option was added at 10.5.1. Can be set to either server or client.
            If not set, the default is client.A targetType of client will generate a replica that
            matches those generated in pre-10.5.1 releases. These are designed to support syncing
            with lightweight mobile clients and have a single generation number (serverGen or
            replicaServerGen).
            A targetType of server generates a replica that supports syncing in one direction
            between 2 feature services running on servers or between an ArcGIS Server feature
            service and an ArcGIS Online feature service. When the targetType is server, the replica
            information includes a second generation number. This second generation number is called
            replicaServerSibGen for perReplica types and serverSibGen for perLayer types.
            target_type server replicas generated with dataFormat SQLite can be published as new
            services in another ArcGIS Online organization or in ArcGIS Enterprise. When published,
            a replica is generated on these new services with a matching replicaID and a
            replicaServerSibGen or serverSibGens. The replicaServerSibGen or serverSibGens values
            can be used as the replicaServerGen or serverGen values when calling synchronize replica
            on the source service to get the latest changes. These changes can then be imported into
            the new service using the synchronizeReplica operation. When calling synchronizeReplica
            on the new service to import the changes, be sure to pass the new replicaServerGen or
            serverGen from the source service as the replicaServerSibGen or serverSibGen. This will
            update the replica metadata appropriately such that it can be used in the next sync.
            Values: server, client
           sync_direction - Defaults to bidirectional when the targetType is client and download
            when the targetType is server. If set, only bidirectional is supported when
            targetType is client. If set, only upload or download are supported when targetType is
            server.
            A syncDirection of bidirectional matches the functionality from replicas generated in
            pre-10.5.1 releases and allows upload and download of edits. It is only supported
            when targetType is client.
            When targetType is server, only a one way sync is supported thus only upload or
            download are valid options.
            A syncDirection of upload means that the synchronizeReplica operation allows only sync
            with an upload direction. Use this option to allow the upload of edits from the source
            service.
            A syncDirection of download means that the synchronizeReplica operation allows only sync
            with a download direction. Use this option to allow the download of edits to provide to
            the source service.
           replicaOptions - This parameter instructs the createReplica operation to create a
            new replica based on an existing replica definition (refReplicaId). It can be used
            to specify parameters for registration of existing data for sync. The operation
            will create a replica but will not return data. The responseType returned in the
            createReplica response will be esriReplicaResponseTypeInfo.
           wait - if async, wait to pause the process until the async operation is completed.
           out_path - folder path to save the file
        """
        if not self.properties.syncEnabled and "Extract" not in self.properties.capabilities:
            return None
        url = self._url + "/createReplica"
        dataformat = ["filegdb", "json", "sqlite", "shapefile"]
        params = {
            "f": "json",
            "replicaName": replica_name,
            "returnAttachments": json.dumps(return_attachments),
            "returnAttachmentsDatabyUrl": json.dumps(return_attachments_data_by_url),
            "async": json.dumps(asynchronous),
            "syncModel": sync_model,
            "layers": layers,
            "targetType" : target_type,

        }
        if attachments_sync_direction:
            params["attachmentsSyncDirection"] = attachments_sync_direction
        if sync_direction:
            params['syncDirection'] = sync_direction
        if data_format.lower() in dataformat:
            params['dataFormat'] = data_format.lower()
        else:
            raise Exception("Invalid dataFormat")
        if layer_queries is not None:
            params['layerQueries'] = layer_queries
        if geometry_filter is not None and \
                isinstance(geometry_filter, dict):
            params['geometry'] = geometry_filter['geometry']
            params['geometryType'] = geometry_filter['geometryType']
            if 'inSR' in geometry_filter:
                params['inSR'] = geometry_filter['inSR']
        if replica_sr is not None:
            params['replicaSR'] = replica_sr
        if replica_options is not None:
            params['replicaOptions'] = replica_options
        if transport_type is not None:
            params['transportType'] = transport_type

        if asynchronous:
            if wait:
                export_job = self._con.post(path=url, postdata=params, token=self._token)
                status = self._replica_status(url=export_job['statusUrl'])
                while status['status'] not in ("Completed", "CompletedWithErrors"):
                    if status['status'] == "Failed":
                        return status
                    # wait before checking again
                    time.sleep(2)
                    status = self._replica_status(url=export_job['statusUrl'])

                res = status

            else:
                res = self._con.post(path=url, postdata=params, token=self._token)
        else:
            res = self._con.post(path=url, postdata=params, token=self._token)

        if out_path is not None and \
                os.path.isdir(out_path):
            dl_url = None
            if 'resultUrl' in res:

                dl_url = res["resultUrl"]
            elif 'responseUrl' in res:
                dl_url = res["responseUrl"]

            if dl_url is not None:

                return self._con.get(path=dl_url, file_name=dl_url.split('/')[-1],
                                     out_folder=out_path, try_json=False, token=self._token)

            else:
                return res
        elif res is not None:
            return res
        return None

    # ----------------------------------------------------------------------
    def _synchronize_replica(self,
                             replica_id,
                             transport_type="esriTransportTypeUrl",
                             replica_server_gen=None,
                             replica_servers_sib_gen=None,
                             return_ids_for_adds=False,
                             edits=None,
                             return_attachment_databy_url=False,
                             asynchronous=False,
                             sync_direction=None,
                             sync_layers="perReplica",
                             edits_upload_id=None,
                             edits_upload_format=None,
                             data_format="json",
                             rollback_on_failure=True,
                             close_replica=False,
                             out_path=None):
        """
        The synchronizeReplica operation is performed on a feature service resource. This operation
        synchronizes changes between the feature service and a client based on the replicaID
        provided by the client. Requires the sync capability. See Sync overview for more information
        on sync.
        The client obtains the replicaID by first calling the _create_replica operation.
        Synchronize applies the client's data changes by importing them into the server's
        geodatabase. It then exports the changes from the server geodatabase that have taken place
        since the last time the client got the data from the server. Edits can be supplied in the
        edits parameter, or, alternatively, by using the editsUploadId and editUploadFormat to
        identify a file containing the edits that were previously uploaded using the upload_item
        operation.
        The response for this operation includes the replicaID, new replica generation number, or
        the layer's generation numbers. The response has edits or layers according to the
        syncDirection/syncLayers. Presence of layers and edits in the response is indicated by the
        responseType.
        If the responseType is esriReplicaResponseTypeEdits or esriReplicaResponseTypeEditsAndData,
        the result of this operation can include arrays of edit results for each layer/table edited
        as specified in edits. Each edit result identifies a single feature on a layer or table and
        indicates if the edits were successful or not. If an edit is not successful, the edit result
        also includes an error code and an error description.
        If syncModel is perReplica and syncDirection is download or bidirectional, the
        _synchronize_replica operation's response will have edits. If syncDirection is snapshot, the
        response will have replacement data.
        If syncModel is perLayer, and syncLayers have syncDirection as download or bidirectional,
        the response will have edits. If syncLayers have syncDirection as download or bidirectional
        for some layers and snapshot for some other layers, the response will have edits and data.
        If syncDirection for all the layers is snapshot, the response will have replacement data.
        When syncModel is perReplica, the createReplica and synchronizeReplica operations' responses
        contain replicaServerGen. When syncModel is perLayer, the createReplica and
        synchronizeReplica operations' responses contain layerServerGens.
        You can provide arguments to the synchronizeReplica operation as defined in the parameters
        table below.

        Parameters:
         :replica_id: The ID of the replica you want to synchronize.
         :transport_type:
         :replica_server_gen: is a generation number that allows the server to keep track of what
          changes have already been synchronized. A new replicaServerGen is sent with the response
          to the synchronizeReplica operation. Clients should persist this value and use it with the
          next synchronizeReplica call.
          It applies to replicas with syncModel = perReplica.
          For replicas with syncModel = perLayer, layer generation numbers are specified using
          parameter: syncLayers; and replicaServerSibGen is not needed.
         :replica_servers_sib_gen:  is a generation number that allows the server to keep track of
          what changes have already been received. It is set when synchronizing where
          syncModel = perReplica and targetType = server. The replicaServerSibGen is updated in the
          replica metadata on the replica resource once the process completes successfully.
          Replicas with targetType = server are designed to allow syncing between services. When
          syncing, the replicaServerSibGen value is derived from the replicaServerGen of the other
          services matching replica.
          For replicas with syncModel = perLayer, layer generation numbers are specified using
          parameter: syncLayers; and replicaServerGen is not needed.
          This value is not set for replicas where the targetType=client.
         :return_ids_for_adds: If true, the objectIDs and globalIDs of features added during the
          synchronize will be returned to the client in the addResults sections of the response.
          Otherwise, the IDs are not returned. The default is false.
          Values: true | false
         :edits: The edits the client wants to apply to the service. Alternatively, the
          edits_upload_ID and editsUploadFormat can be used to specify the edits in a delta file.
          The edits are described using an array where an element in the array includes:
           - The layer or table ID
           - The feature or row edits to apply listed as inserts, updates, and deletes
           - The attachments to apply listed as inserts, updates, and deletes
         For features, adds and updates are specified as feature objects that include geometry and
         attributes.
         Deletes can be specified using globalIDs for features and attachments.
         For attachments, updates and adds are specified using the following set of properties for
         each attachment. If embedding the attachment, set the data property; otherwise, set the url
         property. All other properties are required:
          - globalid - The globalID of the attachment that is to be added or updated.
          - parentGlobalid - The globalID of the feature associated with the attachment.
          - contentType - Describes the file type of the attachment (for example, image/jpeg).
          - name - The file name (for example, hydrant.jpg).
          - data - The base 64 encoded data if embedding the data. Only required if the attachment
            is embedded.
          - url - The location where the service will upload the attachment file (for example,
            http://machinename/arcgisuploads/Hydrant.jpg). Only required if the attachment is not
            embedded.
         :return_attachment_databy_url:  If true, a reference to a URL will be provided for each
          attachment returned from synchronizeReplica. Otherwise, attachments are embedded in the
          response. The default is true. Applies only if attachments are included in the replica.
          Values: true | false
         :asynchronous: If true, the request is processed as an asynchronous job and a URL is
          returned that a client can visit to check the status of the job. See the topic on
          asynchronous usage for more information. The default is false.
          Values: true | false
         :sync_direction: Determines whether to upload, download, or upload and download on sync. By
          default, a replica is synchronized bi-directionally. Only applicable when
          syncModel = perReplica. If syncModel = perLayer, sync direction is specified using
          syncLayers.
          Values: download | upload | bidirectional | snapshot

           - download-The changes that have taken place on the server since last download are
             returned. Client does not need to send any changes. If the changes are sent, service
             will ignore them.
           - upload-The changes submitted in the edits or editsUploadID/editsUploadFormatt
             parameters are applied, and no changes are downloaded from the server.
           - bidirectional-The changes submitted in the edits or editsUploadID/editsUploadFormat
             parameters are applied, and changes on the server are downloaded. This is the default
             value.
           - snapshot-The current state of the features is downloaded from the server. If any edits
             are specified, they will be ignored.
         :sync_layers:  allows a client to specify layer-level generation numbers for a sync
          operation. It can also be used to specify sync directions at layer-level. This parameter
          is needed for replicas with syncModel = perLayer. It is ignored for replicas with
          syncModel = perReplica.
          serverGen is required for layers with syncDirection = bidirectional or download.
          serverSibGen is needed only for replicas where the targetType = server. For replicas with
          syncModel = perLayer, the serverSibGen serves the same purpose at the layer level as the
          replicaServerSibGen does in the case of syncModel = perReplica. See the
          replicaServerSibGen parameter for more information.
          If a sync operation has both the syncDirection and syncLayersparameters, and the replica's
          syncModel is perLayer, the layers that do not have syncDirection values will use the value
          of the syncDirection parameter. If the syncDirection parameter is not specified, the
          default value of bidirectional is used.
          Values: download | upload | bidirectional | snapshot
         :edits_upload_id: The ID for the uploaded item that contains the edits the client wants to
          apply to the service. Used in conjunction with editsUploadFormat.
         :edits_upload_format: The data format of the uploaded data reference in edit_upload_id.
          data_format="json",
         :rollback_on_failure:  Determines the behavior when there are errors while importing edits
          on the server during synchronization. This only applies in cases where edits are being
          uploaded to the server (syncDirection = upload or bidirectional). See the
          RollbackOnFailure and Sync Models topic for more details.
          When true, if an error occurs while importing edits on the server, all edits are rolled
          back (not applied), and the operation returns an error in the response. Use this setting
          when the edits are such that you will either want all or none applied.
          When false, if an error occurs while importing an edit on the server, the import process
          skips the edit and continues. All edits that were skipped are returned in the edits
          results with information describing why the edits were skipped.
         :close_replica:  If true, the replica will be unregistered when the synchronize completes.
          This is the same as calling synchronize and then calling unregisterReplica. Otherwise, the
          replica can continue to be synchronized. The default is false.
          Values: true | false
        """
        url = "{url}/synchronizeReplica".format(url=self._url)
        params = {
            "f": "json",
            "replicaID": replica_id,
        }

        if transport_type is not None:
            params['transportType'] = transport_type
        if edits is not None:
            params['edits'] = edits
        if replica_server_gen is not None:
            params['replicaServerGen'] = replica_server_gen
        if return_ids_for_adds is not None:
            params['returnIdsForAdds'] = return_ids_for_adds
        if return_attachment_databy_url is not None:
            params['returnAttachmentDatabyURL'] = return_attachment_databy_url
        if asynchronous is not None:
            params['async'] = asynchronous
        if sync_direction is not None:
            params['syncDirection'] = sync_direction
        if sync_layers is not None:
            params['syncLayers'] = sync_layers
        if edits_upload_format is not None:
            params['editsUploadFormat'] = edits_upload_format
        if edits_upload_id is not None:
            params['editsUploadID'] = edits_upload_id
        if data_format is not None:
            params['dataFormat'] = data_format
        #if edits_upload_id:
        #    params['dataFormat'] = edits_upload_id
        if rollback_on_failure is not None:
            params['rollbackOnFailure'] = rollback_on_failure
        if close_replica:
            params['closeReplica'] = close_replica
        if replica_servers_sib_gen:
            params['replicaServerSibGen'] = replica_servers_sib_gen
        res = self._con.post(path=url, postdata=params, token=self._token)
        if out_path is not None and \
               os.path.isdir(out_path):
            dl_url = None
            if 'resultUrl' in res:
                dl_url = res["resultUrl"]
            elif 'responseUrl' in res:
                dl_url = res["responseUrl"]
            elif 'URL' in res:
                dl_url = res['URL']
            if dl_url is not None:
                return self._con.get(path=dl_url, file_name=dl_url.split('/')[-1],
                                     out_folder=out_path, try_json=False,
                                     token=self._token)
            else:
                return res
        return res

    # ----------------------------------------------------------------------
    def _replica_status(self, url):
        """gets the replica status when exported async set to True"""
        params = {"f": "json"}
        url += "/status"
        return self._con.get(path=url,
                             params=params, token=self._token)

    #----------------------------------------------------------------------
    def upload(self, path, description=None):
        """
        Uploads a new item to the server. Once the operation is completed
        successfully, the JSON structure of the uploaded item is returned.


        ===============     ====================================================================
        **Argument**        **Description**
        ---------------     --------------------------------------------------------------------
        path                Optional string. Filepath of the file to upload.
        ---------------     --------------------------------------------------------------------
        description         Optional string. Descriptive text for the uploaded item.
        ===============     ====================================================================

        :returns: boolean

        """
        if (os.path.getsize(path) >> 20) <= 9:
            url = self._url + "/uploads/upload"
            params = {
                "f" : "json",
                'filename' : os.path.basename(path),
                'overwrite' : True
            }
            files = {}
            files['file'] = path
            if description:
                params['description'] = description
            res = self._con.post(path=url,
                                 postdata=params,
                                 files=files)
            if 'status' in res and \
               res['status'] == 'success':
                return True, res
            elif 'success' in res:
                return res['success'], res
            return False, res
        else:
            file_path = path
            item_id = self._register_upload(file_path)
            self._upload_by_parts(item_id, file_path)
            return self._commit_upload(item_id)
    #----------------------------------------------------------------------
    def _register_upload(self, file_path):
        """returns the itemid for the upload by parts logic"""
        r_url = "%s/uploads/register" % self._url
        params = {'f' : 'json',
                  'itemName' : os.path.basename(file_path).replace('.', '')
                  }
        reg_res = self._con.post(r_url, params)
        if 'item' in reg_res and \
           'itemID' in reg_res['item']:
            return reg_res['item']['itemID']
        return None
    #----------------------------------------------------------------------
    def _upload_by_parts(self, item_id, file_path):
        """loads a file for attachmens by parts"""
        import mmap, tempfile

        b_url = "%s/uploads/%s" % (self._url, item_id)
        upload_part_url = "%s/uploadPart" % b_url
        params = {
            "f" : "json"
        }
        with open(file_path, 'rb') as f:
            mm = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ)
            size = 1000000
            steps =  int(os.fstat(f.fileno()).st_size / size)
            if os.fstat(f.fileno()).st_size % size > 0:
                steps += 1
            for i in range(steps):
                files = {}
                tempFile = os.path.join(tempfile.gettempdir(), "split.part%s" % i)
                if os.path.isfile(tempFile):
                    os.remove(tempFile)
                with open(tempFile, 'wb') as writer:
                    writer.write(mm.read(size))
                    writer.flush()
                    writer.close()
                del writer
                files['file'] = tempFile
                params['partId'] = i + 1
                res = self._con.post(upload_part_url,
                                     postdata=params,
                                     files=files)
                if 'error' in res:
                    raise Exception(res)
                os.remove(tempFile)
                del files
            del mm
        return True
    #----------------------------------------------------------------------
    def _commit_upload(self, item_id):
        """commits an upload by parts upload"""
        b_url = "%s/uploads/%s" % (self._url, item_id)
        commit_part_url = "%s/commit" % b_url
        params = {
                'f':'json',
                'parts' : self._uploaded_parts(itemid=item_id)
        }
        res = self._con.post(commit_part_url,
                              params)
        if 'error' in res:
            raise Exception(res)
        else:
            return res['item']['itemID']
    #----------------------------------------------------------------------
    def _delete_upload(self, item_id):
        """commits an upload by parts upload"""
        b_url = "%s/uploads/%s" % (self._url, item_id)
        delete_part_url = "%s/delete" % b_url
        params = {
                'f':'json',
        }
        res = self._con.post(delete_part_url,
                              params)
        if 'error' in res:
            raise Exception(res)
        else:
            return res
    #----------------------------------------------------------------------
    def _uploaded_parts(self, itemid):
        """
        returns the parts uploaded for a given item

        ==================   ==============================================
        Arguments           Description
        ------------------   ----------------------------------------------
        itemid               required string. Id of the uploaded by parts item.
        ==================   ==============================================

        """
        url = self._url + "/uploads/%s/parts" % itemid
        params = {
            "f" : "json"
        }
        res = self._con.get(url, params)
        return ",".join(res['parts'])
