"""

These tools are used for the day-to-day management of geographic and tabular data.

copy_to_data_store copies data to your ArcGIS Data Store and creates a layer in your web GIS.
"""
import json as _json
import logging as _logging
import datetime as _datetime
import arcgis as _arcgis
from arcgis.features import FeatureSet as _FeatureSet
from arcgis.geoprocessing._support import _execute_gp_tool
from ._util import _id_generator, _feature_input, _set_context, _create_output_service, GAJob

_log = _logging.getLogger(__name__)

_use_async = True

def run_python_script(code, layers=None, gis=None, context=None, future=False):
    """

    The ``run_python_script`` method executes a Python script on your ArcGIS
    GeoAnalytics Server site. In the script, you can create an analysis
    pipeline by chaining together multiple GeoAnalytics Tools without
    writing intermediate results to a data store. You can also use other
    Python functionality in the script that can be distributed across your
    GeoAnalytics Server.

    For example, suppose that each week you receive a new dataset of
    vehicle locations containing billions of point features. Each time you
    receive a new dataset, you must perform the same workflow involving
    multiple GeoAnalytics Tools to create an information product that you
    share within your organization. This workflow creates several large
    intermediate layers that take up lots of space in your data store. By
    scripting this workflow in Python and executing the code in the Run
    Python Script task, you can avoid creating these unnecessary
    intermediate layers, while simplifying the steps to create the
    information product.

    When you use ``run_python_script``, the Python code is executed on your
    GeoAnalytics Server. The script runs with the Python 3.6 environment
    that is installed with GeoAnalytics Server, and all console output is
    returned as job messages. Some Python modules can be used in your
    script to execute code across multiple cores of one or more machines
    in your GeoAnalytics Server using Spark 2.2.0(the compute platform that
    distributes analysis for GeoAnalytics Tools).

    A geoanalytics module is available and allows you to run GeoAnalytics
    Tools in the script. This package is imported automatically when you
    use Run Python Script.

    To interact directly with Spark in the Run Python Script task, use the
    pyspark module, which is imported automatically when you run the task.
    The pyspark module is the Python API for Spark and provides a
    collection of distributed analysis tools for data management,
    clustering, regression, and more that can be called in Run Python
    Script and run across your GeoAnalytics Server.

    When using the geoanalytics and pyspark packages, most functions return
    analysis results in memory as Spark DataFrames. Spark data frames can be
    written to a data store or used in the script. This allows for the
    chaining together of multiple geoanalytics and pyspark tools, while only
    writing out the final result to a data store, eliminating the need to
    create any intermediate result layers.

    For advanced users, an instance of SparkContext is instantiated
    automatically as sc and can be used in the script to interact with Spark.
    This allows for the execution of custom distributed analysis across your
    GeoAnalytics Server.

    It is recommended that you use an integrated development environment
    (IDE) to write your Python script, and copy the script text into the Run
    Python Script tool. This makes it easier to identify syntax errors and
    typos prior to running your script. It is also recommended that you run
    your script using a small subset of the input data first to verify that
    there are no logic errors or exceptions. You can use the Describe
    Dataset task to create a sample layer for this purpose.

    ================  ===============================================================
    code              Required string. The Python script that will run on your GeoAnalytics Server. This must be the full script as a string.

                      The layers provided in inputLayers can be accessed in the script using the layers object. To learn more, 
                      see `Reading and writing layers in pyspark <https://developers.arcgis.com/rest/services-reference/using-webgis-layers-in-pyspark.htm>`_.
  
                      GeoAnalytics Tools can be accessed with the geoanalytics object, which is instantiated in the script environment automatically. 
                      To learn more, see `Using GeoAnalytics Tools in Run Python Script <https://developers.arcgis.com/rest/services-reference/using-geoanalytics-tools-in-pyspark.htm>`_.
  
                      For a collection of example scripts, see `Examples: Scripting custom analysis with the Run Python Script task <https://developers.arcgis.com/rest/services-reference/run-python-script-examples.htm>`_.
    ----------------  ---------------------------------------------------------------
    layers            Optional list. A list of Feature layers to operate on. See :ref:`Feature Input<FeatureInput>`.
    ----------------  ---------------------------------------------------------------
    gis               optional GIS. The GIS object where the analysis will take place.
    ----------------  ---------------------------------------------------------------
    context           Optional dict. This parameter is not used by the ``run_python_script`` tool.

                      To control the output data store, use the "dataStore" option when writing DataFrames.
  
                      To set the processing or output spatial reference, use the project tool in the geoanalytics package.
  
                      To filter a layer when converting it to a DataFrame, use the "where" or "fields" option when loading the layer's URL.
  
                      To limit the extent of a layer when converting it to a DataFrame, use the "extent" option when loading the layer's URL.
    ----------------  ---------------------------------------------------------------
    future            Optional boolean. If 'True', a GPJob is returned instead of
                      results. The GPJob can be queried on the status of the execution.
    ================  ===============================================================

    :returns: list of dictionary of messages from the code provided.

    .. code-block:: python

            # Usage Example: Execute calculate_density tool using run_python_script.

            code = '''
            import time 
            r = geoanalytics.calculate_density(input_layer=layers[0],
                            fields='Damage',
                            weight='Uniform',
                            bin_type='Square',
                            bin_size=1,
                            bin_size_unit='Meters',
                            radius=2,
                            radius_unit='Yards')        
                            
            r.write.format('webgis').save('RunPythonScriptTest_{0}'.format(time.time()))              
            '''
            run_python_script(code=code, layers=[lyr])
    """
    if layers is None:
        layers = []
    import inspect
    params = {'f': 'json'}

    if inspect.isfunction(code):
        params['code'] = inspect.getsource(code) + '\n' + code.__name__ + '()'
    elif isinstance(code, str):
        params['code'] = code
    else:
        raise ValueError("code must be a string or Python Function.")

    if isinstance(layers, (tuple, list)):
        params['layers'] = layers
    else:
        raise ValueError("layers must be a list or tuple")

    tool_name = "RunPythonScript"
    gis = _arcgis.env.active_gis if gis is None else gis
    url = gis.properties.helperServices.geoanalytics.url

    if context is not None:
        params["context"] = context
    else:
        _set_context(params)

    param_db = {
        "layers": (_FeatureSet, "inputLayers"),
        "code" : (str, "pythonScript"),
        "context": (str, "context"),
    }

    try:
        if future:
            gpjob = _execute_gp_tool(gis, tool_name, params, param_db, [], _use_async, url, True, return_messages=False, future=future)
            return GAJob(gpjob=gpjob, return_service=None)
        res, msg = _execute_gp_tool(gis, tool_name, params, param_db, [], _use_async, url, True, return_messages=True, future=future)
        msg = msg['messages']
        return msg
    except:
        raise
    return 

def dissolve_boundaries(input_layer,
                        dissolve_fields=None,
                        summary_fields=None,
                        multipart=False,
                        output_name=None,
                        gis=None,
                        context=None,
                        future=False):
    """
    .. image:: _static/images/dissolve_boundaries/dissolve_boundaries.png 

    The ``dissolve_boundaries`` task finds polygons that intersect or have the same field values 
    and merges them together to form a single polygon.

    Example:

        A city council wants to control liquor sales by refusing new licenses to stores within 
        1,000 feet of schools, libraries, and parks. After creating a 1,000-foot buffer around 
        the schools, libraries, and parks, the buffered layers can be joined together and the 
        boundaries can be dissolved to create a single layer of restricted areas.

    .. note::
        Only available at ArcGIS Enterprise 10.7 and later.

    ================  ===============================================================
    **Argument**      **Description**
    ----------------  ---------------------------------------------------------------
    input_layer       Required layer. The layer containing polygon features that will be dissolved. See :ref:`Feature Input<FeatureInput>`.
    ----------------  ---------------------------------------------------------------
    dissolve_fields   Optional string. A comma seperated list of strings for each
                      field that you want to dissolve on.One or more fields in the ``input_layer`` that determine how polygons are merged based on field value.

                      If you don't specify fields, polygons that intersect will be dissolved into one polygon by default.

                      If you do specify fields, polygons that share the same value for each of the specified fields will be dissolved into one polygon.
    ----------------  ---------------------------------------------------------------
    summary_fields    Optional list of dicts. A list of field names and statistical summary types you want to calculate. 
                      Note that the count is always returned. By default, all statistics are returned.

                      Syntax: [{"statisticType" : "<stat>", "onStatisticField" : "<field name>"}]

                      fieldName is the name of the fields in the input point layer.

                      statisticType is one of the following for numeric fields:

                        * ``Count`` - Totals the number of values of all the points in each polygon.
                        * ``Sum`` - Adds the total value of all the points in each polygon.
                        * ``Mean`` - Calculates the average of all the points in each polygon.
                        * ``Min`` - Finds the smallest value of all the points in each polygon.
                        * ``Max`` - Finds the largest value of all the points in each polygon.
                        * ``Range`` - Finds the difference between the Min and Max values.
                        * ``Stddev`` - Finds the standard deviation of all the points in each polygon.
                        * ``Var`` - Finds the variance of all the points in each polygon.
                        statisticType is one of the following for string fields:

                        * ``Count`` - Totals the number of strings for all the points in each polygon.
                        * ``Any`` - Returns a sample string of a point in each polygon.
                  
                      Example: summary_fields = [{"statisticType" : "Sum", "onStatisticField" : "quadrat_area_km2"}, {"statisticType" : "Mean", "onStatisticField" : "soil_depth_cm"}, {"statisticType" : "Any", "onStatisticField" : "quadrat_desc"}]
    ----------------  ---------------------------------------------------------------
    multipart         Optional boolean. If 'True', the output service can contain
                      multipart features. If 'False', the output service
                      will only contain single-part features, and individual features
                      will be created for each part.

                      The default value is 'False'.
    ----------------  ---------------------------------------------------------------
    output_name       optional string. The task will create a feature service of the results. You define the name of the service.
    ----------------  ---------------------------------------------------------------
    gis               optional GIS. The GIS object where the analysis will take place.
    ----------------  ---------------------------------------------------------------
    context           Optional dict. The context parameter contains additional settings that affect task execution. For this task, there are five settings:

                      #. Extent (``extent``) - A bounding box that defines the analysis area. Only those features that intersect the bounding box will be analyzed.
                      #. Processing spatial reference (``processSR``) - The features will be projected into this coordinate system for analysis.
                      #. Output spatial reference (``outSR``) - The features will be projected into this coordinate system after the analysis to be saved. The output spatial reference for the spatiotemporal big data store is always WGS84.
                      #. Data store (``dataStore``) - Results will be saved to the specified data store. The default is the spatiotemporal big data store.
                      #. Default aggregation styles (``defaultAggregationStyles``) - If set to true, results will have square, hexagon, and triangle aggregation styles enabled on results map services.
    ----------------  ---------------------------------------------------------------
    future            optional boolean. If True, a GPJob is returned instead of
                      results. The GPJob can be queried on the status of the execution.
    ================  ===============================================================

    :returns: result_layer : Output Features as feature layer collection item.

    .. code-block:: python

            # Usage Example: This example dissolves boundaries of soil areas in Nebraska if they have 
            # the same solubility. For dissolved features, it calculates the sum of the quadrat area, 
            # the mean soil depth, and an example of the quadrat description.
            
            arcgis.env.out_spatial_reference = 3310
            arcgis.env.output_datastore= "relational"
            arcgis.env.defaultAggregations= True

            summary_fields = [{"statisticType" : "Sum", "onStatisticField" : "quadrat_area_km2"},
                            {"statisticType" : "Mean", "onStatisticField" : "soil_depth_cm"},
                            {"statisticType" : "Any", "onStatisticField" : "quadrat_desc"}]

            dissolve_result = dissolve_boundaries(input_layer=study_area_lyr,
                                                  dissolve_fields="soil_suitability",
                                                  summary_fields=summary_fields,
                                                  multipart=True,
                                                  output_name="Soil_Suitability_dissolved")
    """
    kwargs = locals()
    tool_name = "DissolveBoundaries"
    gis = _arcgis.env.active_gis if gis is None else gis
    url = gis.properties.helperServices.geoanalytics.url
    params = {
        "f" : "json",
    }
    for key, value in kwargs.items():
        if value is not None:
            params[key] = value

    if output_name is None:
        output_service_name = 'Dissolve_Bounds_' + _id_generator()
        output_name = output_service_name.replace(' ', '_')
    else:
        output_service_name = output_name.replace(' ', '_')

    output_service = _create_output_service(gis, output_name, output_service_name, 'Merge Layers')

    params['output_name'] = _json.dumps({
        "serviceProperties": {"name" : output_name, "serviceUrl" : output_service.url},
        "itemProperties": {"itemId" : output_service.itemid}})

    if context is not None:
        params["context"] = context
    else:
        _set_context(params)

    param_db = {
        "input_layer": (_FeatureSet, "inputLayer"),
        "multipart": (bool, "multipart"),
        "summary_fields": (list, "summaryFields"),
        "dissolve_fields" : (list, "dissolveFields"),
        "output_name": (str, "OutputName"),
        "context": (str, "context"),
        "output": (_FeatureSet, "output"),
    }

    return_values = [
        {"name": "output", "display_name": "Output Features", "type": _FeatureSet},
    ]

    try:
        if future:
            gpjob = _execute_gp_tool(gis, tool_name, params, param_db, return_values, _use_async, url, True, future=future)
            return GAJob(gpjob=gpjob, return_service=output_service)
        _execute_gp_tool(gis, tool_name, params, param_db, return_values, _use_async, url, True, future=future)
        return output_service
    except:
        output_service.delete()
        raise
    return

def merge_layers(input_layer, 
                 merge_layer,
                 merge_attributes=None, 
                 output_name=None,
                 gis=None, 
                 context=None,
                 future=False):
    """

    .. image:: _static/images/merge_layers/merge_layers.png 
    
    The ``merge_layers`` task combines two feature layers to create a single output layer. The tool
    requires that both layers have the same geometry type (tabular, point, line, or polygon). If
    time is enabled on one layer, the other must also be time enabled and have the same time type
    (instant or interval). The result will always contain all fields from the input layer. All
    fields from the merge layer will be included by default, or you can specify custom merge rules
    to define the resulting schema. For example:

        * I have three layers for England, Wales, and Scotland and I want a single layer of Great
          Britain. I can use Merge Layers to combine the areas and maintain all fields from each area.
        * I have two layers containing parcel information for contiguous townships. I want to join them
          together into a single layer, keeping only the fields that have the same name and type in the
          two layers.

    .. note::
        Only available at **ArcGIS Enterprise 10.7** and later.

    ================  ===============================================================
    **Argument**      **Description**
    ----------------  ---------------------------------------------------------------
    input_layer       Required layer. The table, point, line, or polygon features to 
                      merge with the ``merge_layer`` parameter. All fields in ``input_layer`` 
                      will be included in the result layer.  See :ref:`Feature Input<FeatureInput>`.
    ----------------  ---------------------------------------------------------------
    merge_layer       Required layer. The point, line, or polygon features to
                      merge with the ``input_layer``. The ``merge_layer`` must contain the
                      same geometry type (tabular, point, line, or polygon) and the
                      same time type (none, instant, or interval) as the ``input_layer``.
                      All fields in the ``merge_layer`` will be included in the result
                      layer by default or you can define ``merge_attributes`` to
                      customize the resulting schema.  See :ref:`Feature Input<FeatureInput>`.
    ----------------  ---------------------------------------------------------------
    merge_attributes  Optional list of dicts. Defines how the fields in ``merge_layer`` will be
                      modified. By default, all fields from both inputs will be
                      included in the output layer.

                      If a field exists in one layer but not the other, the output
                      layer will still contain the field. The output field will
                      contain null values for the input features that did not have the
                      field. For example, if the ``input_layer`` contains a field named
                      TYPE but the ``merge_layer`` does not contain TYPE, the output will
                      contain TYPE, but its values will be null for all the features
                      copied from the ``merge_layer``.

                      You can control how fields in the ``merge_layer`` are written to the
                      output layer using the following merge types that operate on a
                      specified ``merge_layer`` field:

                            * ``Remove`` - The field in the ``merge_layer`` will be removed from the output layer.
                            * ``Rename`` - The field in the ``merge_layer`` will be renamed in the output layer. 
                              You cannot rename a field in the ``merge_layer`` to a field in the ``input_layer``. 
                              If you want to make field names equivalent, use Match.
                            * ``Match`` - A field in the merge_layer is made equivalent to a field in the ``input_layer`` 
                              specified by ``merge_layer``. For example, the input_layer has a field named CODE 
                              and the merge_layer has a field named STATUS. You can match STATUS to CODE, and 
                              the output will contain the CODE field with values of the STATUS field used for 
                              features copied from the merge_layer. Type casting is supported (for example, 
                              double to integer, integer to string) except for string to numeric.
                    
                      Example: [{"mergeLayerField": "Mean_Sales","mergeType": "Match","mergeValue": "Average_Sales"},{"mergeLayerField": "Bonus","mergeType": "Remove",},{"mergeLayerField": "Field4","mergeType": "Rename","mergeValue": "Errors"}]
    ----------------  ---------------------------------------------------------------
    output_name       Optional string. The task will create a feature service of the results. You define the name of the service.
    ----------------  ---------------------------------------------------------------
    gis               Optional GIS. The GIS object where the analysis will take place.
    ----------------  ---------------------------------------------------------------
    context           Optional dict. The context parameter contains additional settings that affect task execution. For this task, there are five settings:

                      #. Extent (``extent``) - A bounding box that defines the analysis area. Only those features that intersect the bounding box will be analyzed.
                      #. Processing spatial reference (``processSR``) - The features will be projected into this coordinate system for analysis.
                      #. Output spatial reference (``outSR``) - The features will be projected into this coordinate system after the analysis to be saved. The output spatial reference for the spatiotemporal big data store is always WGS84.
                      #. Data store (``dataStore``) - Results will be saved to the specified data store. The default is the spatiotemporal big data store.
                      #. Default aggregation styles (``defaultAggregationStyles``) - If set to 'True', results will have square, hexagon, and triangle aggregation styles enabled on results map services.
    ----------------  ---------------------------------------------------------------    
    future            Optional boolean. If 'True', a GPJob is returned instead of
                      results. The GPJob can be queried on the status of the execution.

                      The default value is 'False'.
    ================  ===============================================================

    :returns: result_layer : Output Features as feature layer item.

    .. code-block:: python

            # Usage Example: To merge census blocks from two states into one output layer.

            merge_result = merge_layers(input_layer=il_block,
                                       merge_layer=wi_block,
                                       merge_attributes=[{"mergeLayerField" : "State_Code", "mergeType" : "Match", "mergeValue" : "statecode"}],
                                       output_name="IL_WI_Census_Blocks")
    """
    kwargs = locals()
    tool_name = "MergeLayers"
    gis = _arcgis.env.active_gis if gis is None else gis
    url = gis.properties.helperServices.geoanalytics.url
    params = {
        "f" : "json",
    }
    for key, value in kwargs.items():

        if value is not None:
            params[key] = value
        elif key == 'merge_attributes' and value is None:
            params[key] = []
    if output_name is None:
        output_service_name = 'Merge_Layers_' + _id_generator()
        output_name = output_service_name.replace(' ', '_')
    else:
        output_service_name = output_name.replace(' ', '_')

    output_service = _create_output_service(gis, output_name, output_service_name, 'Merge Layers')

    params['output_name'] = _json.dumps({
        "serviceProperties": {"name" : output_name, "serviceUrl" : output_service.url},
        "itemProperties": {"itemId" : output_service.itemid}})

    if context is not None:
        params["context"] = context
    else:
        _set_context(params)

    param_db = {
        "input_layer": (_FeatureSet, "inputLayer"),
        "merge_layer": (_FeatureSet, "mergeLayer"),
        "merge_attributes" : (list, "mergingAttributes"),
        "output_name": (str, "outputName"),
        "context": (str, "context"),
        "output": (_FeatureSet, "output"),
    }
    return_values = [
        {"name": "output", "display_name": "Output Features", "type": _FeatureSet},
    ]
    try:
        if future:
            gpjob = _execute_gp_tool(gis, tool_name, params, param_db, return_values, _use_async, url, True, future=future)
            return GAJob(gpjob=gpjob, return_service=output_service)
        _execute_gp_tool(gis, tool_name, params, param_db, return_values, _use_async, url, True, future=future)
        return output_service
    except:
        output_service.delete()
        raise

    return


def clip_layer(input_layer, clip_layer, output_name=None, gis=None, context=None, future=False):
    """
    .. image:: _static/images/clip_layer/clip_layer.png 

    ``clip_layer`` features from one layer to the extent of a boundary layer. Use this tool to cut out a piece
    of one feature class using one or more of the features in another feature class as a cookie
    cutter. This is particularly useful for creating a new feature layers - also referred to as study
    area or area of interest (AOI) - that contains a geographic subset of the features in another,
    larger feature class.

    Only available at **ArcGIS Enterprise 10.7** and later.

    ================  ===============================================================
    **Argument**      **Description**
    ----------------  ---------------------------------------------------------------
    input_layer       Required feature layer. The point, line, or polygon features 
                      that will be clipped to the areas of ``clip_layer`` features. 
                      See :ref:`Feature Input<FeatureInput>`.
    ----------------  ---------------------------------------------------------------
    clip_layer        Required feature layer. The polygon features that define the 
                      areas to which ``input_layer`` features will be clipped. 
                      See :ref:`Feature Input<FeatureInput>`.
    ----------------  ---------------------------------------------------------------
    output_name       Optional string. The task will create a feature service of 
                      the results. You define the name of the service.
    ----------------  ---------------------------------------------------------------
    context           Optional strin. The context parameter contains additional 
                      settings that affect task execution. For this task, there are four settings:

                      #. Extent (``extent``) - A bounding box that defines the analysis area. 
                         Only those features that intersect the bounding box will be analyzed.
                      #. Processing spatial reference (``processSR``) - The features will be 
                         projected into this coordinate system for analysis.
                      #. Output spatial reference (``outSR``) - The features will be projected 
                         into this coordinate system after the analysis to be saved. 
                         The output spatial reference for the spatiotemporal big data store is always WGS84.
                      #. Data store (``dataStore``) - Results will be saved to the specified data store. 
                         The default is the spatiotemporal big data store.
    ----------------  ---------------------------------------------------------------
    gis               optional GIS. The GIS object where the analysis will take place.
    ----------------  ---------------------------------------------------------------
    future            Optional boolean. If True, a GPJob is returned instead of
                      results. The GPJob can be queried on the status of the execution.
    ================  ===============================================================

    :returns: feature layer collection

    .. code-block:: python

            # Usage Example: To clip the buffered area in the shape of Capitol Hill boundary.

            clipped = clip_layer(input_layer=buffer, 
                                 clip_layer=boundary,
                                 output_name="clipped_buffer",
                                 context={"extent":{'xmin': -77.50941999999998,'ymin': 38.389560000000074,'xmax': -76.50941999999998,'ymax': 39.389560000000074,"spatialReference":{"wkid":102100,"latestWkid":3857}}})

    """
    kwargs = locals()
    tool_name = "ClipLayer"
    gis = _arcgis.env.active_gis if gis is None else gis
    url = gis.properties.helperServices.geoanalytics.url
    params = {
        "f" : "json",
    }
    for key, value in kwargs.items():
        if value is not None:
            params[key] = value
    if output_name is None:
        output_service_name = 'Clip_Layers_' + _id_generator()
        output_name = output_service_name.replace(' ', '_')
    else:
        output_service_name = output_name.replace(' ', '_')

    output_service = _create_output_service(gis, output_name, output_service_name, 'Overlay Layers')

    params['output_name'] = _json.dumps({
        "serviceProperties": {"name" : output_name, "serviceUrl" : output_service.url},
        "itemProperties": {"itemId" : output_service.itemid}})

    if context is not None:
        params["context"] = context
    else:
        _set_context(params)

    param_db = {
        "input_layer": (_FeatureSet, "inputLayer"),
        "clip_layer": (_FeatureSet, "clipLayer"),
        "outputType" : (str, 'outputType'),
        "output_name": (str, "outputName"),
        "context": (str, "context"),
        "output": (_FeatureSet, "output"),
    }
    return_values = [
        {"name": "output", "display_name": "Output Features", "type": _FeatureSet},
    ]
    try:
        if future:
            gpjob = _execute_gp_tool(gis, tool_name, params, param_db, return_values, _use_async, url, True, future=future)
            return GAJob(gpjob=gpjob, return_service=output_service)
        _execute_gp_tool(gis, tool_name, params, param_db, return_values, _use_async, url, True, future=future)
        return output_service
    except:
        output_service.delete()
        raise

    return


def overlay_data(input_layer, 
                 overlay_layer, 
                 overlay_type="intersect", 
                 output_name=None, 
                 gis=None, 
                 include_overlaps=True,
                 context=None,
                 future=False):
    """
    .. image:: _static/images//overlay_layers/overlay_layers.png

    .. |Intersect| image:: _static/images/overlay_layers/Intersect.png
    .. |Union| image:: _static/images/overlay_layers/Union.png
    .. |Erase| image:: _static/images/overlay_layers/Erase.png
    .. |identity| image:: _static/images/overlay_layers/identity.png
    .. |symm| image:: _static/images/overlay_layers/Symmetric_Difference.png

    The ``overlay_data`` task combines two or more layers into one single layer. 
    You can think of overlay as peering through a stack of maps and creating a single 
    map containing all the information found in the stack. Overlay is used to answer 
    one of the most basic questions of geography: What is on top of what? 
    The following are examples:

        * What parcels are within the 100-year floodplain? ("Within" is another way of saying "on top of.")
        * What land use is within what soil type?
        * What mines are within abandoned military bases?

    .. note::
        Only available at ArcGIS Enterprise 10.6.1 and later.

    ======================  ===============================================================================
    **Argument**            **Description**
    ----------------------  -------------------------------------------------------------------------------
    input_layer             Required layer. The point, line, or polygon features that will be overlaid with the ``overlay_layer`` features. See :ref:`Feature Input<FeatureInput>`.
    ----------------------  -------------------------------------------------------------------------------
    overlay_layer           Required layer. The features that will be overlaid with the ``input_layer`` features.
    ----------------------  -------------------------------------------------------------------------------
    overlay_type            optional string. The type of overlay to be performed.

                            Choice list: ['intersect', 'erase', 'union', 'identity', 'symmetricaldifference']

                            +------------------------------------+-----------------------------------------------------------------------------------+
                            | |Intersect| ``intersect``          | Computes a geometric intersection of the input layers.                            |
                            |                                    | Features or portions of features that overlap in both the ``input_layer``         |
                            |                                    | and ``overlay_layer`` layers will be written to the output layer.                 |
                            |                                    |                                                                                   |
                            |                                    | * Point — Point, Line, Polygon                                                    |
                            |                                    | * Line — Point, Line, Polygon                                                     |
                            |                                    | * Polygon— Point, Line, Polygon                                                   |
                            +------------------------------------+-----------------------------------------------------------------------------------+
                            | |Erase| ``erase``                  | Only those features or portions of features in the ``overlay_layer``              |
                            |                                    | that are not within the features in the ``input_layer`` layer are                 |
                            |                                    | written to the output.                                                            |
                            |                                    |                                                                                   | 
                            |                                    | * Point — Point                                                                   |
                            |                                    | * Line — Line                                                                     |
                            |                                    | * Polygon — Polygon                                                               |
                            +------------------------------------+-----------------------------------------------------------------------------------+
                            | |Union| ``union``                  | Computes a geometric union of the ``input_layer`` and ``overlay_layer``.          |
                            |                                    | All features and their attributes will be written to the layer.                   |
                            |                                    |                                                                                   |
                            |                                    | * Polygon — Polygon                                                               |
                            +------------------------------------+-----------------------------------------------------------------------------------+
                            | |identity| ``identity``            | Computes a geometric intersection of the input features and                       |
                            |                                    | identity features. Features or portions of features that overlap in both          |
                            |                                    | ``input_layer`` and ``overlay_layer`` will be written to the output layer.        |
                            |                                    |                                                                                   |
                            |                                    |  * Point — Point, Polygon                                                         |
                            |                                    |  * Line — Line, Polygon                                                           |
                            |                                    |  * Polygon— Polygon                                                               |
                            +------------------------------------+-----------------------------------------------------------------------------------+
                            | |symm| ``symmetricaldifference``   | Features or portions of features in the ``input_layer``                           |
                            |                                    | and ``overlay_layer`` that do not overlap will be written to the output layer.    | 
                            |                                    |                                                                                   |
                            |                                    | * Point — Point                                                                   |
                            |                                    | * Line — Line                                                                     |
                            |                                    | * Polygon— Polygon                                                                |
                            +------------------------------------+-----------------------------------------------------------------------------------+
    ----------------------  -------------------------------------------------------------------------------
    include_overlaps        Optional boolean. Determines whether input features in the same dataset contain any overlapping features. 
                            This option should only be modified if you're not interested in self-intersection between 
                            features for the input layer and self-intersection between features for the overlay layer. 
                            Setting this value to false will improve performance. This parameter is only used when 
                            ``include_overlaps`` is Intersect with 10.6 and 10.6.1.

                            The default value is 'True'.
    ----------------------  -------------------------------------------------------------------------------
    output_name             Optional string. The task will create a feature service of the results. You define the name of the service.
    ----------------------  -------------------------------------------------------------------------------
    gis                     Optional GIS. The GIS object where the analysis will take place.
    ----------------------  -------------------------------------------------------------------------------
    context                 Optional dict. The context parameter contains additional settings that affect task execution. For this task, there are four settings:

                            #. Extent (``extent``) - A bounding box that defines the analysis area. Only those features that intersect the bounding box will be analyzed.
                            #. Processing spatial reference (``processSR``) - The features will be projected into this coordinate system for analysis.
                            #. Output spatial reference (``outSR``) - The features will be projected into this coordinate system after the analysis to be saved. The output spatial reference for the spatiotemporal big data store is always WGS84.
                            #. Data store (``dataStore``) - Results will be saved to the specified data store. The default is the spatiotemporal big data store.
    ----------------------  -------------------------------------------------------------------------------
    future                  Optional boolean. If 'True', a GPJob is returned instead of
                            results. The GPJob can be queried on the status of the execution.

                            The default value is 'False'.
    ======================  ===============================================================================

    :returns: result_layer : Output Features as feature layer item.

    .. code-block:: python

            # Usage Example: To find the intersecting areas between watersheds and grazing land in Missouri.

            overlay_result = manage_data.overlay_data(input_layer=grazing_land, 
                                          overlay_layer=watersheds_layer, 
                                          overlay_type="Intersect", 
                                          output_name="Watershed_intersections")   
    """
    kwargs = locals()
    tool_name = "OverlayLayers"
    gis = _arcgis.env.active_gis if gis is None else gis
    url = gis.properties.helperServices.geoanalytics.url
    params = {
        "f" : "json",
        "outputType" : "Input",
        'tolerance' : 0,
        'snapToInput' : 'false'
    }
    for key, value in kwargs.items():
        if value is not None:
            params[key] = value

    if output_name is None:
        output_service_name = 'Overlay_Layers_' + _id_generator()
        output_name = output_service_name.replace(' ', '_')
    else:
        output_service_name = output_name.replace(' ', '_')

    output_service = _create_output_service(gis, output_name, output_service_name, 'Overlay Layers')

    params['output_name'] = _json.dumps({
        "serviceProperties": {"name" : output_name, "serviceUrl" : output_service.url},
        "itemProperties": {"itemId" : output_service.itemid}})

    if context is not None:
        params["context"] = context
    else:
        _set_context(params)

    param_db = {
        "input_layer": (_FeatureSet, "inputLayer"),
        "overlay_layer": (_FeatureSet, "overlayLayer"),
        "outputType" : (str, 'outputType'),
        "overlay_type" : (str, "overlayType"),
        "output_name": (str, "OutputName"),
        "context": (str, "context"),
        'tolerance' : (int, 'tolerance'),
        "output": (_FeatureSet, "output"),
        'snapToInput' : (str, 'snapToInput')
    }
    return_values = [
        {"name": "output", "display_name": "Output Features", "type": _FeatureSet},
    ]
    try:
        if future:
            gpjob = _execute_gp_tool(gis, tool_name, params, param_db, return_values, _use_async, url, True, future=future)
            return GAJob(gpjob=gpjob, return_service=output_service)
        _execute_gp_tool(gis, tool_name, params, param_db, return_values, _use_async, url, True, future=future)
        return output_service
    except:
        output_service.delete()
        raise

    return


def append_data(input_layer, append_layer, field_mapping=None, gis=None, future=False):
    """
    Only available at ArcGIS Enterprise 10.6.1 and later.

    The Append Data task appends tabular, point, line, or polygon data to an existing layer.
    The input layer must be a hosted feature layer. The tool will add the appended data as
    rows to the input layer. No new output layer is created.

    ================  ===============================================================
    **Argument**      **Description**
    ----------------  ---------------------------------------------------------------
    input_layer       required FeatureLayer , The table, point, line or polygon features.
    ----------------  ---------------------------------------------------------------
    append_layer      required FeatureLayer. The table, point, line, or polygon features
                      to be appended to the input_layer. To append geometry, the
                      append_layer must have the same geometry type as the
                      input_layer. If the geometry types are not the same, the
                      append_layer geometry will be removed and all other matching
                      fields will be appended. The geometry of the input_layer will
                      always be maintained.
    ----------------  ---------------------------------------------------------------
    field_mapping     Defines how the fields in append_layer are appended to the
                      input_layer.

                      The following are set by default:

                        - All append_layer fields that match input_layer schema will be appended.
                        - Fields that exist in the input_layer and not in the append_layer will be appended with null values.
                        - Fields that exist in the append_layer and not in the input_layer will not be appended.

                      Optionally choose how input_layer fields will be appended from the following:

                      - AppendField - Matches the input_layer field with an append_layer field of a different name. Field types must match.
                      - Expression - Calculates values for the resulting field. Values are calculated using Arcade expressions. To assign null values, use 'null'.
    ----------------  ---------------------------------------------------------------
    gis               optional GIS, the GIS on which this tool runs. If not
                      specified, the active GIS is used.
    ----------------  ---------------------------------------------------------------
    future            optional Boolean. If True, a GPJob is returned instead of
                      results. The GPJob can be queried on the status of the execution.
    ================  ===============================================================

    :returns: boolean

    """
    kwargs = locals()
    tool_name = "AppendData"
    gis = _arcgis.env.active_gis if gis is None else gis
    url = gis.properties.helperServices.geoanalytics.url
    params = {
        "f" : "json"
    }
    for key, value in kwargs.items():
        if value is not None:
            params[key] = value

    _set_context(params)

    param_db = {
        "input_layer": (_FeatureSet, "inputLayer"),
        "append_layer": (_FeatureSet, "appendLayer"),
        "field_mapping" : (str, "fieldMapping"),
        "context": (str, "context")
    }
    return_values = [
    ]
    try:
        if future:
            gpjob = _execute_gp_tool(gis, tool_name, params, param_db, return_values, _use_async, url, True, future=future)
            return GAJob(gpjob=gpjob, return_service=None)
        _execute_gp_tool(gis, tool_name, params, param_db, return_values, _use_async, url, True, future=future)
        return True
    except:
        raise

    return False


def calculate_fields(input_layer,
                     field_name,
                     data_type,
                     expression,
                     track_aware=False,
                     track_fields=None,
                     time_boundary_split=None,
                     time_split_unit=None,
                     time_reference=None,
                     output_name=None,
                     gis=None,
                     context=None,
                     future=False
                     ):
    """

    .. image:: _static/images/calculate_field/calculate_field.png 

    The ``calculate_fields`` task works with a layer to create and populate a
    new field. The output is a new feature layer, that is the same as the
    input features, with the additional field added.

    =================================================   ===============================================================
    **Argument**                                        **Description**
    -------------------------------------------------   ---------------------------------------------------------------
    input_layer                                         Required layer. The input features that will have a field added and calculated.
                                                        See :ref:`Feature Input<FeatureInput>`.
    -------------------------------------------------   ---------------------------------------------------------------
    field_name                                          Required string. A string representing the name of the new
                                                        field. If the name already exists in the dataset, then a
                                                        numeric value will be appended to the field name.
    -------------------------------------------------   ---------------------------------------------------------------
    data_type                                           Required string. The type for the new field.
    
                                                        Choice list: ['Date', 'Double', 'Integer', 'String'`].
    -------------------------------------------------   ---------------------------------------------------------------
    expression                                          Required string. An Arcade expression used to calculate the new
                                                        field values. You can use any of the Date, Logical,
                                                        Mathematical or Text function available with Arcade.
    -------------------------------------------------   ---------------------------------------------------------------
    track_aware                                         Optional boolean. Boolean value denoting if the expression is
                                                        track aware.

                                                        The default value is 'False'.
    -------------------------------------------------   ---------------------------------------------------------------
    track_fields (Required if trackAware is True)       Optional string. The fields used to identify distinct tracks.
                                                        There can be multiple track_fields. track_fields are only
                                                        required when ``track_aware`` is True.
    -------------------------------------------------   ---------------------------------------------------------------
    time_boundary_split                                 Optional integer. A time boundary allows your to analyze values within a defined time span. 
                                                        For example, if you use a time boundary of 1 day, starting on January 1st, 1980 tracks will 
                                                        be analyzed 1 day at a time. The time boundary parameter was introduced in ArcGIS Enterprise 10.7.

                                                        The time boundary parameters are only applicable if the analysis is ``track_aware``.
                                                        The ``time_boundary_split`` parameter defines the scale of the time boundary. 
                                                        In the case above, this would be 1. See the portal documentation for this tool to learn more.
    -------------------------------------------------   ---------------------------------------------------------------
    time_split_unit                                     Optional string.  The unit to detect an incident is `time_boundary_split` is used.

                                                        Choice list: ['Years', 'Months', 'Weeks', 'Days', 'Hours', 'Minutes', 'Seconds', 'Milliseconds'].
    -------------------------------------------------   ---------------------------------------------------------------
    time_reference                                      Optional datetime.datetime. The starting date/time where analysis will
                                                        begin from.
    -------------------------------------------------   ---------------------------------------------------------------
    output_name                                         Optional string, The task will create a feature service of the
                                                        results. You define the name of the service.
    -------------------------------------------------   ---------------------------------------------------------------
    gis                                                 Optional GIS, the GIS on which this tool runs. If not
                                                        specified, the active GIS is used.
    -------------------------------------------------   ---------------------------------------------------------------
    context                                             Optional dict. The context parameter contains additional settings that affect task execution. For this task, there are four settings:

                                                        #. Extent (``extent``) - A bounding box that defines the analysis area. Only those features that intersect the bounding box will be analyzed.
                                                        #. Processing spatial reference (``processSR``) - The features will be projected into this coordinate system for analysis.
                                                        #. Output spatial reference (``outSR``) - The features will be projected into this coordinate system after the analysis to be saved. The output spatial reference for the spatiotemporal big data store is always WGS84.
                                                        #. Data store (``dataStore``) - Results will be saved to the specified data store. The default is the spatiotemporal big data store.
    =================================================   ===============================================================
    

    :returns: feature layer collection

    .. code-block:: python

            # Usage Example: To find maximum of the two attributes of an input layer.

            result = calculate_fields(input_layer=lyr,
                                      field_name="avg",
                                      data_type="Double",
                                      expression='max($feature["InputValue"],$feature["Value2"])')
    """
    kwargs = locals()
    tool_name = "CalculateField"
    gis = _arcgis.env.active_gis if gis is None else gis
    url = gis.properties.helperServices.geoanalytics.url
    params = {
        "f" : "json"
    }
    for key, value in kwargs.items():
        if value is not None:
            params[key] = value

    if output_name is None:
        output_service_name = 'Calculate_Fields_' + _id_generator()
        output_name = output_service_name.replace(' ', '_')
    else:
        output_service_name = output_name.replace(' ', '_')

    output_service = _create_output_service(gis, output_name, output_service_name, 'Calculate Fields')

    params['output_name'] = _json.dumps({
        "serviceProperties": {"name" : output_name, "serviceUrl" : output_service.url},
        "itemProperties": {"itemId" : output_service.itemid}})

    if context is not None:
        params["context"] = context
    else:
        _set_context(params)

    param_db = {
        "input_layer": (_FeatureSet, "inputLayer"),
        "field_name" : (str, "fieldName"),
        "data_type" : (str, "dataType"),
        "expression" : (str, "expression"),
        "track_aware" : (bool, "trackAware"),
        "track_fields" : (str, "trackFields"),
        "time_boundary_split" : (int, "timeBoundarySplit"),
        "time_split_unit" : (str, "timeBoundarySplitUnit"),
        "time_reference" : (_datetime.datetime, "timeBoundaryReference"),
        "output_name": (str, "outputName"),
        "output": (_FeatureSet, "output"),
        "context": (str, "context")
    }
    return_values = [
        {"name": "output", "display_name": "Output Features", "type": _FeatureSet},
    ]
    try:
        if future:
            gpjob = _execute_gp_tool(gis, tool_name, params, param_db, return_values, _use_async, url, True, future=future)
            return GAJob(gpjob=gpjob, return_service=output_service)
        _execute_gp_tool(gis, tool_name, params, param_db, return_values, _use_async, url, True, future=future)
        return output_service
    except:
        output_service.delete()
        raise

    return

def copy_to_data_store(
    input_layer,
    output_name=None,
    gis=None,
    context=None,
    future=False):
    """
    .. image:: _static/images/copy_to_data_store/copy_to_data_store.png 

    The ``copy_to_data_store`` task takes an input layer and copies it to a data store. 
    Data is copied to ArcGIS Data Store, configured as either a relational or 
    spatiotemporal big data store.
    
    For example

        * Copy a collection of .csv files in a big data file share to the spatiotemporal data store for visualization.
        * Copy the features in the current map extent that are stored in the spatiotemporal data store to the relational data store.

    This tool will take an input layer and copy it to a data store. Data will be copied to the ArcGIS Data Store 
    and will be stored in your relational or spatiotemporal data store.

    For example, you could copy features that are stored in a big data file share to a relational data store 
    and specify that only features within the current map extent will be copied. This would create a hosted 
    feature service with only those features that were within the specified map extent.

    ==========================   ===============================================================
    **Argument**                 **Description**
    --------------------------   ---------------------------------------------------------------
    input_layer                  Required layer. The table, point, line, or polygon features that will be copied. See :ref:`Feature Input<FeatureInput>`.
    --------------------------   ---------------------------------------------------------------
    output_name                  Optional string. The task will create a feature service of the results. You define the name of the service.
    --------------------------   --------------------------------------------------------------- 
    gis                          Optional GIS. The GIS on which this tool runs. If not specified, the active GIS is used.
    --------------------------   --------------------------------------------------------------- 
    context                      Optional string. The context parameter contains additional settings that affect task execution. For this task, there are five settings:

                                 #. Extent (``extent``) - A bounding box that defines the analysis area. Only those features that intersect the bounding box will be analyzed.
                                 #. Processing spatial reference (``processSR``) - The features will be projected into this coordinate system for analysis.
                                 #. Output spatial reference (``outSR``) - The features will be projected into this coordinate system after the analysis to be saved. The output spatial reference for the spatiotemporal big data store is always WGS84.
                                 #. Data store (``dataStore``) - Results will be saved to the specified data store. The default is the spatiotemporal big data store.
                                 #. Default aggregation styles (``defaultAggregationStyles``) - If set to 'True', results will have square, hexagon, and triangle aggregation styles enabled on results map services.
    --------------------------   --------------------------------------------------------------- 
     future                      Optional boolean. If 'True', the result comes back as a GPJob.

                                 The default value is 'False'.
    ==========================   ===============================================================

    :returns: result_layer : Output Features as feature layer item.

    .. code-block:: python

            # Usage Example: To copy input layer to a data store. 
            copy_result = copy_to_data_store(input_layer=earthquakes,
                                             output_name="copy earthquakes data")
    """
    kwargs = locals()

    gis = _arcgis.env.active_gis if gis is None else gis
    url = gis.properties.helperServices.geoanalytics.url

    params = {}
    for key, value in kwargs.items():
        if value is not None:
            params[key] = value

    if output_name is None:
        output_service_name = 'Data Store Copy_' + _id_generator()
        output_name = output_service_name.replace(' ', '_')
    else:
        output_service_name = output_name.replace(' ', '_')
    _set_context(params)
    if 'context' in params and \
       params['context'].lower().find('bigdatafileshares') > -1:
        params['output_name'] = output_name
        output_service = True
    else:
        output_service = _create_output_service(gis, output_name, output_service_name, 'Copy To Data Store')
        params['output_name'] = _json.dumps({
            "serviceProperties": {"name" : output_name, "serviceUrl" : output_service.url},
            "itemProperties": {"itemId" : output_service.itemid}})

    #_set_context(params)

    if context is not None:
        params["context"] = context
    else:
        _set_context(params)

    param_db = {
        "input_layer": (_FeatureSet, "inputLayer"),
        "output_name": (str, "outputName"),
        "context": (str, "context"),
        "output": (_FeatureSet, "Output Layer"),
    }
    return_values = [
        {"name": "output", "display_name": "Output Layer", "type": _FeatureSet},
    ]
    try:
        if future:
            gpjob = _execute_gp_tool(gis, "CopyToDataStore", params, param_db, return_values, _use_async, url, True, future=future)
            return GAJob(gpjob=gpjob, return_service=output_service)
        _execute_gp_tool(gis, "CopyToDataStore", params, param_db, return_values, _use_async, url, True, future=future)
        return output_service
    except:
        if isinstance(output_service, bool):
            return False
        output_service.delete()
        raise

copy_to_data_store.__annotations__ = {
    'output_name': str}





