# -*- coding: utf-8 -*-
#######
# actinia-core - an open source REST API for scalable, distributed, high
# performance processing of geographical data that uses GRASS GIS for
# computational tasks. For details, see https://actinia.mundialis.de/
#
# Copyright (c) 2016-2021 Sören Gebbert and mundialis GmbH & Co. KG
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <https://www.gnu.org/licenses/>.
#
#######

"""
Process chain
"""
import os
import requests

from actinia_core.core.stac_importer_interface import STACImporter as STAC
from .process_object import Process
from .exceptions import AsyncProcessError
from actinia_core.core.geodata_download_importer import (
    GeoDataDownloadImportSupport,
)
from .config import global_config
from .sentinel_processing_library import Sentinel2Processing
from .landsat_processing_library import LandsatProcessing
from .google_satellite_bigquery_interface import (
    GoogleSatelliteBigQueryInterface,
)
from requests.auth import HTTPBasicAuth
from actinia_core.models.process_chain import (
    ProcessChainModel,
    InputParameter,
    OutputParameter,
    GrassModule,
    StdoutParser,
    Executable,
    SUPPORTED_EXPORT_FORMATS,
)

__license__ = "GPLv3"
__author__ = (
    "Sören Gebbert, Carmen Tawalika, Guido Riembauer, Julia Haas,"
    " Anika Weinmann"
)
__copyright__ = (
    "Copyright 2016-2022, Sören Gebbert and mundialis GmbH & Co. KG"
)
__maintainer__ = "mundialis"


class ProcessChainConverter(object):
    """
    Convert the process chain description into a process list that can be
    executed
    """

    def __init__(
        self,
        config=None,
        temp_file_path=None,
        process_dict=None,
        temporary_pc_files=None,
        required_mapsets=None,
        resource_export_list=None,
        message_logger=None,
        output_parser_list=None,
        send_resource_update=None,
    ):
        """Constructor to convert the process chain into a process list

        Args:
            config: The actinia configuration object
            temp_file_path: The path to the temporary directory to store
                            temporary files. It is assumed that this path is
                            available when the generated commands are executed.
            message_logger: The message logger to be used
            process_dict (dict): The dictionary that describes the process
                                 chain
            temporary_pc_files (dict): A dictionary of temporary process chain
                                       files that are generated by this class
            required_mapsets (list): A list that will be filled with mapsets
                                     names that must be linked for processing
            resource_export_list (list): A list that will be filled with export
                                         definitions found in output
                                         descriptions for geodata export
            output_parser_list (list): A list that will be filled with output
                                       parser definitions found in module
                                       descriptions. The stdout definition will
                                       be stored in a dict that has the process
                                       id a key {process_id:StdoutParser}
            send_resource_update: The function to call for resource updates

        Returns:

        """
        if config is None:
            self.config = global_config
        else:
            self.config = config
        if temp_file_path is None:
            self.temp_file_path = "/tmp"
        else:
            self.temp_file_path = temp_file_path
        self.temp_file_count = 0

        if process_dict is None:
            self.process_dict = {}
        else:
            self.process_dict = process_dict
        if temporary_pc_files is None:
            self.temporary_pc_files = {}
        else:
            self.temporary_pc_files = temporary_pc_files
        if required_mapsets is None:
            self.required_mapsets = []
        else:
            self.required_mapsets = required_mapsets
        if resource_export_list is None:
            self.resource_export_list = []
        else:
            self.resource_export_list = resource_export_list
        if output_parser_list is None:
            self.output_parser_list = []
        else:
            self.output_parser_list = output_parser_list

        self.send_resource_update = send_resource_update
        self.message_logger = message_logger
        self.import_descr_list = []
        self.webhook_finished = None
        self.webhook_update = None
        self.webhook_auth = None

    def process_chain_to_process_list(self, process_chain):
        if not process_chain:
            raise AsyncProcessError("Process chain is empty")

        if "list" in process_chain and "version" in process_chain:
            return self._process_chain_to_process_list(process_chain)
        else:
            return self._process_chain_to_process_list_legacy(process_chain)

    def _process_chain_to_process_list(self, process_chain):
        """
        Transform the module chain description into an ordered list of process
        runs

        All input map layer from a specific mapset
        MUST be specified with the mapset they belong to: map@mapset.
        Maps from an ephemeral mapset should not contain a mapset name,
        because the mapset names are temporary and created with random names.

        Args:
            process_chain (dict): The process chain

        Return:
             list
             A list of ordered grass processes

        """
        process_list = []

        if "version" not in process_chain:
            raise AsyncProcessError(
                "Version information is missing "
                "in the process chain definition"
            )

        if "list" not in process_chain:
            raise AsyncProcessError(
                "List of processes to be executed is missing "
                "in the process chain definition"
            )

        # Check for the webhooks
        if "webhooks" in process_chain:
            if "finished" in process_chain["webhooks"]:
                self.webhook_finished = process_chain["webhooks"]["finished"]
                self._check_if_webhook_exists(
                    self.webhook_finished, process_chain, "finished"
                )
            else:
                raise AsyncProcessError(
                    "The finished URL is missing in the webhooks definition."
                )

            if "update" in process_chain["webhooks"]:
                self.webhook_update = process_chain["webhooks"]["update"]
                self._check_if_webhook_exists(
                    self.webhook_update, process_chain, "update"
                )

        for process_descr in process_chain["list"]:
            if "module" in process_descr:
                module = self._create_module_process(process_descr)
                if module:
                    process_list.append(module)
            elif "exe" in process_descr:
                exe = self._create_exec_process(process_descr)
                if exe:
                    process_list.append(exe)
            elif "evaluate" in process_descr:
                process_list.append(("python", process_descr["evaluate"]))
            else:
                raise AsyncProcessError(
                    "Unknown process description "
                    "in the process chain definition"
                )
        downimp_list = self._create_download_process_list()
        downimp_list.extend(process_list)

        return downimp_list

    def _get_landsat_import_download_commands(self, entry):
        """Helper method to get the landsat import and download commands.

        Args:
            entry (dict): Entry of the import description list

        Returns:
            landsat_commands: The landsat download and import commands
        """
        # Check for band information
        if "landsat_atcor" not in entry["import_descr"]:
            raise AsyncProcessError(
                "Atmospheric detection specification is required for Landsat "
                "scene import"
            )

        atcor = entry["import_descr"]["landsat_atcor"].lower()

        if atcor not in [
            "uncorrected",
            "dos1",
            "dos2",
            "dos2b",
            "dos3",
            "dos4",
        ]:
            raise AsyncProcessError(
                "Atmospheric detection specification is wrong for "
                "Landsat scene import."
            )

        scene = entry["import_descr"]["source"]

        lp = LandsatProcessing(
            config=self.config,
            download_cache=self.temp_file_path,
            temp_file_path=self.temp_file_path,
            message_logger=self.message_logger,
            send_resource_update=self.send_resource_update,
            scene_id=scene,
        )

        download_commands, import_file_info = lp.get_download_process_list()
        import_commands = lp.get_import_process_list()
        atcor_commands = lp.get_i_landsat_toar_process_list(atcor)
        landsat_commands = download_commands
        landsat_commands.extend(import_commands)
        landsat_commands.extend(atcor_commands)

        return landsat_commands

    def _get_sentinel_import_command(self, entry):
        """Helper method to get the sentinel import command.

        Args:
            entry (dict): Entry of the import description list

        Returns:
            sentinel_commands: The sentinel import commands
        """
        # Check for band information
        if "sentinel_band" not in entry["import_descr"]:
            raise AsyncProcessError(
                "Band specification is required for Sentinel2 scene import"
            )

        scene = entry["import_descr"]["source"]
        band = entry["import_descr"]["sentinel_band"]

        gqi = GoogleSatelliteBigQueryInterface(config=self.config)
        query_result = gqi.get_sentinel_urls(
            product_ids=[
                scene,
            ],
            bands=[
                band,
            ],
        )
        sp = Sentinel2Processing(
            config=self.config,
            bands=[
                band,
            ],
            download_cache=self.temp_file_path,
            temp_file_path=self.temp_file_path,
            message_logger=self.message_logger,
            send_resource_update=self.send_resource_update,
            product_id=scene,
            query_result=query_result,
        )

        (
            download_commands,
            import_file_info,
        ) = sp.get_sentinel2_download_process_list()
        sentinel_commands = download_commands
        import_commands = sp.get_sentinel2_import_process_list()
        sentinel_commands.extend(import_commands)

        input_file, map_name = import_file_info[band]
        p = Process(
            exec_type="grass",
            executable="g.rename",
            id=f"rename_{entry['value']}",
            executable_params=[
                "raster=%s,%s" % (map_name, entry["value"]),
            ],
        )
        sentinel_commands.append(p)

        return sentinel_commands

    def _collect_sentinel_scenes_bands(self, entries):
        """Helper Method to collect all individual scenes and bands from
        different importer modules used throughout the process chain
        """
        scenes_bands = []
        # sort by source (scene ID) and bands
        scene_ids = []
        for entry in entries:
            scene_id = entry["import_descr"]["source"]
            band = entry["import_descr"]["sentinel_band"]
            output = entry["value"]
            if scene_id not in scene_ids:
                scene_ids.append(scene_id)
                scene = {
                    "scene_id": scene_id,
                    "bands": [band],
                    "outputs": [output],
                }
                scenes_bands.append(scene)
            else:
                scindex = scene_ids.index(scene_id)
                scenes_bands[scindex]["bands"].append(band)
                scenes_bands[scindex]["outputs"].append(output)
        return scenes_bands

    def _get_sentinel_import_commands(self, entries):
        """Method to get the Sentinel download and import commands using GCS
        without login
        """
        sentinel_commands = []
        scenes_bands = self._collect_sentinel_scenes_bands(entries)
        for scene in scenes_bands:
            scene_id = scene["scene_id"]
            download_cache = f"download_cache_{scene_id}"
            self.temporary_pc_files[
                download_cache
            ] = self.generate_temp_file_path()
            sp = Sentinel2Processing(
                bands=scene["bands"],
                download_cache=self.temporary_pc_files[download_cache],
                message_logger=self.message_logger,
                send_resource_update=self.send_resource_update,
                product_id=scene_id,
            )

            (
                download_commands,
                import_file_info,
            ) = sp.get_sentinel2_download_process_list_without_query()
            scene_commands = download_commands
            import_commands = (
                sp.get_sentinel2_import_process_list_without_query()
            )
            scene_commands.extend(import_commands)
            for idx, band in enumerate(scene["bands"]):
                output = scene["outputs"][idx]
                p = Process(
                    exec_type="grass",
                    executable="g.rename",
                    id=f"rename_{scene_id}_{band}",
                    executable_params=[
                        f"raster={import_file_info[band][1]},{output}"
                    ],
                )
                scene_commands.append(p)
            sentinel_commands.extend(scene_commands)
        return sentinel_commands

    def _get_postgis_import_command(self, entry):
        """Helper method to get the import command for postgis.

        Args:
            entry (dict): Entry of the import description list

        Returns:
            import_command: The import command for postgis
        """
        dbstring = "%s" % entry["import_descr"]["source"]
        vector_name = entry["value"]
        layer = None
        if "vector_layer" in entry["import_descr"]:
            layer = entry["import_descr"]["vector_layer"]
        import_command = (
            GeoDataDownloadImportSupport.get_vector_import_command(
                input_source=dbstring,
                vector_name=vector_name,
                layer_name=layer,
            )
        )
        return import_command

    def _get_raster_vector_file_download_import_command(self, entry):
        """
        Helper method to get the download and import commands for raster/vector
        and files.

        Args:
            entry (dict): Entry of the import description list

        Returns:
            rvf_downimport_commands (list): The raster/vector/file download and
                                            import commands
        """
        rvf_downimport_commands = list()
        url = entry["import_descr"]["source"]

        if not url.startswith("/vsicurl/"):
            gdis = GeoDataDownloadImportSupport(
                config=self.config,
                temp_file_path=self.temp_file_path,
                download_cache=self.temp_file_path,
                message_logger=self.message_logger,
                send_resource_update=self.send_resource_update,
                url_list=[
                    url,
                ],
            )
            (
                download_commands,
                import_file_info,
            ) = gdis.get_download_process_list()
            rvf_downimport_commands.extend(download_commands)
            input_source = import_file_info[0][2]
        else:
            input_source = url

        map_name = entry["value"]
        layer = None

        if "vector_layer" in entry["import_descr"]:
            layer = entry["import_descr"]["vector_layer"]
        if entry["import_descr"]["type"] == "raster":
            kwargs = {"file_path": input_source, "raster_name": entry["value"]}
            resamp_opt = [
                "nearest",
                "bilinear",
                "bicubic, lanczos",
                "bilinear_f",
                "bicubic_f",
                "lanzcos_f",
            ]
            resol_opt = ["estimated", "value", "region"]

            if "resample" in entry["import_descr"]:
                if entry["import_descr"]["resample"] in resamp_opt:
                    kwargs["resample"] = entry["import_descr"]["resample"]
                else:
                    raise AsyncProcessError(
                        "Error while running executable <r.import>"
                        " Please check if parameter"
                        " <resample> is set correctly."
                    )

            if "resolution" in entry["import_descr"]:
                if entry["import_descr"]["resolution"] in resol_opt:
                    kwargs["resolution"] = entry["import_descr"]["resolution"]
                else:
                    raise AsyncProcessError(
                        "Error while running executable <r.import>."
                        " Please check if parameter"
                        " <resolution> is set correctly."
                    )
                if (
                    kwargs["resolution"] == "value"
                    and "resolution_value" not in entry["import_descr"]
                ):
                    raise AsyncProcessError(
                        "Error while running executable <r.import>."
                        " Please check if parameter"
                        " <resolution_value> is set."
                    )

            if "resolution_value" in entry["import_descr"]:
                try:
                    float(entry["import_descr"]["resolution_value"])
                    kwargs["resolution_value"] = entry["import_descr"][
                        "resolution_value"
                    ]
                except ValueError:
                    raise AsyncProcessError(
                        "Error while running executable <r.import>. Value for "
                        "parameter <resolution_value> is not a float."
                    )
                if "resolution" not in entry["import_descr"]:
                    raise AsyncProcessError(
                        "Error while running executable <r.import>. Please "
                        "check if parameter <resolution> is set."
                    )
                if (
                    "resolution" in entry["import_descr"]
                    and entry["import_descr"]["resolution"] != "value"
                ):
                    raise AsyncProcessError(
                        "Error while running executable <r.import>. Please "
                        "check if parameter <resolution> is set to <value>."
                    )

            import_command = (
                GeoDataDownloadImportSupport.get_raster_import_command(
                    **kwargs
                )
            )
            rvf_downimport_commands.append(import_command)
        if entry["import_descr"]["type"] == "vector":
            import_command = (
                GeoDataDownloadImportSupport.get_vector_import_command(
                    input_source=input_source,
                    vector_name=map_name,
                    layer_name=layer,
                )
            )
            rvf_downimport_commands.append(import_command)
        if entry["import_descr"]["type"] == "file":
            # Search for file identifiers
            if "$file" in map_name and "::" in map_name:
                file_id = map_name.split("::")[1]
                # Use the temporary file name as copy target
                if file_id in self.temporary_pc_files:
                    rename_commands = (
                        GeoDataDownloadImportSupport.get_file_rename_command(
                            import_file_info[0][2],
                            self.temporary_pc_files[file_id],
                        )
                    )
                    rvf_downimport_commands.append(rename_commands)
                else:
                    raise AsyncProcessError(
                        "A file id is required for a download file "
                        "to use it in the process chain."
                    )
            else:
                raise AsyncProcessError(
                    "A file id is required for a download file "
                    "to use it in the process chain."
                )
        return rvf_downimport_commands

    def generate_temp_file_path(self):
        """
        Generate the path of a new unique temporary file that will be removed
        when the processe finishes. The _setup() method must be called for
        correct file generate.

        Returns: The file path to a unique temporary file
        """

        if not self.temp_file_path:
            raise AsyncProcessError(
                "Unable to create temporary file, "
                "temp_file_path is not set."
            )

        self.temp_file_count += 1

        return os.path.join(
            self.temp_file_path, "temp_file_%i" % self.temp_file_count
        )

    def _create_module_process(self, module_descr):
        """Analyse a grass process description dict and create a process
        that is used to execute a GRASS GIS binary.

        - Identify the required mapsets from the input definition and store
          them in a list.
        - Identify input and output options
        - Add export options to the export list

        Args:
            module_descr (dict): The module description

        Returns: Process:
            An object of type Process that contains the module name,
            the parameter list and stdout/stdin definitions

        """
        if self.message_logger:
            self.message_logger.info(str(module_descr))
        params = []

        if "id" not in module_descr:
            raise AsyncProcessError(
                "The <id> is missing from the process description."
            )

        id = module_descr["id"]

        stdin_func = None
        if "stdin" in module_descr:
            stdin_func = self._create_stdin_process(module_descr, id)

        # Check for stdout parser that can be of type table, list or key/value
        # pairs and store the definition in a list
        if "stdout" in module_descr:
            self._check_and_get_stdout_command(module_descr, id)

        if "module" not in module_descr:
            raise AsyncProcessError(
                "Missing module name in module description of id %s" % str(id)
            )

        module_name = module_descr["module"]

        if "inputs" in module_descr:
            self._add_grass_module_input_parameter_to_list(
                module_descr, params, id
            )

        if "outputs" in module_descr:
            self._add_grass_module_output_parameter_to_list(
                module_descr, params, id, module_name
            )

        if "flags" in module_descr:
            if "flags" in module_descr:
                params.append("-" + str(module_descr["flags"]))

        if "overwrite" in module_descr and module_descr["overwrite"] is True:
            params.append("--o")

        if "superquiet" in module_descr and module_descr["superquiet"] is True:
            params.append("--qq")

        if "verbose" in module_descr and module_descr["verbose"] is True:
            params.append("--v")

        if (
            "interface-description" in module_descr
            and module_descr["interface-description"] is True
        ):
            params.append("--interface-description")

        # Check for un-allowed characters in the parameter list
        for entry in params:
            if "&" in entry and module_name not in (
                "r.mapcalc",
                "t.rast.mapcalc",
                "t.rast.algebra",
                "t.rast.bandcalc",
            ):
                raise AsyncProcessError(
                    "Character '&' not allowed in "
                    "parameters for %s" % module_name
                )

        if (
            module_name != "importer" and module_name != "exporter"
        ) or params == ["--interface-description"]:
            p = Process(
                exec_type="grass",
                executable=module_name,
                executable_params=params,
                stdin_source=stdin_func,
                id=id,
            )

            self.process_dict[id] = p
            return p

        return None

    def _create_stdin_process(self, module_descr, id):
        """Helper methods to create stdin process.

        Args:
            module_descr (dict): The module description
            id (str): The id of this process in the process chain

        Returns:
            stdin_func(Process): An object of type Process that
                                 contains the module name, the
                                 parameter list and stdin definitions

        """
        if "::" not in module_descr["stdin"]:
            raise AsyncProcessError(
                "The stdin option in id %s misses the ::" % str(id)
            )
        object_id, method = module_descr["stdin"].split("::")
        if "stdout" == method is True:
            stdin_func = self.process_dict[object_id].stdout
        elif "stderr" == method is True:
            stdin_func = self.process_dict[object_id].stderr
        else:
            raise AsyncProcessError(
                "The stdout or stderr flag in id %s is missing" % str(id)
            )
        return stdin_func

    def _create_exec_process(self, module_descr):
        """Analyse a grass process description dict and create a Process
        that is used to execute a common Linux binary.

        Args:
            id (str): The id of this process in the process chain
            module_descr (dict): The module description

        Returns: Process:
            An object of type Process that contains the module name,
            the parameter list and stdout/stdin definitions

        """
        if self.message_logger:
            self.message_logger.info(str(module_descr))

        id = module_descr["id"]

        stdin_func = None
        if "stdin" in module_descr:
            if "::" not in module_descr["stdin"]:
                raise AsyncProcessError(
                    "The stdin option in id %s misses the ::" % str(id)
                )

            object_id, method = module_descr["stdin"].split("::")

            if "stdout" in method:
                stdin_func = self.process_dict[object_id].get_stdout
            elif "stderr" in method:
                stdin_func = self.process_dict[object_id].get_stderr
            else:
                raise AsyncProcessError(
                    "The stdout or stderr flag in "
                    "id %s is missing" % str(id)
                )

        if "exe" not in module_descr:
            raise AsyncProcessError(
                "Missing executable name in module "
                "description of id %s" % str(id)
            )

        executable = module_descr["exe"]

        params = []

        if "params" in module_descr:
            for search_string in module_descr["params"]:
                # Search for file identifiers and generate the temporary file
                # path
                if "$file" in search_string and "::" in search_string:
                    file_id = search_string.split("::")[1]
                    # Generate the temporary file path and store it in the dict
                    if file_id not in self.temporary_pc_files:
                        self.temporary_pc_files[
                            file_id
                        ] = self.generate_temp_file_path()

                    param = "%s" % self.temporary_pc_files[file_id]
                else:
                    param = "%s" % search_string

                params.append(param)

        # Check for un-allowed characters in the parameter list
        for entry in params:
            if "&" in entry:
                raise AsyncProcessError(
                    "Character '&' not supported in process "
                    "description for %s" % executable
                )

        p = Process(
            exec_type="exec",
            executable=executable,
            executable_params=params,
            stdin_source=stdin_func,
            id=id,
        )

        self.process_dict[id] = p

        return p

    def _create_download_process_list(self):
        """This function analysis the process chain import options and creates
        download and import commands.

        Returns:

        """

        downimp_list = []

        if self.message_logger:
            self.message_logger.info(
                "Creating download process " "list for all import definitions"
            )

        sentinel2_entries = []
        for entry in self.import_descr_list:
            if self.message_logger:
                self.message_logger.info(entry)

            check_required_keys_for_download_process_chain(entry)

            # RASTER; VECTOR, FILE
            if (
                entry["import_descr"]["type"].lower() == "raster"
                or entry["import_descr"]["type"].lower() == "vector"
                or entry["import_descr"]["type"].lower() == "file"
            ):
                rvf_downimport_commands = (
                    self._get_raster_vector_file_download_import_command(entry)
                )
                downimp_list.extend(rvf_downimport_commands)

            # POSTGIS
            elif entry["import_descr"]["type"].lower() == "postgis":
                import_command = self._get_postgis_import_command(entry)
                downimp_list.append(import_command)

            # SENTINEL
            elif entry["import_descr"]["type"].lower() == "sentinel2":
                # Old style using Google Big Query. Uncomment/Comment in here
                # to switch.
                # sentinel_commands = self._get_sentinel_import_command(entry)
                # downimp_list.extend(sentinel_commands)
                sentinel2_entries.append(entry)

            # LANDSAT
            elif entry["import_descr"]["type"].lower() == "landsat":
                landsat_commands = self._get_landsat_import_download_commands(
                    entry
                )
                downimp_list.extend(landsat_commands)

            # STAC
            elif entry["import_descr"]["type"].lower() == "stac":
                stac = STAC()
                stac_commands = stac.get_stac_import_download_commands(
                    stac_entry=entry
                )

                downimp_list.extend(stac_commands)

            else:
                raise AsyncProcessError(
                    "Unknown import type specification: %s"
                    % entry["import_descr"]["type"]
                )

        if len(sentinel2_entries) > 0:
            # put all Sentinel-2 downloading together
            sentinel_commands = self._get_sentinel_import_commands(
                sentinel2_entries
            )
            downimp_list.extend(sentinel_commands)
        return downimp_list

    # TODO: remove legacy methods and do no use them in actinia_core
    def _create_module_process_legacy(self, id, module_descr):
        """Analyse a grass process description dict and create a Process
        that is used to execute a GRASS GIS binary.

        Identify the required mapsets from the input definition and stores them
        in a list.

        Args:
            id (str): The id of this process in the process chain
            module_descr (dict): The module description

        Returns: Process:
            An object of type Process that contains the module name,
            the parameter list and stdout/stdin definitions

        """
        self.message_logger.info(str(module_descr))
        parameters = []

        stdin_func = None
        if "stdin" in module_descr:
            if "::" not in module_descr["stdin"]:
                raise AsyncProcessError(
                    "The stdin option in id %s misses the ::" % str(id)
                )

            object_id, method = module_descr["stdin"].split("::")

            if "stdout" == method is True:
                stdin_func = self.process_dict[object_id].stdout

            elif "stderr" == method is True:
                stdin_func = self.process_dict[object_id].stderr
            else:
                raise AsyncProcessError(
                    "The stdout or stderr flag in id %s is missing" % str(id)
                )

        if "module" not in module_descr:
            raise AsyncProcessError(
                "Missing module name in module description of id %s" % str(id)
            )

        module_name = module_descr["module"]

        if "inputs" in module_descr:
            for key in module_descr["inputs"]:
                search_string = str(module_descr["inputs"][key])

                # Search for file identifiers and generate the temporary file
                # path
                if "$file" in search_string and "::" in search_string:
                    file_id = search_string.split("::")[1]
                    # Generate the temporary file path and store it in the dict
                    if file_id not in self.temporary_pc_files:
                        self.temporary_pc_files[
                            file_id
                        ] = self.generate_temp_file_path()

                    param = "%s=%s" % (key, self.temporary_pc_files[file_id])
                else:
                    param = "%s=%s" % (key, module_descr["inputs"][key])
                    # Check for mapset in input name and append it
                    # to the list of required mapsets
                    if "@" in str(module_descr["inputs"][key]):
                        # Mapset names are after an @ symbol
                        # Mapsets in expressions can be detected by replacing
                        # the symbols like *, +, :, /, {, (,},], ... by spaces
                        # and split the string by spaces, searching in each
                        # substring for @

                        symbols = [
                            "*",
                            "+",
                            "-",
                            "/",
                            "%",
                            "$",
                            "!",
                            ":",
                            "(",
                            ")",
                            "{",
                            "}",
                            "&",
                            "?",
                            "#",
                            "=",
                            "^",
                            "~",
                            "<",
                            ">",
                            "\\",
                        ]

                        for symbol in symbols:
                            search_string = search_string.replace(symbol, " ")

                        search_strings = search_string.split(" ")

                        for entry in search_strings:
                            for subentry in entry.split(","):
                                if "@" in subentry:
                                    mapset = subentry.split("@")[1]
                                    if mapset not in self.required_mapsets:
                                        self.required_mapsets.append(mapset)

                parameters.append(param)

        if "outputs" in module_descr:
            for key in module_descr["outputs"]:
                if "name" in module_descr["outputs"][key]:
                    search_string = module_descr["outputs"][key]["name"]

                    # Search for file identifiers and generate the temporary
                    # file path
                    if "$file" in search_string and "::" in search_string:
                        file_id = search_string.split("::")[1]
                        # Generate the temporary file path and store it in the
                        # dict
                        if file_id not in self.temporary_pc_files:
                            self.temporary_pc_files[
                                file_id
                            ] = self.generate_temp_file_path()

                        param = "%s=%s" % (
                            key,
                            self.temporary_pc_files[file_id],
                        )
                    else:
                        param = "%s=%s" % (key, search_string)
                    parameters.append(param)
                    # List the resource for potential export
                    if "export" in module_descr["outputs"][key]:
                        exp = module_descr["outputs"][key]["export"]
                        if "format" not in exp or "type" not in exp:
                            raise AsyncProcessError(
                                "Invalid export parameter in description of "
                                "module <%s>" % module_name
                            )
                        if exp["format"] not in SUPPORTED_EXPORT_FORMATS:
                            raise AsyncProcessError(
                                "Invalid export <format> parameter in "
                                "description of module <%s>" % module_name
                            )
                        if exp["type"] not in [
                            "raster",
                            "vector",
                            "strds",
                            "file",
                            "stvds",
                        ]:
                            raise AsyncProcessError(
                                "Invalid export <type> parameter in "
                                "description of module <%s>" % module_name
                            )
                        self.resource_export_list.append(
                            module_descr["outputs"][key]
                        )

        if "flags" in module_descr:
            if "flags" in module_descr:
                parameters.append("-" + str(module_descr["flags"]))

        if "overwrite" in module_descr and module_descr["overwrite"] is True:
            parameters.append("--o")

        if "superquiet" in module_descr and module_descr["superquiet"] is True:
            parameters.append("--qq")

        if "verbose" in module_descr and module_descr["verbose"] is True:
            parameters.append("--v")

        if (
            "interface-description" in module_descr
            and module_descr["interface-description"] is True
        ):
            parameters.append("--interface-description")

        # Check for un-allowed characters in the parameter list
        for entry in parameters:
            if "&" in entry:
                raise AsyncProcessError(
                    "Character '&' not supported in process "
                    "description for %s" % module_name
                )

        p = Process(
            exec_type="grass",
            executable=module_name,
            executable_params=parameters,
            stdin_source=stdin_func,
            id=id,
        )

        self.process_dict[id] = p

        return p

    # TODO: remove legacy methods and do no use them in actinia_core
    def _create_exec_process_legacy(self, id, module_descr):
        """Analyse a grass process description dict and create a Process
        that is used to execute a common Linux binary.

        Args:
            id (str): The id of this process in the process chain
            module_descr (dict): The module description

        Returns: Process:
            An object of type Process that contains the module name,
            the parameter list and stdout/stdin definitions

        """
        self.message_logger.info(str(module_descr))
        parameters = []

        stdin_func = None
        if "stdin" in module_descr:
            if "::" not in module_descr["stdin"]:
                raise AsyncProcessError(
                    "The stdin option in id %s misses the ::" % str(id)
                )

            object_id, method = module_descr["stdin"].split("::")

            if "stdout" in method:
                stdin_func = self.process_dict[object_id].get_stdout
            elif "stderr" in method:
                stdin_func = self.process_dict[object_id].get_stderr
            else:
                raise AsyncProcessError(
                    "The stdout or stderr flag in id %s is missing" % str(id)
                )

        if "executable" not in module_descr:
            raise AsyncProcessError(
                "Missing executable name in module description of id %s"
                % str(id)
            )

        executable = module_descr["executable"]
        if "parameters" in module_descr:
            for search_string in module_descr["parameters"]:
                # Search for file identifiers and generate the temporary file
                # path
                if "$file" in search_string and "::" in search_string:
                    file_id = search_string.split("::")[1]
                    # Generate the temporary file path and store it in the dict
                    if file_id not in self.temporary_pc_files:
                        self.temporary_pc_files[
                            file_id
                        ] = self.generate_temp_file_path()

                    param = "%s" % self.temporary_pc_files[file_id]
                else:
                    param = "%s" % search_string

                parameters.append(param)

        # Check for un-allowed characters in the parameter list
        for entry in parameters:
            if "&" in entry:
                raise AsyncProcessError(
                    "Character '&' not supported in process "
                    "description for %s" % executable
                )

        p = Process(
            exec_type="exec",
            executable=executable,
            executable_params=parameters,
            stdin_source=stdin_func,
            id=id,
        )

        self.process_dict[id] = p

        return p

    def _add_grass_module_input_parameter_to_list(
        self, module_descr, params, id
    ):
        """Helper method to set the input parameters of a grass module and add
        them to the params list.

        Args:
            module_descr (dict): The module description
            params (list): The list of the grass module inputs parameters with
                           param=value entries (here the input parameter are
                           added)
            id (str): The id of this process in the process chain
        """
        if isinstance(module_descr["inputs"], list) is False:
            raise AsyncProcessError(
                "Inputs in the process chain definition "
                "must be of type list"
            )

        for input in module_descr["inputs"]:
            # Add import description to the import list
            if "import_descr" in input:
                self.import_descr_list.append(input)
            if "value" not in input:
                raise AsyncProcessError(
                    "<value> is missing in input description of process id: %s"
                    % id
                )
            if "param" not in input:
                raise AsyncProcessError(
                    "<param> is missing in input description of process id: %s"
                    % id
                )

            value = input["value"]
            param = input["param"]
            # Search for file identifiers and generate the temporary file path
            if "$file" in value and "::" in value:
                file_id = value.split("::")[1]
                # Generate the temporary file path and store it in the dict
                if file_id not in self.temporary_pc_files:
                    self.temporary_pc_files[
                        file_id
                    ] = self.generate_temp_file_path()
                param = "%s=%s" % (param, self.temporary_pc_files[file_id])
            else:
                param = "%s=%s" % (param, value)
                # Check for mapset in input name and append it,
                # to the list of required mapsets
                if "@" in str(value):
                    # Mapset names are after an @ symbol
                    # Mapsets in expressions can be detected by replacing the
                    # symbols like *, +, :, /, {, (,},], ... by spaces and
                    # split the string by spaces, searching in each substring
                    # for @
                    symbols = [
                        "*",
                        "+",
                        "-",
                        "/",
                        "%",
                        "$",
                        "!",
                        ":",
                        "(",
                        ")",
                        "{",
                        "}",
                        "&",
                        "?",
                        "#",
                        "=",
                        "^",
                        "~",
                        "<",
                        ">",
                        "\\",
                    ]
                    for symbol in symbols:
                        value = value.replace(symbol, " ")
                    values = value.split(" ")
                    for entry in values:
                        for subentry in entry.split(","):
                            if "@" in subentry:
                                mapset = subentry.split("@")[1]
                                if mapset not in self.required_mapsets:
                                    self.required_mapsets.append(mapset)
            params.append(param)

    def _add_grass_module_output_parameter_to_list(
        self, module_descr, params, id, module_name
    ):
        """Helper method to set the output parameters of a grass module and add
        them to the params list. If export is in the output parameter the
        output is appended to the resource_export_list.

        Args:
            module_descr (dict): The module description
            params (list): The list of the grass module parameters with
                           param=value entries (here the output parameter are
                           added)
            id (str): The id of this process in the process chain
            module_name (str): The name of the grass module

        """
        for output in module_descr["outputs"]:
            if "value" not in output:
                raise AsyncProcessError(
                    "<value> is missing in input description of process id: %s"
                    % id
                )
            if "param" not in output:
                raise AsyncProcessError(
                    " <param> is missing in input description of process "
                    "id: %s" % id
                )

            value = output["value"]
            param = output["param"]
            # Check the resource for potential export
            if "export" in output:
                exp = output["export"]
                if "format" not in exp or "type" not in exp:
                    raise AsyncProcessError(
                        "Invalid export parameter in description of module "
                        "<%s>" % module_name
                    )
                if exp["format"] not in SUPPORTED_EXPORT_FORMATS:
                    raise AsyncProcessError(
                        "Invalid export <format> parameter in description of "
                        "module <%s>" % module_name
                    )
                if "PostgreSQL" in exp["format"] and "dbstring" not in exp:
                    raise AsyncProcessError(
                        "The dbstring parameter is missing for PostgreSQL "
                        "export"
                    )
                if exp["type"] not in [
                    "raster",
                    "vector",
                    "strds",
                    "file",
                    "stvds",
                ]:
                    raise AsyncProcessError(
                        "Invalid export <type> parameter in description of "
                        "module <%s>" % module_name
                    )
                if (
                    "file" in exp["type"]
                    and ("$file" in value and "::" in value) is False
                ):
                    raise AsyncProcessError(
                        "The value filed must contain a file identifier "
                        "($file::unique_id) to export a "
                        "file generated from module <%s> as resource."
                        % module_name
                    )
            # Search for file identifiers and generate the temporary file path
            if "$file" in value and "::" in value:
                file_id = value.split("::")[1]
                # Generate the temporary file path and store it in the dict
                if file_id not in self.temporary_pc_files:
                    self.temporary_pc_files[
                        file_id
                    ] = self.generate_temp_file_path()
                # Store the file path in the output description for export
                param = "%s=%s" % (param, self.temporary_pc_files[file_id])
                # Add the temp file path and the new file name with suffix to ,
                # the output dict
                if "export" in output:
                    output["tmp_file"] = self.temporary_pc_files[file_id]
                    output["file_name"] = "%s.%s" % (
                        file_id,
                        output["export"]["format"].lower(),
                    )
            else:
                param = "%s=%s" % (param, value)
            params.append(param)
            # save the output dict in a resource export list
            if "export" in output:
                self.resource_export_list.append(output)

    def _check_if_webhook_exists(self, webhook_url, process_chain, type):
        """
        Helper method to check if the URL of a webhook exists by investigating
        the HTTP header

        Args:
            webhook_url (str): The webhook URL
            process_chain (dict): The process chain
            type (str): The webhook type: 'finished' or 'update'

        Raises:
            This method will raise an AsyncProcessError if the request to the
            webhook url header fails.
        """
        if "auth" in process_chain["webhooks"]:
            self.webhook_auth = process_chain["webhooks"]["auth"]
            # username is expected to be without colon (':')
            resp = requests.head(
                webhook_url,
                auth=HTTPBasicAuth(
                    self.webhook_auth.split(":")[0],
                    ":".join(self.webhook_auth.split(":")[1:]),
                ),
            )
        else:
            resp = requests.head(webhook_url)
        if resp.status_code != 200:
            raise AsyncProcessError(
                "The %s webhook URL %s can not be accessed."
                % (type, webhook_url)
            )

    def _check_and_get_stdout_command(self, module_descr, id):
        """
        Helper method to check the stdout in the module description and add it
        to the output_parser_list.

        Args:
            module_descr (dict): The module description
            id (str): The id of this process in the process chain
        """
        if "id" not in module_descr["stdout"]:
            raise AsyncProcessError(
                "Missing unique *id* in stdout parser description of process "
                "id %s" % str(id)
            )
        if "format" not in module_descr["stdout"]:
            raise AsyncProcessError(
                "Missing *format* in stdout parser description of process "
                "id %s" % str(id)
            )
        if "delimiter" not in module_descr["stdout"]:
            raise AsyncProcessError(
                "Missing *delimiter* in stdout parser description of process "
                "id %s" % str(id)
            )
        self.output_parser_list.append({id: module_descr["stdout"]})

    def _process_chain_to_process_list_legacy(self, process_chain):
        """
        Transform a legacy module chain description into an ordered list of
        process runs

        All input map layer MUST be specified with the mapset they belong to:
        map@mapset

        Args:
            process_chain (dict): The process chain

        Return:
             list
             A list of ordered grass processes

        Process chain input format::

            {                                           # A process chain is a
                                                          dict with entries for
                                                          each module that
                                                          should be run
               Id:{                                     # Id must be numerical
                                                          and indicates the
                                                          process order
                    "module":<module_name>,             # Name of the module to
                                                          run
                    "stdin":<Id::stdout | Id::stderr>   # Use the output of a
                                                          specific module as
                                                          input for this module
                                                        # Id:stdout, Id:stderr
                                                          are available
                    "inputs":{                          # Definition of all
                                                          input parameters as
                                                          key:value pairs
                             <parameter name>:<value>,  # e.g.: value ==
                                                          "raster_name@mapset_
                                                          name" or "degree"
                                                          or 0.0
                             <parameter name>:<value>   # e.g.: value ==
                                                          $file::slope_output_
                                                          file to specify an
                                                          output file
                                                        # that name will be
                                                          automatically
                                                          generated by the API.
                    },
                    "outputs":{                         # Definition of all
                                                          outputs using
                                                          key:value pairs
                        <parameter name>:{
                            "name":<value>,             # Output name e.g.
                                                          "my_aspect_map" or a
                                                          temporary file id
                                                        # definition: $file::id
                                                          eg: $file::aspect_
                                                          output_file
                                                        # This file can be used
                                                          in other module as
                                                          input
                            "export":{                  # Export options, if
                                                          present this map will
                                                          be exported
                                "format":<value>        # Set the export format
                                                          raster=GeoTiff
                                                          (default), vector =
                                                          shape (default)
                                "type":<output type>,   # Output type e.g.:
                                                          raster, vector, file,
                                                          stds
                            }
                        },
                        <parameter name>:{
                            "name":<value>,             # Output name e.g.
                                                          "my_slope_map"
                            "export":{                  # Export options, if
                                                          present this map will
                                                          be exported
                                "format":<value>        # Set the export format
                                                          raster=GeoTiff
                                                          (default), vector =
                                                          shape (default)
                                "type":<output type>,   # Output type e.g.:
                                                          raster, vector, file,
                                                          stds
                            }
                        }
                    },
                    "flags":<flags>,                    # All flags in a string
                                                          e.g.: "ge"
                    "overwrite":<True|False>,           # Set True to overwrite
                                                          existing data
                    "verbose":<True|False>              # Verbosity of the
                                                          module
                },
               Id:{                                     # The id of an
                                                          executable command,
                                                          that is not a grass
                                                          module
                    "executable":<path>,                # The name and path of
                                                          the executable e.g.
                                                          /bin/cp
                    "stdin":<Id::stdout | Id::stderr>   # Use the output of a
                                                          specific module as
                                                          input for this module
                                                        # Id::stdout, Id::
                                                          stderr are available
                    "parameters":[<parameter>,]         # A list of strings
                                                          that represent the
                                                          parameters that may
                                                          contain
                                                        # temporary file
                                                          definitions: $file::
                                                          id  eg: $file::aspect
                                                          _output_file
               },
                ...
            }

        """
        process_list = []

        keys = process_chain.keys()
        int_keys = []
        # Convert the keys to integer to sort correctly

        for k in keys:
            int_keys.append(int(k))

        int_keys = sorted(int_keys)
        # Re-order the process chain by integer sorting index
        for index in int_keys:
            program = process_chain[str(index)]

            if "module" in program:
                process_list.append(
                    self._create_module_process_legacy(str(index), program)
                )
            elif "executable" in program:
                process_list.append(
                    self._create_exec_process_legacy(str(index), program)
                )
            elif "evaluate" in program:
                process_list.append(("python", program["evaluate"]))

        return process_list


def check_required_keys_for_download_process_chain(entry):
    """Helper function to check if the required keys to create the download
    process chain are set.

    Args:
        entry (dict): Entry of the import description list

    Raises:
        If not all required keys are set an AsyncProcessError is raised
    """
    if "import_descr" not in entry:
        raise AsyncProcessError(
            "import_descr specification is required in import definition"
        )
    if "type" not in entry["import_descr"]:
        raise AsyncProcessError(
            "Type specification is required in import definition"
        )
    if "source" not in entry["import_descr"]:
        raise AsyncProcessError(
            "Source specification is required in import definition"
        )
    if entry["import_descr"]["type"] not in [
        "raster",
        "vector",
        "sentinel2",
        "landsat",
        "file",
        "postgis",
        "stac",
    ]:
        raise AsyncProcessError(
            "Unknown type specification: %s" % entry["import_descr"]["type"]
        )


def test_process_chain():
    from pprint import pprint

    elev_in = InputParameter(
        param="elevation",
        value="elev_10m",
        import_descr={
            "source": "https://storage.googleapis.com/graas-geodata/"
            "elev_ned_30m.tif",
            "type": "raster",
        },
    )
    format_in = InputParameter(param="format", value="degree")
    precision_in = InputParameter(param="precision", value="DCELL")
    export = {"type": "raster", "format": "GTiff"}

    slope_out = OutputParameter(
        param="slope", value="elev_10m_slope", export=export
    )

    aspect_out = OutputParameter(
        param="aspect", value="elev_10m_aspect", export=export
    )

    module_1 = GrassModule(
        id="r_slope_aspect_1",
        module="r.slope.aspect",
        inputs=[elev_in, format_in, precision_in],
        outputs=[slope_out, aspect_out],
        flags="a",
        overwrite=True,
        verbose=False,
    )

    file_in = InputParameter(
        param="name",
        value="$file::polygon",
        import_descr={
            "source": "https://storage.googleapis.com/graas-geodata/"
            "brazil_polygon.json",
            "type": "file",
        },
    )

    module_2 = GrassModule(id="importer", module="importer", inputs=[file_in])

    exe_1 = Executable(
        id="cat_1", exe="/bin/cat", params=[], stdin="r_slope_aspect_1::stderr"
    )

    func_in_1 = InputParameter(param="pyfile", value="$file::polygon")

    module_3 = GrassModule(
        id="udf", module="t.rast.aggr_func", inputs=[func_in_1]
    )

    func_in_2 = InputParameter(
        param="pyfile",
        value="$file::polygon",
        import_descr={
            "source": "https://storage.googleapis.com/graas-geodata/"
            "brazil_polygon.json",
            "type": "file",
        },
    )

    module_4 = GrassModule(
        id="udf", module="t.rast.aggr_func", inputs=[func_in_2]
    )

    landsat_import = InputParameter(
        param="map",
        value="ignored",
        import_descr={
            "source": "LT52170762005240COA00",
            "type": "landsat",
            "landsat_atcor": "dos1",
        },
    )

    stdout_parser = StdoutParser(id="stats", format="kv", delimiter="=")

    module_5 = GrassModule(
        id="r_univar_1",
        module="r.univar",
        inputs=[landsat_import],
        stdout=stdout_parser,
    )

    exporter_output = OutputParameter(
        param="map",
        value="LT52170762005240COA00_dos1.1",
        export={"format": "GTiff", "type": "raster"},
    )

    module_export = GrassModule(
        id="exporter_1", module="exporter", outputs=[exporter_output]
    )

    pc = ProcessChainModel(
        version="1",
        list=[
            module_1,
            module_2,
            exe_1,
            module_3,
            module_4,
            module_5,
            module_export,
        ],
    )

    pprint(pc)

    pconv = ProcessChainConverter()
    pl = pconv.process_chain_to_process_list(pc)

    for proc in pl:
        pprint(str(proc))

    pprint(pconv.process_dict)
    pprint(pconv.required_mapsets)
    pprint(pconv.resource_export_list)
    pprint(pconv.import_descr_list)
    pprint(pconv.output_parser_list)
    pprint(pconv.temporary_pc_files)


if __name__ == "__main__":
    test_process_chain()
