# Copyright (c) 2020 Adam Souzis
# SPDX-License-Identifier: MIT
"""
A Job is generated by comparing a list of specs with the last known state of the system.
Job runs tasks, each of which has a configuration spec that is executed on the running system
Each task tracks and records its modifications to the system's state
"""

import collections
from datetime import datetime
import types
import itertools
import os
import json
from typing import Any, Dict, Iterable, List, Mapping, Optional, Tuple, Union, cast, TYPE_CHECKING

from .support import Status, Priority, Defaults, AttributeManager, Reason, NodeState
from .result import ResourceRef, serialize_value, ChangeRecord
from .util import UnfurlError, UnfurlTaskError, to_enum, change_cwd
from .merge import merge_dicts
from .runtime import NodeInstance, Operational, OperationalInstance
from . import logs
from .configurator import (
    TaskView,
    ConfiguratorResult,
)
from .projectpaths import Folders, rmtree
from .planrequests import (
    PlanRequest,
    TaskRequest,
    TaskRequestGroup,
    SetStateRequest,
    JobRequest,
    do_render_requests,
    get_render_requests,
    set_fulfilled,
    create_instance_from_spec,
)
from .plan import Plan, get_success_status
from .localenv import LocalEnv

from . import configurators  # need to import configurators even though it is unused
from . import display

from time import perf_counter
import logging

if TYPE_CHECKING:
    from unfurl.yamlmanifest import YamlManifest


logger = cast(logs.UnfurlLogger, logging.getLogger("unfurl"))

class ConfigChange(OperationalInstance, ChangeRecord):
    """
    Represents a configuration change made to the system.
    It has a operating status and a list of dependencies that contribute to its status.
    There are two kinds of dependencies:

    1. Live resource attributes that the configuration's inputs depend on.
    2. Other configurations and resources it relies on to function properly.
    """

    def __init__(
        self,
        parentJob: Optional["Job"]=None,
        startTime: Optional[datetime]=None,
        status: Optional[Union[OperationalInstance, int, str]]=None,
        previousId: Optional[str]=None,
        **kw: Any
    ) -> None:
        OperationalInstance.__init__(self, status, **kw)
        if parentJob:  # use the parent's job id and startTime
            ChangeRecord.__init__(self, parentJob.changeId, parentJob.startTime)
        else:  # generate a new job id and use the given startTime
            ChangeRecord.__init__(self, startTime=startTime, previousId=previousId)

    def get_operational_dependencies(self) -> Iterable[Operational]:
        for d in self.dependencies:
            if d.target != self.target:  # type: ignore
                yield d


class JobOptions:
    """
    Options available to select which tasks are run, e.g. read-only

    does the config apply to the operation?
    is it out of date?
    is it in a ok state?
    """

    global_defaults = dict(
        # global options that we also want to apply to any child or external job
        verbose=0,
        masterJob=None,
        startTime=None,
        starttime=None,
        out=None,
        dryrun=False,
        planOnly=False,
        readonly=False,  # only run configurations that won't alter the system
        requiredOnly=False,
        commit=False,
        dirty="auto",  # run the job even if the repository has uncommitted changrs
        message=None,
    )

    defaults = dict(
        global_defaults,
        parentJob=None,
        instance=None,
        instances=None,
        template=None,
        # default options:
        add=True,  # add new templates
        skip_new=False,  # don't create newly defined instances (override add)
        update=True,  # run configurations that that have changed
        change_detection="evaluate",  # skip, spec, evaluate (skip sets update to False)
        repair="error",  # or 'degraded' or "none", run configurations that are not operational and/or degraded
        upgrade=False,  # run configurations with major version changes or whose spec has changed
        force=False,  # (re)run operation regardless of instance's status or state
        verify=False,  # XXX3 discover first and set status if it differs from expected state
        check=False,  # if new instances exist before deploying
        prune=False,
        destroyunmanaged=False,
        append=None,
        replace=None,
        workflow=Defaults.workflow,
    )

    def __init__(self, **kw: Any) -> None:
        options = self.defaults.copy()
        if kw.get("starttime"):  # rename
            options["startTime"] = kw["starttime"]
        if kw.get("skip_new"):
            options["add"] = False
        if kw.get("change_detection") == "skip":
            options["update"] = False
        options.update(kw)
        self.__dict__.update(options)
        self.userConfig = kw

    def copy(self, **kw: Any) -> "JobOptions":
        # only copy global
        _copy = {
            k: self.userConfig[k]
            for k in set(self.userConfig) & set(self.global_defaults)
        }
        return JobOptions(**dict(_copy, **kw))

    def get_user_settings(self) -> dict:
        # only include settings different from the defaults
        return {
            k: self.userConfig[k]
            for k in set(self.userConfig) & set(self.defaults)
            if k not in ["out", "masterJob"] and self.userConfig[k] != self.defaults[k]
        }


class ConfigTask(ConfigChange, TaskView):
    """
    receives a configSpec and a target node instance
    instantiates and runs Configurator
    updates Configurator's target's status and lastConfigChange
    """

    def __init__(self, job, configSpec, target, reason=None):
        ConfigChange.__init__(self, job)
        TaskView.__init__(self, job.manifest, configSpec, target, reason)
        self.dry_run = job.dry_run
        self.verbose = job.verbose
        self._configurator = None
        self.generator = None
        self.job = job
        self.changeList = []
        self.result = None
        self.outputs = None
        # for summary:
        self.modified_target = False
        self.target_status = target.status
        self.target_state = target.state

        # set the attribute manager on the root resource
        self._attributeManager = AttributeManager(self._manifest.yaml, self)
        self.target.root.attributeManager = self._attributeManager
        self._resolved_inputs = {}

    def _status(self, seen):
        return self.local_status

    def priority():
        doc = "The priority property."

        def fget(self):
            if self._priority is None:
                return self.configSpec.should_run()
            else:
                return self._priority

        def fset(self, value):
            self._priority = value

        def fdel(self):
            del self._priority

        return locals()

    priority = property(**priority())

    @property
    def configurator(self):
        if self._configurator is None:
            self._configurator = self.configSpec.create()
        return self._configurator

    def start_run(self):
        self.generator = self.configurator.get_generator(self)
        assert isinstance(self.generator, types.GeneratorType)

    def send(self, change):
        result = None
        try:
            result = self.generator.send(change)
        finally:
            # serialize configuration changes
            self.commit_changes()
        return result

    def start(self):
        self.start_run()
        self.target.root.attributeManager = self._attributeManager
        self.target_status = self.target.status
        self.target_state = self.target.state

    def _update_status(self, result):
        """
        Update the instances status with the result of the operation.
        If status wasn't explicitly set but the operation changed the instance's configuration
        or state, choose a status based on the type of operation.
        """
        if result.status is not None:
            # status was explicitly set
            self.target.local_status = result.status
            if self.target.present and self.target.created is None:
                if self.configSpec.operation not in [
                    "check",
                    "discover",
                ]:
                    self.target.created = self.changeId
            self.logger.trace(
                "status was explicitly set for %s with local_status %s",
                self.target.name,
                self.target.local_status,
            )

            return True
        elif not result.success:
            # if any task failed and (maybe) modified, target.status will be set to error or unknown
            if result.modified:
                self.target.local_status = (
                    Status.error if self.required else Status.degraded
                )
                return True
            elif result.modified is None:
                self.target.local_status = Status.unknown
                return True
            # otherwise doesn't modify target status
        return False

    def _update_last_change(self, result):
        """
        If the target's configuration or state has changed, set the instance's lastChange
        state to this tasks' changeid.
        """
        if self.target.last_change is None and self.target.status != Status.pending:
            # hacky but always save _lastConfigChange the first time to
            # distinguish this from a brand new resource
            self.target._lastConfigChange = self.changeId
            return True
        if result.modified or self._resourceChanges.get_attribute_changes(
            self.target.key
        ):
            if self.target.last_change != self.changeId:
                # save to create a linked list of tasks that modified the target
                self.previousId = self.target.last_change
            self.target._lastConfigChange = self.changeId
            return True
        return False

    def _finished_workflow(self, successStatus, workflow):
        instance = self.target
        self.modified_target = True
        instance.local_status = successStatus
        self.target_status = successStatus
        if instance.last_change != self.changeId:
            # save to create a linked list of tasks that modified the target
            self.previousId = instance.last_change
        instance._lastConfigChange = self.changeId
        if (
            workflow == "deploy"
            and successStatus == Status.ok
            and instance.created is None
        ):
            instance.created = self.changeId

    def finished(self, result):
        assert result
        if self.generator:
            self.generator.close()
            self.generator = None

        self.outputs = result.outputs

        # don't set the changeId until we're finish so that we have a higher changeid
        # than nested tasks and jobs that ran
        # (task that never run will have the same changeId as its parent)
        self.set_task_id(self.job.increment_task_count())
        # XXX2 if attributes changed validate using attributesSchema
        # XXX2 Check that configuration provided the metadata that it declared (check postCondition)

        if self.changeList:
            # merge changes together (will be saved with changeset)
            changes = self.changeList
            accum = changes.pop(0)
            while changes:
                accum = merge_dicts(accum, changes.pop(0))

            # note: this might set _lastConfigChange on instances other than this target
            self._resourceChanges.update_changes(
                accum, self._attributeManager.statuses, self.target, self.changeId
            )
            # XXX implement:
            # if not result.applied:
            #    self._resourceChanges.rollback(self.target)

        # now that resourceChanges finalized:
        self._update_status(result)
        targetChanged = self._update_last_change(result)
        self.result = result
        self.local_status = Status.ok if result.success else Status.error
        self.modified_target = targetChanged or self.target_status != self.target.status
        self.target_status = self.target.status
        self.target_state = self.target.state
        return self

    def commit_changes(self):
        """
        This can be called multiple times if the configurator yields multiple times.
        Save the changes made each time.
        """
        changes, dependencies = self._attributeManager.commit_changes()
        self.changeList.append(changes)

        if self._inputs is not None:
            self._resolved_inputs.update(self.inputs.get_resolved())
        # need to recreate this because _attributeManager was reset
        self._inputs = None

        # record the live attributes that we are dependent on
        # note: task start fresh with no dependencies so don't need to worry update or removing previous ones
        for key, (target, attributes) in dependencies.items():
            if key.startswith("::.requirements"):
                # hackish: exclude default connections (which are represented as root relationships)
                continue
            for name, (live, value) in attributes.items():
                # hackish: we only want the status of a dependency to reflect its target instance's status
                # when the attribute is live (as opposed to a static property set in the spec)
                # so don't set its target unless the attribute is live
                self.add_dependency(
                    key + "::" + name, value, target=target if live else None
                )
        return changes, dependencies

    # unused
    # def findLastConfigureOperation(self):
    #     if not self._manifest.changeSets:
    #         return None
    #     previousId = self.target.lastChange
    #     while previousId:
    #         previousChange = self._manifest.changeSets.get(previousId)
    #         if not previousChange:
    #             return None
    #         if previousChange.target != self.target.key:
    #             return (
    #                 None  # XXX handle case where lastChange was set by another target
    #             )
    #         if previousChange.operation == self.configSpec.operation:
    #             return previousChange
    #         previousId = previousChange.previousId
    #     return None

    def has_inputs_changed(self):
        """
        Evaluate configuration spec's inputs and compare with the current inputs' values
        """
        changeset = self._manifest.find_last_operation(
            self.target.key, self.configSpec.operation
        )
        if not changeset:
            self.logger.debug(
                'Can\'t check for changes: could not find previous "%s" operation for "%s"',
                self.target.key,
                self.configSpec.operation,
            )
            return False

        return self.configurator.check_digest(self, changeset)

    def has_dependencies_changed(self):
        return False
        # XXX artifacts
        # XXX identity of requirements (how? what about imported nodes? instance keys?)
        # dynamic dependencies:
        # return any(d.hasChanged(self) for d in self.dependencies)

    # XXX unused
    # def refreshDependencies(self):
    #     for d in self.dependencies:
    #         d.refresh(self)

    def find_missing_dependencies(self):
        missing = []
        reason = ""
        # XXX this logic should be replaced with entry_state on check and delete operations
        if self.configSpec.operation not in ["check", "delete"] and (
            self.reason not in [Reason.force, Reason.run]
        ):
            # check if the live attributes this task depends on has been set
            missing, reason = _dependency_check(self)
            # don't check if this operation does depend on the operational dependencies being live
            if not missing and self.configSpec.entry_state > NodeState.initial:
                missing, reason =_dependency_check(self.target)
        return missing, reason

    @property
    def name(self):
        name = self.configSpec.name
        if self.configSpec.operation and self.configSpec.operation not in name:
            name = name + ":" + self.configSpec.operation
        if self.reason and self.reason not in name:
            return name + ":" + self.reason
        return name

    def summary(self, asJson=False):
        if self.target.name != self.target.template.name:
            rname = f"{self.target.name} ({self.target.template.name})"
        else:
            rname = self.target.name

        if self.configSpec.name != self.configSpec.className:
            cname = f"{self.configSpec.name} ({self.configSpec.className})"
        else:
            cname = self.configSpec.name

        if self._configurator:
            cClass = self._configurator.__class__
            configurator = f"{cClass.__module__}.{cClass.__name__}"
        else:
            configurator = self.configSpec.className

        summary = dict(
            status=self.status.name,
            target=self.target.name,
            operation=self.configSpec.operation,
            template=self.target.template.name,
            type=self.target.template.type,
            targetStatus=self.target_status.name,
            targetState=self.target_state and self.target_state.name or None,
            changed=self.modified_target,
            configurator=configurator,
            priority=self.priority.name,
            reason=self.reason or "",
        )

        if asJson:
            return summary
        else:
            return (
                "{operation} on instance {rname} (type {type}, status {targetStatus}) "
                + "using configurator {cname}, priority: {priority}, reason: {reason}"
            ).format(cname=cname, rname=rname, **summary)

    def __repr__(self):
        return f"ConfigTask({self.target}:{self.name})"


def _dependency_check(instance):
    dependencies = list(instance.get_operational_dependencies())
    missing = [dep for dep in dependencies if not dep.operational and dep.required]
    if missing:
        reason = "required dependencies not operational: %s" % ", ".join(
            [f"{dep.name} is {dep.status.name}" for dep in missing]
        )
    else:
        reason = ""
    return missing, reason


class Job(ConfigChange):
    """
    runs ConfigTasks and child Jobs
    """

    MAX_NESTED_SUBTASKS = 100

    def __init__(
        self,
        manifest: "YamlManifest",
        rootResource: NodeInstance,
        jobOptions: JobOptions,
        previousId: Optional[str]=None
    ) -> None:
        assert isinstance(jobOptions, JobOptions)
        self.__dict__.update(jobOptions.__dict__)
        super().__init__(self.parentJob, self.startTime, Status.ok, previousId)  # type: ignore
        self.dry_run = jobOptions.dryrun  # type: ignore
        self.jobOptions = jobOptions
        self.manifest = manifest
        self.rootResource = rootResource
        self.jobRequestQueue: List[JobRequest] = []
        self.unexpectedAbort: Optional[UnfurlError] = None
        self.workDone: collections.OrderedDict = collections.OrderedDict()
        self.timeElapsed: float = 0
        self.plan_requests: Optional[List[TaskRequestGroup]] = None
        self.task_count = 0
        self.external_requests: Optional[List[Tuple[Any, List[JobRequest]]]] = None
        self.external_jobs: Optional[List["Job"]] = None

    def get_operational_dependencies(self) -> Iterable[ConfigTask]:
        # XXX3 this isn't right, root job might have too many and child job might not have enough
        # plus dynamic configurations probably shouldn't be included if yielded by a configurator
        for task in self.workDone.values():
            yield task

    def get_outputs(self) -> Any:
        return self.rootResource.attributes["outputs"]  # type: ignore

    def is_filtered(self) -> bool:
        return self.instance or self.instances or self.template  # type: ignore

    def run_query(self, query: Union[str, Mapping], trace: int=0) -> list:
        from .eval import eval_for_func, RefContext

        return eval_for_func(query, RefContext(self.rootResource, trace=trace))

    def create_task(self, configSpec: ConfiguratorResult, target: NodeInstance, reason: Optional[str]=None) -> ConfigTask:
        task = ConfigTask(self, configSpec, target, reason=reason)
        try:
            task.inputs
            task.configurator
        except Exception:
            UnfurlTaskError(task, "unable to create task")
        return task

    def validate_job_options(self) -> None:
        if self.jobOptions.instance and not self.rootResource.find_resource(  # type: ignore  # mypy doesn't like the way JobOptions is defined
            self.jobOptions.instance  # type: ignore
        ):
            logger.warning(
                'selected instance not found: "%s"', self.jobOptions.instance  # type: ignore
            )

    def render(self) -> Tuple[list, list, list]:
        if self.plan_requests is None:
            ready = self.create_plan()
        else:
            ready = self.plan_requests[:]

        # run artifact job requests before render
        if self.external_requests:
            msg = "Running local installation tasks"
            plan, count = self._plan_summary([], self.external_requests)
            logger.info(msg + "\n%s", plan)

        # currently external jobs are just for installing artifacts
        # we want to run these even if we just generating a plan
        self.external_jobs = self.run_external(planOnly=False)
        if self.external_jobs and self.external_jobs[-1].status == Status.error:
            return [], [], ["error running job on external ensemble"]

        ready, notReady, errors = do_render_requests(self, ready)
        return ready, notReady, errors

    def _run(self, rendered_requests: Optional[Tuple[list, list, list]]=None) -> ResourceRef:
        if rendered_requests:
            ready, notReady, errors = rendered_requests
        else:
            ready, notReady, errors = self.render()

        self.workDone = collections.OrderedDict()

        if errors:
            logger.error("Aborting job: there were errors during rendering: %s", errors)
            return self.rootResource

        # XXX update_plan(ready, unfulfilled) # try to reorder so we can add to ready
        while ready:
            # XXX need to call self.run_external() here if update_plan() adds external job
            # create and run tasks for requests that have their dependencies fulfilled
            self.apply(ready)
            # remove requests from notReady if they've had all their dependencies fulfilled
            ready, notReady = set_fulfilled(notReady, ready)
            logger.trace("ready %s; not ready %s", ready, notReady)
            # the first time we render them all, after that only re-render requests if their dependencies were fulfilled
            ready, unfulfilled, errors = do_render_requests(self, ready)
            # XXX update_plan(ready, unfulfilled) # try to reorder so we can add to ready
            notReady.extend(unfulfilled)

        # if there were circular dependencies or errors then notReady won't be empty
        if notReady:
            for parent, req in get_render_requests(notReady):
                if self.workflow == "deploy" and not req.target.template.required:  # type: ignore # we don't want to run these
                    continue
                message = f"can't fulfill {req.target.name}: never ran {req.future_dependencies}"
                logger.info(message)
                req.task.finished(ConfiguratorResult(False, False, result=message))

        # the jobRequestQueue will have jobs that were added dynamically by a configurator
        # but were not yielding inside runTask
        while self.jobRequestQueue:
            jobRequest = self.jobRequestQueue[0]
            self.run_job_request(jobRequest)

        # force outputs to be evaluated now and commit the changes
        # so any attributes that were evaluated computing the outputs are saved in the manifest.
        outputs = serialize_value(self.get_outputs())
        if outputs:
            logger.info("Job outputs: %s", outputs)
        self.rootResource.attributeManager.commit_changes()  # type: ignore
        return self.rootResource

    def run(self, rendered: Tuple[list, list, list]) -> None:
        manifest = self.manifest
        startTime = perf_counter()
        jobOptions = self.jobOptions
        with change_cwd(manifest.get_base_dir()):
            try:
                ready, notReady, errors = rendered
                if not jobOptions.out:  # type: ignore
                    # out is used by unit tests to avoid writing to disk
                    manifest.lock()
                if jobOptions.dirty == "auto":  # type: ignore  # default to false if committing
                    checkIfClean = jobOptions.commit  # type: ignore
                else:
                    checkIfClean = jobOptions.dirty == "abort"  # type: ignore
                if checkIfClean:
                    for repo in manifest.repositories.values():
                        if repo.is_dirty():
                            logger.error(
                                "aborting run: uncommitted files in %s (--dirty=ok to override)",
                                repo.working_dir,
                            )
                            return None
                try:
                    display.verbosity = jobOptions.verbose  # type: ignore
                    self._run((ready, notReady, errors))
                except Exception:
                    self.local_status = Status.error  # type: ignore
                    self.unexpectedAbort = UnfurlError(
                        "unexpected exception while running job", True, True
                    )

                self._apply_workfolders()
                manifest.commit_job(self)
            finally:
                self.timeElapsed = perf_counter() - startTime
                manifest.unlock()

    def _apply_workfolders(self) -> None:
        for task in self.workDone.values():
            if task.status == Status.ok:
                task.apply_work_folders()

    def _update_joboption_instances(self) -> None:
        if not self.jobOptions.instances:  # type: ignore
            return
        # process any instances that are a full resource spec
        self.jobOptions.instances = [  # type: ignore
            resourceSpec
            if isinstance(resourceSpec, str)
            else create_instance_from_spec(
                self.manifest, self.rootResource, resourceSpec["name"], resourceSpec
            ).name
            for resourceSpec in self.jobOptions.instances  # type: ignore
        ]
        self.instances = self.jobOptions.instances  # type: ignore

    def create_plan(self) -> List[TaskRequestGroup]:
        self.validate_job_options()
        joboptions = self.jobOptions
        self._update_joboption_instances()
        self.plan_requests = []
        WorkflowPlan = Plan.get_plan_class_for_workflow(joboptions.workflow)  # type: ignore
        if not WorkflowPlan:
            raise UnfurlError(f"unknown workflow: {joboptions.workflow}")  # type: ignore

        rmtree(os.path.join(self.rootResource.base_dir, Folders.Planned))
        plan = WorkflowPlan(self.rootResource, self.manifest.tosca, joboptions)
        plan_requests = list(plan.execute_plan())

        request_artifacts = []
        for r in plan_requests:
            artifacts = r.get_operation_artifacts()
            if artifacts:
                request_artifacts.extend(artifacts)

        # remove duplicates
        artifact_jobs = list({ajr.name: ajr for ajr in request_artifacts}.values())

        # create JobRequests for each external job we need to run by grouping requests by their manifest
        external_requests = []
        for key, reqs in itertools.groupby(artifact_jobs, lambda r: id(r.root)):
            # external manifest activating an instance via artifact reification
            # XXX or substitution mapping -- but unique inputs require dynamically creating ensembles??
            reqs = list(reqs)  # type: ignore
            externalManifest = self.manifest._importedManifests.get(key)
            if externalManifest:
                external_requests.append((externalManifest, reqs))
            else:
                # run artifact jobs as a seperate external job since we need to run them
                # before the render stage of this job
                external_requests.append((self.manifest, reqs))

        self.external_requests = external_requests  # type: ignore
        self.plan_requests = plan_requests
        return self.plan_requests[:]

    def _get_success_status(self, workflow: str, success: Any) -> Optional[Status]:
        if isinstance(success, Status):
            return success
        if success:
            return get_success_status(workflow)
        return None

    def apply_group(self, depth: int, groupRequest: TaskRequestGroup) -> ConfigTask:
        workflow = groupRequest.workflow
        starting_status = groupRequest.target.local_status
        task, success = self.apply(groupRequest.children, depth, groupRequest)
        if task:
            successStatus = self._get_success_status(workflow, success)
            if successStatus is not None and starting_status != successStatus:
                # target's status needs to change
                task.logger.trace(
                    "successStatus %s for %s with local_status %s",
                    successStatus.name,
                    task.target.name,
                    task.target.local_status,  # type: ignore
                )
                # one of the child tasks succeeded and the workflow is one that modifies the target
                # update the target's status
                task._finished_workflow(successStatus, workflow)
        return task

    def apply(
        self,
        taskRequests: List[Union[JobRequest, TaskRequestGroup]],
        depth: int=0,
        parent: "TaskRequestGroup"=None
    ) -> Tuple[ConfigTask, bool]:
        failed = False
        task: Optional[ConfigTask] = None
        successStatus = False
        if parent:
            workflow = parent.workflow
        else:
            workflow = None

        for taskRequest in taskRequests:
            # if parent is set, stop processing requests once one fails
            if parent and failed:
                logger.debug(
                    "Skipping task %s because previous operation failed", taskRequest
                )
                continue
            if isinstance(taskRequest, JobRequest):
                self.jobRequestQueue.append(taskRequest)
                self.run_job_request(taskRequest)
                continue
            elif isinstance(taskRequest, TaskRequestGroup):
                _task = self.apply_group(depth, taskRequest)
            else:
                _task = self._run_operation(taskRequest, workflow, depth)
            if not _task:
                continue
            task = _task

            if task.result.success:  # type: ignore
                if parent and task.target is parent.target:
                    # if the task explicitly set the status use that
                    if task.result.status is not None:  # type: ignore
                        successStatus = task.result.status  # type: ignore
                    else:
                        successStatus = True
            else:
                failed = True
        return task, successStatus  # type: ignore

    def run_job_request(self, jobRequest: JobRequest) -> "Job":
        logger.debug("running jobrequest: %s", jobRequest)
        if self.jobRequestQueue:
            self.jobRequestQueue.remove(jobRequest)
        instance_specs = jobRequest.get_instance_specs()
        jobOptions = self.jobOptions.copy(parentJob=self, instances=instance_specs)
        childJob = create_job(self.manifest, jobOptions)
        childJob.set_task_id(self.increment_task_count())
        assert childJob.parentJob is self
        childJob._run()
        return childJob

    def run_external(self, **opts: Any) -> List["Job"]:
        # note: manifest.lock() will raise error if there circular dependencies
        external_jobs = []
        external_requests = self.external_requests[:]  # type: ignore
        while external_requests:
            manifest, requests = external_requests.pop(0)
            instance_specs = []
            for request in requests:
                assert isinstance(
                    request, JobRequest
                ), "only JobRequest currently supported"
                instance_specs.extend(request.get_instance_specs())
            jobOptions = self.jobOptions.copy(
                instances=instance_specs,
                masterJob=self.jobOptions.masterJob or self,  # type: ignore
                **opts,
            )
            external_job = create_job(manifest, jobOptions)
            external_jobs.append(external_job)
            rendered, count = _render(external_job)
            if not jobOptions.planOnly and count:  # type: ignore
                external_job.run(rendered)
            if external_job.status == Status.error:
                break
        return external_jobs

    def should_run_task(self, task: ConfigTask) -> Tuple[bool, str]:
        """
        Checked at runtime right before each task is run
        """
        try:
            if task._configurator:
                priority = task.configurator.should_run(task)
            else:
                priority = task.priority  # type: ignore
        except Exception:
            # unexpected error don't run this
            UnfurlTaskError(task, "shouldRun failed unexpectedly")
            return False, "unexpected failure"

        if isinstance(priority, bool):
            priority = priority and Priority.required or Priority.ignore
        else:
            priority = to_enum(Priority, priority)
        if priority != task.priority:  # type: ignore
            logger.debug(
                "configurator changed task %s priority from %s to %s",
                task,
                task.priority,  # type: ignore
                priority,
            )
            task.priority = priority  # type: ignore
        if not priority > Priority.ignore:
            return False, "configurator cancelled"

        if task.reason == Reason.reconfigure:  # type: ignore
            if task.has_inputs_changed() or task.has_dependencies_changed():
                return True, "change detected"
            else:
                return False, "no change detected"

        return True, "proceed"

    def can_run_task(self, task: ConfigTask) -> Tuple[bool, str]:
        """
        Checked at runtime right before each task is run

        * validate inputs
        * check pre-conditions to see if it can be run
        * check task if it can be run
        """
        can_run = False
        reason = ""
        try:
            if task._errors:
                can_run = False
                reason = "Error while creating task"  # XXX + str(_errors)
            elif task.dry_run and not task.configurator.can_dry_run(task):
                can_run = False
                reason = "dry run not supported"
            else:
                missing, reason = task.find_missing_dependencies()
                if not missing:
                    errors = task.configSpec.find_invalidate_inputs(task.inputs)
                    if errors:
                        reason = f"invalid inputs: {str(errors)}"
                    else:
                        preErrors = task.configSpec.find_invalid_preconditions(
                            task.target
                        )
                        if preErrors:
                            reason = f"invalid preconditions: {str(preErrors)}"
                        else:
                            errors = task.configurator.can_run(task)
                            if not errors or not isinstance(errors, bool):
                                reason = f"configurator declined: {str(errors)}"
                            else:
                                can_run = True
        except Exception:
            UnfurlTaskError(task, "can_run_task failed unexpectedly")
            reason = "unexpected exception in can_run_task"
            can_run = False

        if can_run:
            return True, ""
        else:
            logger.info("could not run task %s: %s", task, reason)
            return False, "could not run: " + reason

    def _set_state(self, req: SetStateRequest) -> None:
        logger.debug("setting state %s for %s", req.set_state, req.target)
        resource = req.target
        if "managed" in req.set_state:
            resource.created = False if req.set_state == "unmanaged" else self.changeId
        else:
            try:
                resource.state = req.set_state
            except KeyError:
                resource.local_status = to_enum(Status, req.set_state)

    def _entry_test(self, req: TaskRequest, workflow: str) -> Tuple[bool, str]:
        """
        Operations can dynamically advance the state of a instance so that an operation
        added by the plan no longer needs to run.
        For example, a check operation may determine a resource is already created
        or a create operation may also configure and start an instance.
        """
        resource = req.target
        logger.trace(
            "checking operation entry test: current state %s start state %s op %s workflow %s",
            resource.state,
            req.startState,
            req.configSpec.operation,
            workflow,
        )
        if req.configSpec.operation == "check":
            missing, reason = _dependency_check(resource)
            if missing:
                return False, reason
            if not workflow:
                if (
                    self.is_change_id(resource.parent.created)
                    and self.get_job_id(resource.parent.created) == self.changeId
                ):
                    # optimization:
                    # if parent was created during this job we don't need to run check operation
                    # we know we couldn't have existed
                    resource._localStatus = Status.pending
                    return False, "skipping check on a new instance"
                else:
                    return True, "passed"

        if self.jobOptions.force:  # type: ignore  # always run
            return True, "passed"

        if get_success_status(workflow) == resource.status:
            return False, "instance already has desired status"

        if req.startState and resource.state and workflow == "deploy":
            # if we set a startState and force isn't set then only run
            # if we haven't already advanced to that state by another operation
            entryTest = NodeState(req.startState + 1)
            if resource.state > NodeState.started:
                # "started" is the final deploy state, target state must be stopped or deleted or error
                if req.configSpec.operation == "start":
                    # start operations can't handle deleted nodes
                    return (
                        resource.state <= NodeState.stopped,
                        "can't start a missing instance",
                    )
            elif resource.state > entryTest:
                return False, "instance already entered state"
        return True, "passed"

    def _run_operation(self, req: TaskRequest, workflow: str, depth: int) -> Optional[ConfigTask]:
        if isinstance(req, SetStateRequest):
            logger.debug("Setting state with %s", req)
            self._set_state(req)
            return None

        assert isinstance(req, TaskRequest)
        if req.required is False:  # set after should_run() is called
            return None
        if req.error:
            return None

        logger.info("Running task %s", req)
        test, msg = self._entry_test(req, workflow)
        if not test:
            logger.debug(
                "skipping operation %s for instance %s with state %s and status %s: %s",
                req.configSpec.operation,
                req.target.name,
                req.target.state,
                req.target.status,
                msg,
            )
            return None

        task = req.task or self.create_task(
            req.configSpec, req.target, reason=req.reason
        )
        if task:
            resource = req.target
            startingStatus = resource._localStatus
            if req.startState is not None:
                resource.state = req.startState
            startingState = resource.state
            self.add_work(task)
            self.run_task(task, depth)

            # if # task succeeded but didn't update nodestate
            if task.result.success and resource.state == startingState:  # type: ignore
                if req.startState is not None:
                    # advance the state if a startState was set in the TaskRequest
                    resource.state = NodeState(req.startState + 1)
                elif (
                    req.configSpec.operation == "check"
                    and startingStatus != resource._localStatus
                ):
                    # if check operation set status but didn't update state, set a default state
                    state_map = {
                        Status.ok: NodeState.started,
                        Status.error: NodeState.error,
                        Status.absent: NodeState.deleted,
                        Status.pending: NodeState.initial,
                    }
                    if not resource.state or resource.state == NodeState.initial:
                        # it is a resource we didn't create
                        state_map[Status.absent] = NodeState.initial
                    else:
                        state_map[Status.absent] = NodeState.deleted
                    state = state_map.get(resource._localStatus)
                    if state is not None:
                        resource.state = state
                task.target_state = resource.state

            # logger.debug(
            #     "changed %s to %s, wanted %s",
            #     startingState,
            #     task.target.state,
            #     req.startState,
            # )
            logger.info(
                "finished taskrequest %s: %s %s %s",
                task,
                "success" if task.result.success else "failed",  # type: ignore
                task.target.status.name,
                task.target.state and task.target.state.name or "",  # type: ignore
            )

        return task

    def run_task(self, task: ConfigTask, depth: int=0) -> ConfigTask:
        """
        During each task run:
        * Notification of metadata changes that reflect changes made to resources
        * Notification of add or removing dependency on a resource or properties of a resource
        * Notification of creation or deletion of a resource
        * Requests a resource with requested metadata, if it doesn't exist, a task is run to make it so
        (e.g. add a dns entry, install a package).

        Returns a task.
        """
        task.target.root.attributeManager = task._attributeManager
        ok, errors = self.can_run_task(task)
        if not ok:
            return task.finished(ConfiguratorResult(False, False, result=errors))

        task.start()
        change = None
        while True:
            try:
                result = task.send(change)
            except Exception:
                UnfurlTaskError(task, "configurator.run failed")
                return task.finished(ConfiguratorResult(False, None, Status.error))
            if isinstance(result, PlanRequest):
                if depth >= self.MAX_NESTED_SUBTASKS:
                    UnfurlTaskError(task, "too many subtasks spawned")
                    change = task.finished(ConfiguratorResult(False, None))
                else:
                    ready, _, errors = do_render_requests(self, [result])
                    if not ready:
                        return result.task.finished(  # type: ignore
                            ConfiguratorResult(False, False, result=errors)
                        )
                    change, success = self.apply(ready, depth + 1)
            elif isinstance(result, JobRequest):
                job = self.run_job_request(result)
                change = job
            elif isinstance(result, ConfiguratorResult):
                retVal = task.finished(result)
                logger.debug(
                    "completed task %s: %s; %s", task, task.target.status, result
                )
                return retVal
            else:
                UnfurlTaskError(task, "unexpected result from configurator")
                return task.finished(ConfiguratorResult(False, None, Status.error))

    def add_work(self, task: ConfigTask) -> None:
        key = id(task)
        self.workDone[key] = task
        if self.parentJob:  # type: ignore
            self.parentJob.add_work(task)  # type: ignore

    def increment_task_count(self) -> int:
        if self.parentJob:  # type: ignore
            return self.parentJob.increment_task_count()  # type: ignore
        else:
            self.task_count += 1
        return self.task_count

    ###########################################################################
    ### Reporting methods
    ###########################################################################
    @staticmethod
    def _job_request_summary(requests: List[JobRequest], manifest: Optional["YamlManifest"]) -> Iterable[dict]:
        for request in requests:
            # XXX better reporting
            node = dict(instance=request.name)
            if manifest:
                node["job_request"] = manifest.path
            else:
                node["job_request"] = "local"
            if request.target:
                node["status"] = str(request.target.status)
            yield node

    @staticmethod
    def _switch_target(target: NodeInstance, old_summary_list: List[dict]) -> List[dict]:
        new_summary_list: List[dict] = []
        node = dict(
            instance=target.name,
            status=str(target.status),
            state=str(target.state),  # type: ignore
            managed=target.created,
            plan=new_summary_list,
        )
        old_summary_list.append(node)
        return new_summary_list

    @staticmethod
    def _list_plan_summary(
        requests: List[JobRequest],
        target: NodeInstance,
        parent_summary_list: List[dict],
        include_rendered: bool,
        workflow: str
    ) -> None:
        summary_list = parent_summary_list
        for request in requests:
            if isinstance(request, JobRequest):
                summary_list.extend(Job._job_request_summary([request], None))
                continue
            isGroup = isinstance(request, TaskRequestGroup)
            if isGroup and not request.children:
                continue  # don't include in the plan
            if request.target is not target:
                if workflow == "deploy" and not request.target.template.required:
                    continue
                # target changed, add it to the parent's list
                # switch to the "plan" member of the new target
                target = request.target
                summary_list = Job._switch_target(target, parent_summary_list)
            if isGroup:
                sequence = []
                group = {}
                if request.workflow:
                    group["workflow"] = str(request.workflow)
                group["sequence"] = sequence
                summary_list.append(group)
                Job._list_plan_summary(
                    request.children, target, sequence, include_rendered, workflow
                )
            else:
                summary_list.append(request._summary_dict(include_rendered))

    def _json_plan_summary(self, pretty: bool=False, include_rendered: bool=True) -> Union[str, list]:
        """
        Return a list of items that look like:

          {
          instance: target_name,
          status: target_status,
          plan: [
              {"operation": "check"
                "sequence": [
                    <items like these>
                  ]
              }
            ]
          }
        """
        summary: List[dict] = []
        for (m, requests) in self.external_requests:  # type: ignore
            summary.extend(self._job_request_summary(requests, m))
        self._list_plan_summary(self.plan_requests, None, summary, include_rendered, self.workflow)  # type: ignore
        if not pretty:
            return summary
        else:
            return json.dumps(summary, indent=2)

    def json_summary(self, pretty: bool=False, external: bool=False) -> Union[str, Dict[str, Any]]:
        job = dict(id=self.changeId, status=self.status.name)
        job.update(self.stats())  # type: ignore
        if not self.startTime:  # skip if startTime was explicitly set
            job["timeElapsed"] = self.timeElapsed  # type: ignore
        summary = dict(
            job=job,
            outputs=serialize_value(self.get_outputs()),
            tasks=[task.summary(True) for task in self.workDone.values()],
        )
        if external:
            summary["ensemble"] = self.manifest.path
        if self.external_jobs:
            summary["external_jobs"] = [
                external.json_summary(external=True) for external in self.external_jobs
            ]
        if pretty:
            return json.dumps(summary, indent=2)
        return summary

    def stats(self, asMessage: bool=False) -> Union[Dict[str, int], str]:
        tasks = self.workDone.values()
        key = lambda t: t._localStatus or Status.unknown
        tasks = sorted(tasks, key=key)  # type: ignore
        stats = dict(total=len(tasks), ok=0, error=0, unknown=0, skipped=0)
        for k, g in itertools.groupby(tasks, key):
            if not k:  # is a Status
                stats["skipped"] = len(list(g))
            else:
                stats[k.name] = len(list(g))
        stats["changed"] = len([t for t in tasks if t.modified_target])
        if asMessage:
            return "{total} tasks ({changed} changed, {ok} ok, {error} failed, {unknown} unknown, {skipped} skipped)".format(
                **stats
            )
        return stats

    def _plan_summary(
        self,
        plan_requests: List[TaskRequest],
        external_requests: Iterable[Tuple[Any, Any]]
    ) -> Tuple[str, int]:
        """
        Node "site" (status, state, created):
          check: Install.check
          workflow: # if group
            Standard.create (reason add)
            Standard.configure (reason add)
        """
        INDENT = 4
        count = 0

        def _summary(
            requests: List[Union[JobRequest, TaskRequest, TaskRequestGroup]],
            target: Optional[NodeInstance],
            indent: int
        ) -> None:
            nonlocal count
            for request in requests:
                isGroup = isinstance(request, TaskRequestGroup)
                if isGroup and not request.children:  # type: ignore
                    continue
                if not self.is_filtered() and self.workflow == "deploy":  # type: ignore
                    if not request.target.template.required:
                        logger.trace('excluding "%s" from plan: not required', request.target.name)
                        continue
                if isinstance(request, JobRequest):
                    count += 1
                    nodeStr = f'Job for "{request.name}":'
                    output.append(" " * indent + nodeStr)
                    continue
                if request.target is not target:
                    target = request.target
                    status = ", ".join(
                        filter(
                            None,
                            (
                                target.status.name if target.status is not None else "",  # type: ignore
                                target.state.name if target.state is not None else "",  # type: ignore
                                "managed" if target.created else "",  # type: ignore
                            ),
                        )
                    )
                    nodeStr = f'Node "{target.name}" ({status}):'  # type: ignore
                    output.append(" " * indent + nodeStr)
                if isGroup:
                    output.append(
                        "%s- %s:" % (" " * indent, (request.workflow or "sequence"))  # type: ignore
                    )
                    _summary(request.children, target, indent + INDENT)  # type: ignore
                else:
                    count += 1
                    output.append(" " * indent + f"- operation {request.name}")  # type: ignore
                    if request.task:
                        if request.task._workFolders:
                            for wf in request.task._workFolders.values():
                                output.append(" " * indent + f"   rendered at {wf.cwd}")
                        if request.future_dependencies:
                            output.append(
                                " " * indent + "   (render waiting for dependents)"
                            )
                        elif request.task._errors:  # don't report error if waiting
                            output.append(" " * indent + "   (errors while rendering)")

        opts = self.jobOptions.get_user_settings()
        options = ",".join([f"{k} = {opts[k]}" for k in opts if k != "planOnly"])
        header = f"Plan for {self.workflow}"  # type: ignore
        if options:
            header += f" ({options})"
        output = [header + ":\n"]

        for m, jr in external_requests:
            if jr:
                count += 1
                output += [f" External jobs on {m.path}:"]
                for j in jr:
                    output.append(" " * INDENT + j.name)

        _summary(plan_requests, None, 0)  # type: ignore
        if not count:
            output.append("Nothing to do.")
        return "\n".join(output), count

    def summary(self) -> str:
        outputString = ""
        outputs = self.get_outputs()
        if outputs:
            outputString = "\nOutputs:\n    " + "\n    ".join(
                f"{name}: {value}" for name, value in serialize_value(outputs).items()
            )

        if not self.workDone:
            return f"Job {self.changeId} completed: {self.status.name}. Found nothing to do. {outputString}"

        def format(i: int, task: ConfigTask) -> str:
            return "%d. %s; %s" % (i, task.summary(), task.result or "skipped")

        line1 = "Job %s completed in %.3fs: %s. %s:\n    " % (
            self.changeId,
            self.timeElapsed,
            self.status.name,
            self.stats(asMessage=True),
        )
        tasks = "\n    ".join(
            format(i + 1, task) for i, task in enumerate(self.workDone.values())
        )
        return line1 + tasks + outputString

    @property
    def log_path(self) -> str:
        log_name = (
            self.startTime.strftime("%Y-%m-%d-%H-%M-%S") + "-" + self.changeId[:-4]
        )
        return self.manifest.get_job_log_path(log_name, ".log")


def create_job(manifest, joboptions, previousId=None):
    """
    Selects task to run based on the workflow and job options
    """
    root = manifest.get_root_resource()
    assert manifest.tosca
    job = Job(manifest, root, joboptions, previousId)

    if manifest.localEnv and not joboptions.parentJob and not joboptions.startTime:
        path = manifest.path
        if joboptions.planOnly:
            logger.info("creating %s plan for %s", joboptions.workflow, path)
        else:
            logger.info("starting %s job for %s", joboptions.workflow, path)

    return job


def _plan(manifest, jobOptions):
    assert jobOptions
    job = create_job(
        manifest,
        jobOptions,
        manifest.lastJob and manifest.lastJob["changeId"],
    )
    job.create_plan()
    if jobOptions.masterJob:
        msg = "Created static plan for external job:"
    else:
        msg = "Created static plan:"
    msg, count = job._plan_summary(job.plan_requests, job.external_requests)
    logger.debug(msg + "\n%s", msg)
    return job


def _render(job):
    # note: we need to call render() before lock because render might run this ensemble as an external_job
    with change_cwd(job.manifest.get_base_dir()):
        ready, notReady, errors = job.render()
        msg, count = job._plan_summary(ready + notReady, [])
        logger.info(msg)
    return (ready, notReady, errors), count


def start_job(manifestPath=None, _opts=None):
    _opts = _opts or {}
    localEnv = LocalEnv(manifestPath, _opts.get("home"), override_context=_opts.get("use_environment"))
    opts = JobOptions(**_opts)
    path = localEnv.manifestPath
    if not opts.planOnly:
        logger.info("creating %s job for %s", opts.workflow, path)
    try:
        manifest = localEnv.get_manifest()
    except Exception as e:
        logger.error(
            "failed to load manifest at %s: %s",
            path,
            str(e),
            exc_info=opts.verbose >= 2,
        )
        return None, None, False

    job = _plan(manifest, opts)
    rendered, count = _render(job)
    errors = rendered[2]
    if errors:
        logger.error("Aborting job: there were errors during rendering: %s", errors)
        job.local_status = Status.error  # type: ignore
    return job, rendered, count and not errors


def run_job(manifestPath: Optional[str]=None, _opts: Optional[dict]=None) -> Job:
    """
    Loads the given Ensemble and creates and runs a job.

    Args:
        manifestPath (:obj:`str`, optional) The path the Ensemble manifest.
         If None, it will look for an ensemble in the current working directory.
        _opts (:obj:`dict`, optional) A dictionary of job options. Names and values should match
          the names of the command line options for creating jobs.

    Returns:
        (:obj:`Job`): The job that just ran or None if it couldn't be created.
    """
    job, rendered, proceed = start_job(manifestPath, _opts)
    if job:
        if not job.unexpectedAbort and not job.planOnly and proceed:
            job.run(rendered)
    return job


class Runner:
    # this class is only used by unit tests, application uses start_job() above

    def __init__(self, manifest):
        self.manifest = manifest
        assert self.manifest.tosca

    def run(self, jobOptions=None):
        if jobOptions is None:
            jobOptions = JobOptions()
        job = _plan(self.manifest, jobOptions)
        rendered, count = _render(job)
        if not jobOptions.planOnly:
            job.run(rendered)
        return job
