# -*- coding: utf-8 -*-
#
# Copyright 2018-2020- Swiss Data Science Center (SDSC)
# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
# Eidgenössische Technische Hochschule Zürich (ETHZ).
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Graph builder."""

import os
from collections import OrderedDict, defaultdict, deque
from pathlib import Path

import attr

from renku.core import errors
from renku.core.models.entities import Collection, Entity
from renku.core.models.git import Range
from renku.core.models.provenance.activities import Activity, ProcessRun, Usage
from renku.core.models.provenance.processes import Process
from renku.core.models.provenance.qualified import Generation
from renku.core.models.workflow.run import Run


def _safe_path(filepath, can_be_cwl=False):
    """Check if the path should be used in output."""
    if isinstance(filepath, Path):
        filepath = str(filepath)

    # Should not be in ignore paths.
    if filepath in {'.gitignore', '.gitattributes'}:
        return False

    # Ignore everything in .renku ...
    if filepath.startswith('.renku'):
        # ... unless it can be a CWL.
        if can_be_cwl and filepath.startswith('.renku/workflow/'):
            return True
        return False

    return True


@attr.s(cmp=False)
class Graph(object):
    """Represent the provenance graph."""

    client = attr.ib()
    activities = attr.ib(default=attr.Factory(dict))
    generated = attr.ib(default=attr.Factory(dict))

    _sorted_commits = attr.ib(default=attr.Factory(list))
    _latest_commits = attr.ib(default=attr.Factory(dict))
    _nodes = attr.ib()
    _need_update = attr.ib(default=attr.Factory(dict))

    cwl_prefix = attr.ib(init=False)

    def __attrs_post_init__(self):
        """Derive basic informations."""
        self.cwl_prefix = self.client.cwl_prefix

    @_nodes.default
    def default_nodes(self):
        """Build node index."""
        self.generated = {}
        nodes = OrderedDict()

        for commit in reversed(self._sorted_commits):
            try:
                activity = self.activities[commit]
                nodes.update(((node.commit, node.path), node)
                             for node in reversed(list(activity.nodes)))

                if isinstance(activity, ProcessRun):
                    self.generated.update({
                        generation.entity._id: generation
                        for generation in activity.generated
                    })

            except KeyError:
                pass

        return nodes

    def need_update(self, node):
        """Return out-dated nodes."""
        if node is None:
            return

        skip = True
        if isinstance(node, ProcessRun):
            node = node.association.plan
            skip = False

        if node._id in self._need_update:
            return self._need_update[node._id]

        latest = self.latest(node)
        if latest:
            return self._need_update.setdefault(node._id, [node])

        need_update_ = []

        for parent in self.parents(node):
            # Skip Collections if it is not an input
            if skip and isinstance(parent, Collection):
                continue

            parent_updates = self.need_update(parent)
            if parent_updates:
                need_update_.extend(parent_updates)

        return self._need_update.setdefault(node._id, need_update_)

    def parents(self, node):
        """Return parents for a given node."""
        import warnings

        def _from_entity(entity, check_parents=True):
            """Find parent from entity."""
            try:
                return [self.generated[entity._id].activity]
            except KeyError:
                id_ = Path(entity._id)
                while check_parents and id_ != id_.parent:
                    try:
                        # TODO include selection step here
                        return [self.generated[str(id_)]]
                    except KeyError:
                        id_ = id_.parent
                return []

        if isinstance(node, Generation):
            result = [node.parent] if node.parent is not None else []
            if node.activity and isinstance(node.activity, ProcessRun):
                return result + self.parents(node.activity.association.plan)
            return result
        elif isinstance(node, Usage):
            return _from_entity(node.entity)
        elif isinstance(node, Entity):
            # Link files and directories and generations.
            return ([node.parent] if node.parent is not None else
                    []) + _from_entity(node, check_parents=False)
        elif isinstance(node, Process) or isinstance(node, Run):
            # warnings.warn('Called on run {0}'.format(node), stacklevel=2)
            return self.parents(node.activity)
        elif isinstance(node, ProcessRun):
            return node.qualified_usage
        elif isinstance(node, Activity):
            warnings.warn('Called parents on {0}'.format(node), stacklevel=2)
            return []

        raise NotImplementedError(node)

    def latest(self, node):
        """Return a latest commit where the node was modified."""
        if node.path and node.path not in self._latest_commits:
            try:
                latest = Usage.from_revision(
                    node.client,
                    path=node.path,
                    # TODO support range queries
                    # revision='{0}'.format(node.commit.hexsha),
                ).commit
            except KeyError:
                latest = None

            self._latest_commits[node.path] = latest
        else:
            latest = self._latest_commits.get(node.path)

        if latest and latest != node.commit:
            return latest

    @property
    def nodes(self):
        """Return topologically sorted nodes."""
        return reversed(self._nodes.values())

    def normalize_path(self, path):
        """Normalize path relative to the Git workdir."""
        start = self.client.path.resolve()
        try:
            p = Path(path).resolve()
            p.relative_to(self.client.path)
        except ValueError:  # External file
            path = Path(os.path.abspath(path))
        else:
            path = p
        return os.path.relpath(str(path), start=str(start))

    def _format_path(self, path):
        """Return a relative path based on the client configuration."""
        return os.path.relpath(str(self.client.path / path))

    def dependencies(self, revision='HEAD', paths=None):
        """Return dependencies from a revision or paths."""
        result = []

        if paths:
            paths = (self.normalize_path(path) for path in paths)
        else:
            if revision == 'HEAD':
                index = self.client.repo.index
            else:
                from git import IndexFile
                index = IndexFile.from_tree(self.client.repo, revision)

            paths = (path for path, _ in index.entries.keys())

        for path in paths:
            try:
                result.append(
                    Usage.from_revision(
                        self.client,
                        path=path,
                        revision=revision,
                    )
                )
            except KeyError:
                continue

        return result

    def process_dependencies(self, dependencies, visited=None):
        """Process given dependencies."""
        for dependency in dependencies:
            # We can't simply reuse information from submodules
            if dependency.client != self.client:
                continue
            self._latest_commits[dependency.path] = dependency.commit

        visited = visited or set()
        queue = deque(dependencies)
        usage_paths = []

        while queue:
            processing = queue.popleft()

            if processing.commit in visited:
                continue

            # Mark as visited:
            visited.add(processing.commit)

            activity = processing.client.process_commit(processing.commit)
            if activity is None:
                continue

            self.activities[activity.commit] = activity

            # Iterate over parents.
            if isinstance(activity, ProcessRun):
                for entity in activity.qualified_usage:
                    for member in entity.entities:
                        parent_activities = self.client.activities_for_paths(
                            member.path, revision='HEAD'
                        )
                        for a in parent_activities:
                            if a.commit and a.commit not in visited:
                                self.activities[a.commit] = a
                        if member.commit not in visited:
                            queue.append(member)
                        usage_paths.append(member.path)
                for entity in activity.generated:
                    for member in entity.entities:
                        if (
                            all(member.path != d.path
                                for d in dependencies) and
                            any(
                                u.startswith(member.path) for u in usage_paths
                            )
                        ):
                            dependencies = [
                                d for d in dependencies
                                if not d.path.startswith(member.path)
                            ]
                            dependencies.append(member)

        from renku.core.models.sort import topological

        commit_nodes = {
            commit: activity.parents
            for commit, activity in self.activities.items()
        }

        # add dependencies between processes
        for activity in self.activities.values():
            if not isinstance(activity, ProcessRun):
                continue
            for usage in activity.qualified_usage:
                for other_activity in self.activities.values():
                    if other_activity == activity:
                        continue
                    if any(
                        g.path == usage.path and
                        g.commit.hexsha == usage.commit.hexsha
                        for g in other_activity.generated
                    ):
                        parents = commit_nodes[activity.commit]
                        if other_activity.commit in parents:
                            continue
                        parents.append(other_activity.commit)

        self._sorted_commits = topological(commit_nodes)
        self._nodes = self.default_nodes()

        return dependencies

    def build(
        self, revision='HEAD', paths=None, dependencies=None, can_be_cwl=False
    ):
        """Build graph from paths and/or revision."""
        interval = Range.rev_parse(self.client.repo, revision)

        if dependencies is None:
            dependencies = self.dependencies(revision=revision, paths=paths)

        ignore = {
            commit
            for commit in self.client.repo.iter_commits(interval.start)
        } if interval.start else set()
        dependencies = self.process_dependencies(dependencies, visited=ignore)

        return {
            self._nodes.get((dependency.commit, dependency.path), dependency)
            for dependency in dependencies
            if _safe_path(dependency.path, can_be_cwl=can_be_cwl)
        }

    @property
    def output_paths(self):
        """Return all output paths."""
        paths = set()
        for activity in self.activities.values():
            if (
                isinstance(activity, ProcessRun) and activity.association and
                activity.association.plan
            ):
                paths |= {
                    o.produces.path
                    for o in activity.association.plan.outputs
                    if o.produces.path
                }
        return paths

    def build_status(self, revision='HEAD', can_be_cwl=False):
        """Return files from the revision grouped by their status."""
        status = {
            'up-to-date': {},
            'outdated': {},
            'multiple-versions': {},
            'deleted': {},
        }

        dependencies = self.dependencies(revision=revision)
        current_files = self.build(
            dependencies=dependencies,
            can_be_cwl=can_be_cwl,
        )

        # TODO check only outputs
        paths = {}
        for commit in reversed(self._sorted_commits):
            activity = self.activities.get(commit)

            if isinstance(activity, ProcessRun):
                nodes = activity.nodes if can_be_cwl else activity.generated

                for node in nodes:
                    paths[node.path] = node

        # First find all up-to-date nodes.
        for node in paths.values():
            # for node in current_files:
            need_update = [
                dependency for dependency in self.need_update(node)
                if dependency.path != node.path
            ]

            if need_update:
                status['outdated'][node.path] = need_update
            else:
                status['up-to-date'][node.path] = node.commit

        # Merge all versions of used inputs in outdated file.
        multiple_versions = defaultdict(set)

        for need_update in status['outdated'].values():
            for node in need_update:
                multiple_versions[node.path].add(node)

        for node in current_files:
            if node.path in multiple_versions:
                multiple_versions[node.path].add(node)

        status['multiple-versions'] = {
            key: value
            for key, value in multiple_versions.items() if len(value) > 1
        }

        # Build a list of used files that have been deleted.
        current_paths = {node.path for node in current_files}
        status['deleted'] = {
            node.path: node
            for node in self.nodes
            if _safe_path(node.path, can_be_cwl=can_be_cwl) and
            node.path not in current_paths and
            not ((self.client.path / node.path).exists() or
                 (self.client.path / node.path).is_dir())
        }
        return status

    def siblings(self, node):
        """Return siblings for a given node.

        The key is part of the result set, hence to check if the node has
        siblings you should check the lenght is greater than 1.
        """
        parent = None

        if isinstance(node, Entity):
            if not node.parent:
                return {node}
            parent_siblings = self.siblings(node.parent) - {node.parent}
            return set(node.parent.members) | parent_siblings
        elif isinstance(node, Generation):
            parent = node.activity
        elif isinstance(node, Usage):
            parent = self.activities[node.commit]
        elif isinstance(node, Process) or isinstance(node, Run):
            return {node}

        if parent is None or not isinstance(parent, ProcessRun):
            raise errors.InvalidOutputPath(
                'The file "{0}" was not created by a renku command. \n\n'
                'Check the file history using: git log --follow "{0}"'.format(
                    node.path
                )
            )

        return set(parent.generated)

    def as_workflow(
        self,
        input_paths=None,
        output_paths=None,
        outputs=None,
        use_latest=True,
    ):
        """Serialize graph to renku ``Run`` workflow.

        :param global_step_outputs: Make all step outputs global.
        """
        if output_paths is None:
            output_paths = {
                node.path
                for node in outputs if _safe_path(node.path)
            }

        processes = set()
        stack = []

        output_keys = {(node.commit, node.path) for node in outputs}
        nodes = {(node.commit, node.path): node for node in self.nodes}

        for node in self.nodes:
            if (node.commit, node.path) not in output_keys:
                continue

            assert hasattr(node, 'activity'), node
            assert isinstance(node.activity, ProcessRun)

            plan = node.activity.association.plan
            process_run = plan.activity

            if (
                input_paths and
                any(g.path in input_paths for g in process_run.generated)
            ):
                continue

            if process_run not in processes:
                stack.append(process_run)
                processes.add(process_run)

        while stack:
            action = stack.pop()

            if (
                not hasattr(action, 'association') or
                not hasattr(action.association.plan, 'inputs')
            ):
                continue

            for inp in action.association.plan.inputs:
                path = inp.consumes.path
                dependency = inp.consumes
                # Do not follow defined input paths.
                if input_paths and path in input_paths:
                    continue

                node = nodes.get((dependency.commit, dependency.path),
                                 dependency)

                if isinstance(node, Generation):
                    process_run = node.activity
                elif isinstance(node, Collection) and node.parent:
                    raise NotImplementedError('Can not connect subdirectory')
                else:
                    process_run = None

                # Skip existing commits
                if process_run and isinstance(process_run, ProcessRun):
                    plan = process_run.association.plan
                    latest = self.latest(plan)
                    if process_run.path and use_latest and latest:
                        plan = nodes[(latest, plan.path)]

                    process_run = plan.activity

                    if process_run not in processes:
                        stack.append(process_run)
                        processes.add(process_run)

        parent_process = Run()

        for step in processes:
            # Loop through runs and add them as sub processes to parent.
            parent_process.add_subprocess(step.association.plan)

        return parent_process
