# *************************************************************************
#
#  Copyright (c) 2026 - Datatailr Inc.
#  All Rights Reserved.
#
#  This file is part of Datatailr and subject to the terms and conditions
#  defined in 'LICENSE.txt'. Unauthorized copying and/or distribution
#  of this file, in parts or full, via any medium is strictly prohibited.
# *************************************************************************

from __future__ import annotations

from collections import defaultdict
from datetime import datetime
import inspect
import functools
import importlib.util
from importlib.metadata import packages_distributions
import json
import os
import tempfile
import uuid
from dataclasses import dataclass
from enum import Enum
import re
import sys
import types
from typing import Callable, Dict, Optional, Tuple, Union, List

from datatailr import ACL, Environment, User
from datatailr.scheduler.utils import get_base_url
from datatailr.wrapper import dt__Job
from datatailr.scheduler.constants import DEFAULT_TASK_MEMORY, DEFAULT_TASK_CPU
from datatailr.scheduler.utils import (
    normalize_name,
    get_path_to_module,
    get_path_to_repo,
    get_imports_from_code,
    RepoValidationError,
)
from datatailr.build.image import Image
from datatailr.errors import TaskError
from datatailr.logging import GREEN, YELLOW, BLUE, BOLD, RED, DatatailrLogger
from datatailr.utils import (
    run_shell_command,
    dict_to_env_vars,
    is_dt_installed,
    is_remote_cli_dt_installed,
)

logger = DatatailrLogger(os.path.abspath(__file__)).get_logger()
__client__ = dt__Job()


@functools.lru_cache(maxsize=1)
def _cached_packages_distributions():
    return packages_distributions()


def set_allow_unsafe_scheduling(allow: bool):
    """
    Set whether to allow unsafe scheduling of jobs.
    This is a global setting that affects how jobs are scheduled.
    """
    if allow:
        os.environ["DATATAILR_ALLOW_UNSAFE_SCHEDULING"] = "true"
    else:
        os.environ.pop("DATATAILR_ALLOW_UNSAFE_SCHEDULING", None)


class JobType(Enum):
    """
    Enum representing different types of Datatailr jobs.
    """

    BATCH = "batch"
    WORKFLOW = "workflow"
    SERVICE = "service"
    APP = "app"
    EXCEL = "excel"
    WORKSTATION = "workstation"
    UNKNOWN = "unknown"

    def __str__(self):
        if self == JobType.WORKFLOW:
            return JobType.BATCH.value
        return self.value

    def __repr__(self):
        return f"JobType.{self.name}('{self.value}')"

    def display_name(self) -> str:
        return {
            JobType.BATCH.name: "Workflow",
            JobType.WORKFLOW.name: "Workflow",
            JobType.EXCEL.name: "Excel Add-in",
            JobType.WORKSTATION.name: "Workstation",
            JobType.UNKNOWN.name: "Job",
        }.get(self.name, self.name.capitalize())

    @classmethod
    def from_str(cls, value: str | JobType | None) -> JobType:
        """Create a JobType from a string while normalizing legacy aliases."""
        if isinstance(value, cls):
            return value
        normalized = str((value or cls.UNKNOWN.value)).lower()
        aliases = {
            cls.BATCH.value: cls.WORKFLOW.value,
        }
        return cls(aliases.get(normalized, normalized))


@dataclass
class Resources:
    """Represents the compute resources allocated to a job container.

    Attributes:
        memory: Memory limit as a string (e.g. ``"256m"``, ``"1g"``).
        cpu: Number of CPU cores to allocate (e.g. ``1``, ``0.5``).
    """

    memory: str = DEFAULT_TASK_MEMORY
    cpu: float = DEFAULT_TASK_CPU

    @classmethod
    def from_job_dict(cls, job_dict: dict) -> Resources:
        if "memory" in job_dict:
            memory = job_dict["memory"]
            memory = f"{int(int(memory) / 1024 ** 2)}m"
        else:
            memory = DEFAULT_TASK_MEMORY
        if "cpu" in job_dict:
            cpu = job_dict["cpu"]
        else:
            cpu = DEFAULT_TASK_CPU
        return cls(memory=memory, cpu=float(cpu))


class EntryPoint:
    """
    Represents an entry point for a Datatailr job.
    This can be a function or a callable object.
    """

    def __init__(
        self,
        job_type: JobType,
        func_or_module: Callable | str,
    ):
        self.func = func_or_module
        if isinstance(func_or_module, str):
            self.module_name = func_or_module
            self.function_name = func_or_module
        elif isinstance(func_or_module, types.ModuleType):
            self.module_name = func_or_module.__name__
            self.function_name = func_or_module.__name__
        elif isinstance(func_or_module, (types.FunctionType, types.MethodType)):
            self.module_name = (
                func_or_module.__module__
                if hasattr(func_or_module, "__module__")
                else ""
            )
            self.function_name = (
                str(func_or_module)
                if isinstance(func_or_module, str)
                else getattr(func_or_module, "__name__", str(func_or_module))
            )
        else:
            raise ValueError(f"Invalid type: {type(func_or_module)}")
        self.type = job_type
        try:
            self.detected_imports = (
                get_imports_from_code(inspect.getfile(func_or_module))[1]
                if not isinstance(func_or_module, str)
                else set()
            )
        except (TypeError, FileNotFoundError):
            self.detected_imports = set()

        if isinstance(func_or_module, str):
            return
        # Find the absolute path to the repository and then the relative path to the module.
        # This will be used in the creation of the code 'bundle' when building the image.
        path_to_module = get_path_to_module(func_or_module)
        path_to_repo = get_path_to_repo(path_to_module)
        path_to_module = (
            os.path.relpath(path_to_module, path_to_repo)
            if path_to_repo != path_to_module
            else "."
        )
        self.path_to_repo = path_to_repo
        self.path_to_module = path_to_module
        if self.module_name == "__main__":
            if hasattr(func_or_module, "__code__"):
                code = func_or_module.__code__
                if hasattr(code, "co_filename"):
                    file_name = str(code.co_filename)
                    self.module_name = os.path.splitext(os.path.basename(file_name))[0]

    def __call__(self, *args, **kwargs):
        os.environ.update(kwargs.pop("env", {}))
        if self.type == JobType.WORKFLOW:
            os.environ["DATATAILR_BATCH_DONT_RUN_WORKFLOW"] = "true"
            module = importlib.import_module(self.module_name)
            os.environ.pop("DATATAILR_BATCH_DONT_RUN_WORKFLOW", None)
            func = getattr(module, self.function_name)
            return func(*args, **kwargs)
        else:
            raise NotImplementedError(
                f"EntryPoint of type '{self.type}' is not callable."
            )

    @classmethod
    def from_dict(cls, entrypoint_dict: dict) -> EntryPoint:
        module_name, function_name = entrypoint_dict["entrypoint"].split(":")
        job_type = JobType.from_str(entrypoint_dict.get("type", "unknown"))
        entry_point = cls(job_type=job_type, func_or_module=function_name)
        entry_point.module_name = module_name
        entry_point.path_to_repo = (
            entrypoint_dict["image"]["path_to_repo"]
            if isinstance(entrypoint_dict["image"], dict)
            else entrypoint_dict["image"].path_to_repo
        )
        entry_point.path_to_module = (
            entrypoint_dict["image"]["path_to_module"]
            if isinstance(entrypoint_dict["image"], dict)
            else entrypoint_dict["image"].path_to_module
        )
        return entry_point

    def __repr__(self):
        return f"EntryPoint({self.function_name} from {self.module_name}, type={self.type})"

    def __str__(self):
        return f"{self.module_name}:{self.function_name}"


class Job:
    """Base class for all Datatailr compute processes (App, Service, ExcelAddin, and workflows).

    A Job encapsulates the container image configuration, resource requirements,
    access control, and deployment metadata needed to schedule and run work on
    the Datatailr platform.  In most cases you should use the concrete
    subclasses (``App``, ``Service``, ``ExcelAddin``) or the ``@workflow``
    decorator rather than instantiating ``Job`` directly.
    """

    def __init__(
        self,
        name: str,
        environment: Optional[Environment] = Environment.DEV,
        image: Optional[Image] = None,
        run_as: Optional[Union[str, User]] = None,
        resources: Resources | None = None,
        acl: Optional[ACL] = None,
        python_version: str = "auto",
        python_requirements: str | List[str] = "",
        build_script_pre: str = "",
        build_script_post: str = "",
        env_vars: Dict[str, str | int | float | bool] | None = None,
        type: Optional[JobType] = JobType.UNKNOWN,
        entrypoint: Optional[EntryPoint] = None,
        get_existing: bool = False,
        app_section: str = "",
    ):
        """Initialize a Job.

        Args:
            name: Display name for the job.
            environment: Target environment (dev / pre / prod).
            image: Pre-configured container Image.  When ``None``, an image is
                built from *python_version*, *python_requirements*, and the
                build scripts.
            run_as: User or username under which the job runs.  Defaults to the
                currently signed-in user.
            resources: CPU and memory resources for the container.
            acl: Access control list.  Defaults to standard permissions for the
                current user.
            python_version: Python version for the container image.
            python_requirements: Python dependencies (see ``Image``).
            build_script_pre: Dockerfile commands before pip install.
            build_script_post: Dockerfile commands after pip install.
            env_vars: Environment variables passed to the container.
            type: The kind of job (workflow, service, app, excel).
            entrypoint: The entry point that starts the job.
            get_existing: If ``True``, load an existing job with the same
                name and update it instead of creating a new definition.
            app_section: The section to which the app belongs. If not provided, the app will be assigned to the default section. This affects the app launcher page.
        """
        if environment is None:
            environment = Environment.DEV

        if get_existing:
            existing_job = self.__get_existing__(name, environment, get_tasks=True)
            if existing_job:
                self.from_dict(existing_job)
                return

        if run_as is None:
            run_as = User.signed_user()
        if environment is None:
            environment = Environment.DEV
        elif isinstance(environment, str):
            environment = Environment(environment.lower())
        if isinstance(environment, str):
            environment = Environment(environment)
        self.acl = acl
        self.environment = environment
        self.name = name
        self.run_as = run_as
        self.resources = resources or Resources()
        if image is None:
            image = Image(
                python_version=python_version,
                python_requirements=python_requirements,
                build_script_pre=build_script_pre,
                build_script_post=build_script_post,
            )
            if entrypoint is not None:
                image.path_to_repo = entrypoint.path_to_repo
                image.path_to_module = entrypoint.path_to_module
        self.image = image
        self.type = type if entrypoint is None else entrypoint.type
        self.entrypoint = entrypoint
        self.app_section = app_section
        self.__id = str(uuid.uuid4())
        self.__env_vars = env_vars or {}
        self.__find_missing_python_packages()
        self.__runs: List[dict] = []
        self.__internal_name = None
        self.schedule = None
        self.python_requirements = ""
        self.build_script_pre = ""
        self.build_script_post = ""

    def __find_missing_python_packages(self) -> None:
        """
        Compare the imports detected in the entry point code with the packages listed in python_requirements.
        Raise a warning if there are any imported packages that are not listed in python_requirements.
        """
        if (
            self.type == JobType.WORKFLOW
            and hasattr(self, "_Workflow__tasks")
            and len(self._Workflow__tasks) > 0
        ):
            detected_imports = self._Workflow__tasks[0].entrypoint.detected_imports
            path_to_module = self._Workflow__tasks[0].entrypoint.path_to_module
        elif self.entrypoint:
            detected_imports = self.entrypoint.detected_imports
            path_to_module = self.entrypoint.path_to_module
        else:
            return
        declared_packages = set()
        if self.image.python_requirements:
            if isinstance(self.image.python_requirements, str):
                requirements = self.image.python_requirements.splitlines()
            else:
                requirements = self.image.python_requirements
            pattern = r"^[a-zA-Z0-9\-_.]+"
            for req in requirements:
                pkg = re.match(pattern, req)
                if pkg:
                    normalized_req = re.sub(r"[-_.]+", "-", pkg.group(0).lower())
                    declared_packages.add(normalized_req)

        missing_packages = set()
        dist_map = _cached_packages_distributions()

        included_packages = (
            declared_packages.union(
                {re.sub(r"[-_.]+", "-", x.lower()) for x in sys.builtin_module_names}
            )
            .union(
                {
                    re.sub(r"[-_.]+", "-", x.lower())
                    for x in getattr(sys, "stdlib_module_names", ())
                }
            )
            .union({"datatailr", "dt"})
        )
        for top_level_pkg in detected_imports:
            # Get the PyPI distribution name that provides this module, falling back to the module name
            dist_pkg = dist_map.get(top_level_pkg, [top_level_pkg])[0]
            normalized_dist = re.sub(r"[-_.]+", "-", dist_pkg.lower())

            if (
                normalized_dist not in included_packages
                and top_level_pkg not in path_to_module
            ):
                missing_packages.add(dist_pkg)
        if missing_packages:
            if (
                self.type == JobType.WORKFLOW
                and hasattr(self, "_Workflow__local_run")
                and self._Workflow__local_run
            ):
                return
            print(
                RED(
                    "WARNING: The following packages are imported in the code but not listed in "
                    "python_requirements: "
                    f"{sorted(missing_packages)}"
                )
            )

    @property
    def _internal_name(self) -> str:
        """
        Internal normalized name for the job.
        """
        if self.__internal_name is not None:
            return self.__internal_name
        return normalize_name(self.name)

    @property
    def id(self) -> str:
        """
        Unique identifier for the job.
        """
        return self.__id

    @classmethod
    def __get_existing__(
        cls, job_name: str, environment: Environment, get_tasks: bool = False
    ) -> Optional[dict]:
        """
        Retrieve an existing job instance from the Datatailr platform.
        Based on the job name and environment.
        """
        job_name = normalize_name(job_name)
        job_list = __client__.get(
            job_name, environment=environment, show_children=get_tasks
        )
        if not isinstance(job_list, list) or len(job_list) == 0:
            raise TaskError(
                f"Job with name '{job_name}' does not exist in environment '{environment}'."
            )
        if len(job_list) > 1:
            raise TaskError(
                f"Multiple jobs found with name '{job_name}' in environment '{environment}'."
            )
        return job_list[0]

    def __repr__(self):
        return (
            f"Job(name={self.name}, environment={self.environment}, "
            f"run_as={self.run_as}, resources={self.resources}, "
            f"acl={self.acl}, type={self.type}, "
            f"entrypoint={self.entrypoint}, image={self.image})"
        )

    def to_dict(self):
        """
        Convert the Job instance to a dictionary representation.
        """
        job_dict = {
            "environment": str(self.environment),
            "image": self.image.to_dict(),
            "type": str(self.type) if self.type else None,
            "display_name": self.name,
            "name": self._internal_name,
            "run_as": self.run_as.name
            if isinstance(self.run_as, User)
            else self.run_as,
            "memory": self.resources.memory,
            "num_cpus": self.resources.cpu,
            "app_section": self.app_section if hasattr(self, "app_section") else "",
        }
        if self.acl is not None:
            job_dict["acl"] = self.acl.to_dict()
        if self.type == JobType.EXCEL:
            if "DATATAILR_LOCAL" not in self.__env_vars:
                self.__env_vars.update({"DATATAILR_LOCAL": "false"})
        if self.type != JobType.WORKFLOW:
            job_dict["entrypoint"] = str(self.entrypoint) if self.entrypoint else None
            job_dict["env"] = dict_to_env_vars(self.__env_vars)
        return job_dict

    def from_dict(self, job_dict: dict):
        """Populate this Job instance from a dictionary (e.g. a platform API response).

        Args:
            job_dict: A dictionary containing the serialized job data.
        """
        self.name = job_dict["display_name"]
        self.__internal_name = job_dict["name"]
        environment = job_dict.get("environment", "dev")
        environment = Environment(environment.lower())
        self.environment = environment
        job_type = JobType.from_str(job_dict.get("type", "unknown"))
        if job_type != JobType.WORKFLOW:
            image_dict = job_dict["image"]
        else:
            first_child_name = job_dict["children_full"][0]["name"]
            first_child_job = self.__get_existing__(
                first_child_name, self.environment.value, get_tasks=False
            )
            if isinstance(first_child_job, dict):
                image_dict = first_child_job["image"]
            else:
                raise TypeError(
                    f"Expected a dictionary for task definition, got {type(first_child_job)}"
                )
        self.image = Image().from_dict(image_dict)

        user = job_dict["run_as"]
        user = User(user.lower())
        self.run_as = user

        self.resources = Resources.from_job_dict(job_dict)
        acl = job_dict.get("acl", None)
        if acl is not None:
            acl = ACL.from_dict(acl)
        self.acl = acl
        self.schedule = job_dict.get("schedule", None)
        self.entrypoint = job_dict.get("entrypoint", None)
        self.python_requirements = job_dict.get("python_requirements", "")
        self.build_script_pre = job_dict.get("build_script_pre", "")
        self.build_script_post = job_dict.get("build_script_post", "")
        self.state = job_dict["state"]
        self.create_time = datetime.fromtimestamp(job_dict["create_time"] * 1e-6)
        self.version = job_dict["version"]
        self.__id = job_dict["id"]
        self.type = job_type

    def to_json(self):
        """
        Convert the Job instance to a JSON string representation.
        """
        return json.dumps(self.to_dict())

    def verify_repo_is_ready(self) -> Tuple[str, str]:
        """
        Verify if the repository is ready for job execution.
        The check consists of two parts:
        1. Check if there are uncommitted changes in the repository.
        2. Check if the local commit matches the remote HEAD (the repo is synced with the remote).
        Returns a tuple of (branch: str, commit_hash: str).
        """
        path_to_repo = self.image.path_to_repo or "."
        branch_name, local_commit, return_code = "unknown", "unknown", None
        try:
            local_commit = run_shell_command(
                f"cd {path_to_repo} && git rev-parse HEAD"
            )[0]
            branch_name = run_shell_command(
                f"cd {path_to_repo} && git rev-parse --abbrev-ref HEAD"
            )[0]

            if (
                os.getenv("DATATAILR_ALLOW_UNSAFE_SCHEDULING", "false").lower()
                == "true"
            ):
                return branch_name, local_commit

            return_code = run_shell_command(
                f"cd {path_to_repo} && git status --porcelain"
            )
        except RuntimeError as e:
            if "git: not found" in str(e):
                logger.warning(
                    "Git is not installed or not found in PATH. Repository validation is not possible."
                )
            elif "not a git repository" in str(e):
                print(
                    YELLOW(
                        "The code repository is not a git repository. Skipping repository validation."
                    )
                )
            else:
                raise RepoValidationError(
                    f"Error accessing git repository at {path_to_repo}: {e}"
                ) from e

        is_committed = (
            return_code is not None and return_code[1] == 0 and len(return_code[0]) == 0
        )

        if not is_committed:
            raise RepoValidationError("Uncommited changes detected in the repository")

        try:
            remote_commit = run_shell_command(
                f"cd {path_to_repo} && git ls-remote origin HEAD"
            )[0].split("\t")[0]

            if local_commit != remote_commit:
                raise RepoValidationError(
                    "Please sync your local repository with the remote before running the job"
                )
        except RuntimeError as e:
            raise RepoValidationError(
                f"Error accessing remote git repository: {e}"
            ) from e

        return branch_name, local_commit

    def __prepare__(self) -> str:
        try:
            branch_name, local_commit = self.verify_repo_is_ready()
        except RepoValidationError as e:
            branch_name, local_commit = "unknown", "unknown"
            type_name = self.type.display_name() if self.type is not None else "Job"
            message = f"Repository validation failed: {e}. The {type_name} will be deployed without repo validation. It will not be possible to promote this {type_name} to other environments."
            print(YELLOW(message))
        self.image.update(
            branch_name=branch_name,
            commit_hash=local_commit,
        )
        with tempfile.NamedTemporaryFile(delete=False, suffix=".json") as temp_file:
            temp_file.write(self.to_json().encode())
        return temp_file.name

    def get_schedule_args(self) -> dict:
        """
        Returns additional arguments for scheduling the job.
        Override or extend this method as needed.
        """
        return {}

    def __update_info__(self):
        job_info = __client__.get(
            self._internal_name, environment=self.environment, show_children=False
        )
        if len(job_info) != 1:
            raise TaskError(
                f"Expected one job info for job '{self.name}', got {len(job_info)}."
            )
        job_dict = job_info[0]
        if job_dict and "version" in job_dict:
            self.state = job_dict["state"]
            self.create_time = datetime.fromtimestamp(job_dict["create_time"] * 1e-6)
            self.version = job_dict["version"]
            self.__id = job_dict["id"]

    def __run_command__(self, command: str) -> Tuple[bool, str]:
        """
        Run a command in the context of the job.
        This is used to execute the job's entry point.
        """
        if not is_dt_installed() and not is_remote_cli_dt_installed():
            raise NotImplementedError(
                "DataTailr is not installed. Please install DataTailr to run this job."
            )
        try:
            temp_file_name = self.__prepare__()
            action = {"run": "Running", "save": "Saving", "start": "Starting"}.get(
                command, "Processing"
            )
            if command == "run":
                result = __client__.run(
                    f"file://{temp_file_name}", **self.get_schedule_args()
                )
            elif command == "save":
                result = __client__.save(
                    f"file://{temp_file_name}", **self.get_schedule_args()
                )
            elif command == "start":
                result = __client__.start(self.name, environment=self.environment)
            else:
                raise ValueError(f"Unknown command: {command}")
            os.remove(temp_file_name)
        except Exception as e:
            logger.error(f"Error running command '{command}': {e}")
            return False, str(e)
        self.__update_info__()
        base_url = get_base_url()
        job_url = (
            f"{base_url}/job-details/{self._internal_name}/{self.type}/{self.environment}/{self.version}?tab=definition"
            if base_url
            else "the DataTailr platform"
        )
        job_type_name = self.type.display_name() if self.type is not None else "Job"
        print(
            GREEN(
                f"The {job_type_name} can be referenced in the CLI as '{self._internal_name}'. More details on: {BLUE(BOLD(job_url)) if base_url else 'the DataTailr platform'}."
            )
        )
        return True, result

    def save(self) -> Tuple[bool, str]:
        """
        Save the job to the Datatailr platform.
        If the job already exists, it will be updated.
        The repository state is verified and an image is prepared for execution.
        """
        return self.__run_command__("save")

    def run(self) -> Tuple[bool, str]:
        """
        Run the job.
        This is equivalent to running job.save() and then job.start().
        """
        return self.__run_command__("run")

    def start(self) -> Tuple[bool, str]:
        """
        Start the job. This will start the job execution on a schedule for workflows if a schedule was specified.
        For other types of jobs and for workflows without a schedule the job will be run immediately.
        """
        return self.__run_command__("start")

    def promote(
        self,
        from_environment: Optional[Environment | str] = None,
        version: Optional[str | int] = None,
    ) -> Tuple[bool, str]:
        """
        Promote the job to the next environment.
        This method is used to promote a version of the job from one environment to the next one.
        If none of the environments to promote from are specified, it defaults to promote from all environments.

        :param from_environment: The environment to promote from. If None, it will promote from all environments.
        :param version: The version to promote. If None, it will promote the latest version.
        :return: A tuple of (success: bool, message: str).

        >>> from datatailr import Environment
        >>> from datatailr.scheduler import Job
        >>> job = Job(name="my_job", environment=Environment.DEV)
        >>> job.promote(from_environment=Environment.DEV, version=3)

        This will promote version 3 of the job from the DEV environment to the next environment (PRE).
        """
        promote_kwargs = {}
        if version is not None:
            promote_kwargs["version"] = str(version)
        if from_environment is not None:
            if isinstance(from_environment, str):
                from_environment = Environment(from_environment.lower())
            promote_kwargs["environment"] = str(from_environment)
        try:
            __client__.promote(self._internal_name, **promote_kwargs)
            return True, f"Job '{self.name}' promoted successfully."
        except Exception as e:
            logger.error(f"Error promoting job '{self.name}': {e}")
            return False, str(e)

    def versions(self, environment: Optional[Environment] = None) -> list[str] | None:
        """
        List all versions of the job in the specified environment
        If no environment is specified, it lists versions across all environments.
        """
        command_kwargs = {}
        if environment is not None:
            command_kwargs["environment"] = str(environment)
        return __client__.versions(self._internal_name, **command_kwargs)

    def _process_run_data(self, data: list[dict]) -> List[dict]:
        """
        Process raw run data into a list of mother-job dictionaries.

        Runs are aggregated by run ID. For each run ID the mother job (the
        non-task entry) becomes the top-level dictionary and all associated
        task runs are collected under its ``'tasks'`` key.

        Returns a list where each element has the structure::

            {
                'name': <str>,                   # mother job display name
                'state': <str>,
                'start_time': <int>,
                'end_time': <int>,
                'submit_time': <int>,
                'scheduled_time': <int>,
                'error_message': <str>,
                'start_provisioning_time': <int>,
                'job_version': <int>,
                'tasks': [                       # child task runs
                    {
                        'name': <str>,           # task display name
                        'task_id': <int>,
                        'dependencies': <set>,
                        'state': <str>,
                        'start_time': <int>,
                        'end_time': <int>,
                        'submit_time': <int>,
                        'error_message': <str>,
                        'start_provisioning_time': <int>,
                    },
                    ...
                ]
            }
        """

        def is_task(run: dict) -> bool:
            name = run["name"]
            task_number = name.split("-")[-1].split("_")[0]
            return (
                hasattr(self, "get_task_name_from_id")
                and hasattr(self, "get_task_by_name")
                and task_number.isdigit()
            )

        def strip_common_fields(run: dict) -> None:
            run["run_id"] = run.pop("id", None)
            run["run_name"] = run.pop("job_name", None)
            run.pop("type", None)
            run.pop("environment", None)
            run.pop("priority", None)
            run.pop("job_id", None)
            run.pop("tried_to_start_time", None)

        def parse_datetime_fields(run: dict) -> None:
            for field in [
                "start_time",
                "end_time",
                "submit_time",
                "scheduled_time",
                "start_provisioning_time",
            ]:
                if field in run:
                    time = run[field]
                    if time is not None:
                        run[field] = datetime.fromtimestamp(time * 1e-6)

        def enrich_task(run: dict) -> dict:
            if not hasattr(self, "get_task_name_from_id") or not hasattr(
                self, "get_task_by_name"
            ):
                return run
            name = run["name"]
            task_number = int(name.split("-")[-1].split("_")[0])
            task_name = self.get_task_name_from_id(task_number)
            run["task_id"] = task_number
            run["name"] = task_name
            run["dependencies"] = list(self.get_task_by_name(task_name).dependencies)
            run.pop("scheduled_time", None)
            run.pop("job_version", None)
            return run

        mothers: dict[int, dict] = {}
        tasks: dict[int, list[dict]] = defaultdict(list)

        for run in data:
            run_id = run["id"]
            strip_common_fields(run)
            parse_datetime_fields(run)
            if is_task(run):
                run.pop("run_id", None)
                run.pop("run_name", None)
                tasks[run_id].append(enrich_task(run))
            else:
                run["name"] = self.name
                mothers[run_id] = run

        result: List[dict] = []
        for run_id, mother in mothers.items():
            mother["tasks"] = tasks.pop(run_id, [])
            result.append(mother)

        for run_id, task_list in tasks.items():
            result.append({"name": self.name, "tasks": task_list})

        return result

    def runs(self, refresh: bool = False) -> list[dict]:
        """
        List all runs of the job.

        Returns a list of mother-job dictionaries. Each dictionary
        contains the top-level run metadata and a ``'tasks'`` key
        holding a list of child task run dictionaries.
        See :meth:`_process_run_data` for the full structure.

        :param refresh: If True, it will refresh the cached runs.
        :return: A list of mother-job run dictionaries.
        """
        if not hasattr(self, "__runs"):
            self.__runs = []
        if not refresh and len(self.__runs) > 0:
            return self.__runs
        if not isinstance(self.type, JobType):
            return self.__runs
        _filter = f"type={self.type.value},environment={self.environment.value}"
        data = __client__.runs(filter=_filter)
        data = [run for run in data if run["job_name"].startswith(self._internal_name)]
        runs = self._process_run_data(data)
        return runs
