# Copyright 2015-2026 Earth Sciences Department, BSC-CNS
#
# This file is part of Autosubmit.
#
# Autosubmit is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Autosubmit is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Autosubmit.  If not, see <http://www.gnu.org/licenses/>.


import datetime
import json
import locale
import multiprocessing
import os
import random
import re
import tarfile
import time
from contextlib import suppress
from datetime import timedelta
from pathlib import Path
from threading import Thread
from typing import Optional, TYPE_CHECKING

from bscearth.utils.date import sum_str_hours

from autosubmit.job.job import Job
from autosubmit.job.job_common import Status
from autosubmit.log.log import Log, AutosubmitCritical, AutosubmitError

if TYPE_CHECKING:
    from autosubmit.config.configcommon import AutosubmitConfig

Log.get_logger("Autosubmit")


def threaded(fn):
    def wrapper(*args, **kwargs):
        thread = Thread(target=fn, args=args, kwargs=kwargs)
        thread.name = "data_processing"
        thread.start()
        return thread
    return wrapper


def jobs_in_wrapper_str(as_conf: 'AutosubmitConfig', current_wrapper: str) -> list[str]:
    """Transform to string with _ separator the jobs in the wrapper."""
    return "_".join(as_conf.experiment_data["WRAPPERS"].get(current_wrapper, {}).get("JOBS_IN_WRAPPER", []))


class JobPackageBase(object):
    """
    Class to manage the package of jobs to be submitted by autosubmit
    """

    def __init__(self, jobs: list[Job]):
        self._job_scripts = None
        self.nodes = ""
        self._common_script = None
        self._jobs = jobs
        self._expid = jobs[0].expid
        self.hold = False
        self.export = jobs[0].export
        self.x11 = jobs[0].x11
        self.het = dict()
        self._num_processors = '0'
        self._threads = '0'
        try:
            self._tmp_path = jobs[0]._tmp_path
            self._platform = jobs[0]._platform
            self._custom_directives = set()
            for job in jobs:
                if job._platform.name != self._platform.name or job.platform is None:
                    raise Exception('Only one valid platform per package')
        except IndexError:
            raise Exception('No jobs given')

    def __len__(self):
        return self._jobs.__len__()

    @property
    def jobs(self) -> list[Job]:
        """
        Returns the jobs

        :return: jobs
        """
        return self._jobs

    @property
    def platform(self):
        """
        Returns the platform

        :return: platform
        :rtype: Platform
        """
        return self._platform

    @threaded
    def check_scripts(self, jobs: list[Job], configuration: 'AutosubmitConfig', only_generate: bool = False):
        for job in jobs:
            if only_generate and not os.path.exists(os.path.join(configuration.get_project_dir(), job.file)):
                break
            elif not os.path.exists(os.path.join(configuration.get_project_dir(), job.file)):
                if configuration.get_project_type().lower() != "none" and len(configuration.get_project_type()) > 0:
                    raise AutosubmitCritical(f"Job script:{job.file} does not exists", 7014)
            if not job.check_script(configuration, show_logs=job.check_warnings):
                Log.warning(
                    f'Script {job.name} has some empty variables. An empty value has substituted these variables')
            else:
                Log.result(f"Script {job.name} OK")
            # looking for directives on jobs
            self._custom_directives = self._custom_directives | set(job.custom_directives)

    @threaded
    def _create_scripts_threaded(self, jobs: list[Job], configuration: 'AutosubmitConfig'):
        for i in range(0, len(jobs)):
            self._job_scripts[jobs[i].name] = jobs[i].create_script(configuration)

    def _create_common_script(self, filename: str = ""):
        pass  # pragma: no cover

    def check_job_files_exists(self, configuration: 'AutosubmitConfig', only_generate: bool) -> None:
        """ Check that all job files exist in the project directory.
        :param configuration: Autosubmit basic configuration.
        :type configuration: AutosubmitConfig
        :param only_generate: True to only generate scripts without submitting.
        :type only_generate: bool
        :return: None.
        """

        if not configuration.get_project_type() or (configuration.get_project_type() and str(configuration.get_project_type()).lower() == "none"):
            return

        Log.debug("Checking job files existence")
        project_dir = Path(configuration.get_project_dir())
        # This self.jobs is actually only jobs in the package not all the loaded job_list.
        for job in self.jobs:
            # When SCRIPT is defined, FILE is ignored.
            if not job.script and job.file and not (project_dir / job.file).exists():
                if not only_generate:
                    raise AutosubmitCritical(f"[section:{job.section}]: Job script:{job.file} does not exists", 7014)
                Log.warning(f"[section:{job.section}]: Job script:{job.file} does not exists, skipping check")

            for additional_file in job.additional_files:
                if not additional_file or not (project_dir / additional_file).exists():
                    if not only_generate:
                        raise AutosubmitCritical(f"[section:{job.section}]: Additional file:{additional_file} does not exists", 7014)
                    Log.warning(f"[section:{job.section}]: Additional file:{additional_file} does not exists, skipping check")
                    

    def build_scripts(self, configuration: 'AutosubmitConfig') -> None:
        """Submit jobs one by one without using threads.

        :param configuration: Autosubmit basic configuration.
        :type configuration: AutosubmitConfig
        :return: None.
        """
        Log.debug("Checking Scripts")
        for job in self.jobs:
            if (job.file or job.script) and not job.check_script(configuration, show_logs=job.check_warnings):
                Log.warning(
                    f"Script {job.name} has some empty variables. An empty value has substituted these variables"
                )

            self._custom_directives |= set(getattr(job, "custom_directives", []))

        Log.debug("Building scripts")
        self._create_scripts(configuration)

    def submit(self, configuration: 'AutosubmitConfig', parameters: Optional[dict] = None,
               only_generate: bool = False, hold: bool = False):
        """
        :param hold:
        :param configuration: Autosubmit basic configuration
        :type configuration: AutosubmitConfig object
        :param parameters; Parameters from job list
        :type parameters: JobList parameters
        :param only_generate: True if coming from generate_scripts_andor_wrappers().
            If true, only generates scripts; otherwise, submits.
        :type only_generate: Boolean
        """
        if parameters is None:
            parameters = {}
        thread_number = multiprocessing.cpu_count()
        if len(self.jobs) > 2500:
            thread_number = thread_number * 2
        elif len(self.jobs) > 5000:
            thread_number = thread_number * 3
        elif len(self.jobs) > 7500:
            thread_number = thread_number * 4
        elif len(self.jobs) > 10000:
            thread_number = thread_number * 5
        chunksize = int((len(self.jobs) + thread_number - 1) / thread_number)
        try:
            if len(self.jobs) < thread_number or str(
                    configuration.experiment_data.get("CONFIG", {}).get("ENABLE_WRAPPER_THREADS",
                                                                        "False")).lower() == "false":
                self.check_job_files_exists(configuration, only_generate)
                self.build_scripts(configuration)
            else:
                # TODO: Not related to this PR, but this probably is not working correctly (nobody is using it)
                lhandle = list()
                for i in range(0, len(self.jobs), chunksize):
                    lhandle.append(
                        self.check_scripts(self.jobs[i:i + chunksize], configuration, parameters, only_generate, hold))
                for dataThread in lhandle:
                    dataThread.join()
                for i in range(0, len(self.jobs), chunksize):
                    Log.debug("Creating Scripts")
                    lhandle.append(self._create_scripts_threaded(self.jobs[i:i + chunksize], configuration))
                for dataThread in lhandle:
                    dataThread.join()
                self._common_script = self._create_common_script()
        except AutosubmitCritical:
            raise
        except BaseException:
            raise
        try:
            if not only_generate:
                Log.debug("Sending Files")
                self._send_files()
                Log.debug("Submitting")
                self._do_submission(hold=hold)
        except AutosubmitError:
            raise
        except AutosubmitCritical:
            raise
        except BaseException as e:
            raise AutosubmitCritical(f"Error while submitting jobs: {e}", 7013)

    def _create_scripts(self, configuration: 'AutosubmitConfig'):
        raise Exception('Not implemented')

    def _send_files(self):
        """ Send local files to the platform. """

    def _do_submission(self, job_scripts=None, hold: bool = False):
        """ Submit package to the platform. """
        pass  # pragma: no cover

    def process_jobs_to_submit(self, job_id: str, hold: bool = False) -> None:
        for i, job in enumerate(self.jobs):
            job.hold = hold
            job.id = str(job_id)
            job.status = Status.SUBMITTED
            Log.result(
                f"Job: {job.name} submitted with job_id: {job.id.strip()} and workflow commit: {job.workflow_commit}")
            if hasattr(self, "name"):
                # TODO change this check for a property that checks if it is a wrapper or not, the same change has to be done in other parts of the code
                job.wrapper_name = self.name


class JobPackageSimple(JobPackageBase):
    """
    Class to manage a group of simple jobs, not packaged, to be submitted by autosubmit
    """

    def __init__(self, jobs: list[Job]):
        super(JobPackageSimple, self).__init__(jobs)
        self._job_scripts = {}
        self.export = jobs[0].export
        # TODO: This should be possible, but it crashes across the code.
        #  Add a property that defines what is a package with wrappers
        # self.name = "simple_package"

    def _create_scripts(self, configuration: 'AutosubmitConfig'):
        for job in self.jobs:
            self._job_scripts[job.name] = job.create_script(configuration)

    def _send_files(self):
        # TODO: Add tests when the slurm container is available.
        for job in self.jobs:
            self.platform.send_file(self._job_scripts[job.name])
            for f in job.additional_files:
                real_name = job.construct_real_additional_file_name(f)
                self.platform.send_file(real_name)

    def _do_submission(self, job_scripts: dict[str, str] = "", hold: bool = False) -> None:
        """
        Submits jobs to the platform, cleans previous run logs and stats files and updates job status.

        :param job_scripts: Dictionary of job scripts, defaults to an empty string.
        :param hold: If True, the job won't immediately start, defaults to False.
        """
        if len(job_scripts) == 0:
            job_scripts = self._job_scripts
        for job in self.jobs:
            # This sets the log names but also the submission time for non-vertical wrapped jobs.
            job.update_local_logs()
            # Clean previous run logs on local
            log_completed = os.path.join(self._tmp_path, job.name + '_COMPLETED')
            log_stat = os.path.join(self._tmp_path, job.name + '_STAT')
            if os.path.exists(log_completed):
                os.remove(log_completed)
            if os.path.exists(log_stat):
                os.remove(log_stat)
            self.platform.remove_stat_file(job)
            self.platform.remove_completed_file(job.name)

            # Submit job to the platform
            job.id = self.platform.submit_job(job, job_scripts[job.name], hold=hold, export=self.export)
            if job.id is None or not job.id:
                continue
            Log.info(f"{job.name} submitted")
            job.status = Status.SUBMITTED
            job.wrapper_name = job.name
            job.id = str(job.id)


class JobPackageSimpleWrapped(JobPackageSimple):
    """
    Class to manage a group of simple wrapped jobs, not packaged, to be submitted by autosubmit
    """

    def __init__(self, jobs: list[Job]):
        super(JobPackageSimpleWrapped, self).__init__(jobs)
        self._job_wrapped_scripts = {}

    def _create_scripts(self, configuration: 'AutosubmitConfig'):
        super(JobPackageSimpleWrapped, self)._create_scripts(configuration)
        for job in self.jobs:
            self._job_wrapped_scripts[job.name] = job.create_wrapped_script(configuration)

    def _send_files(self):
        super(JobPackageSimpleWrapped, self)._send_files()
        for job in self.jobs:
            self.platform.send_file(self._job_wrapped_scripts[job.name])

    def _do_submission(self, job_scripts=None, hold=False):
        if job_scripts is None or not job_scripts:
            job_scripts = self._job_wrapped_scripts
        super(JobPackageSimpleWrapped, self)._do_submission(job_scripts, hold=hold)


class JobPackageArray(JobPackageBase):
    """
    Class to manage an array-based package of jobs to be submitted by autosubmit
    """

    def __init__(self, jobs: list[Job]):
        self.name = None
        self._job_inputs = {}
        self._job_scripts = {}
        self._common_script = None
        self._array_size_id = "[1-" + str(len(jobs)) + "]"
        self._wallclock = '00:00'
        self._num_processors = '0'
        self._threads = '0'

        for job in jobs:
            if job.wallclock > self._wallclock:
                self._wallclock = job.wallclock
            if job.processors > self._num_processors:
                self._num_processors = job.processors
        super(JobPackageArray, self).__init__(jobs)

    def _create_scripts(self, configuration: 'AutosubmitConfig'):
        timestamp = str(int(time.time()))
        for i in range(0, len(self.jobs)):
            self._job_scripts[self.jobs[i].name] = self.jobs[i].create_script(configuration)
            self._job_inputs[self.jobs[i].name] = self._create_i_input(timestamp, i)
        self._common_script = self._create_common_script(timestamp)

    def _create_i_input(self, filename, index):
        filename += f'.{index}'
        input_content = self._job_scripts[self.jobs[index].name]
        open(os.path.join(self._tmp_path, filename), 'wb').write(input_content)
        os.chmod(os.path.join(self._tmp_path, filename), 0o755)
        return filename

    def _create_common_script(self, filename=""):
        script_content = self.platform.header.array_header(filename, self._array_size_id, self._wallclock,
                                                           self._num_processors,
                                                           directives=self.platform.custom_directives)
        filename += '.cmd'
        open(os.path.join(self._tmp_path, filename), 'wb').write(script_content)
        os.chmod(os.path.join(self._tmp_path, filename), 0o755)
        return filename

    def _send_files(self):
        for job in self.jobs:
            self.platform.send_file(self._job_scripts[job.name])
            self.platform.send_file(self._job_inputs[job.name])
        self.platform.send_file(self._common_script)

    def _do_submission(self, job_scripts: dict[str, str] = None, hold: bool = False) -> None:
        """
        Submits jobs to the platform, cleans previous run logs, and updates job status.

        :param job_scripts: Dictionary of job scripts, defaults to None.
        :param hold: If True, holds the job submission, defaults to False.
        :type hold: bool
        """
        for job in self.jobs:
            job.update_local_logs()
            self.platform.remove_stat_file(job)
            self.platform.remove_completed_file(job.name)

        package_id = self.platform.submit_job(None, self._common_script, hold=hold, export=self.export)

        if package_id is None or not package_id:  # platforms with a submit.cmd
            return

        for i in range(0, len(self.jobs)):  # platforms without a submit.cmd
            Log.info(f"{self.jobs[i].name} submitted")
            self.jobs[i].id = str(package_id) + f'[{i}]'
            self.jobs[i].status = Status.SUBMITTED
            # Identify to which wrapper this job belongs once it is in the recovery queue
            self.jobs[i].wrapper_name = self.name


class JobPackageThread(JobPackageBase):
    """
    Class to manage a thread-based package of jobs to be submitted by autosubmit

    :param dependency: Name of potential dependency
    :type dependency: String
    """
    FILE_PREFIX = 'ASThread'

    def __init__(self, jobs: list[Job], dependency=None, jobs_resources: Optional[dict] = None,
                 method: str = 'ASThread', configuration: Optional['AutosubmitConfig'] = None,
                 wrapper_section: str = "WRAPPERS", wrapper_info: Optional[list] = None):
        """
        :param dependency: Dependency
        :type dependency: String
        :param jobs_resources: Resources to be used by the jobs, if any
        :type jobs_resources: Dictionary
        :param method: Method to be used to submit the jobs, ASThread by default
        :type method: String
        :param configuration: Autosubmit configuration
        :type configuration: Autosubmitconfigparser instance
        """
        # This function is called from the JobPackageThread constructor
        # and from the JobPackageThread.create_scripts function
        # It is in charge of merging ( switch ) the wrapper info by checking if the value is defined by
        # the user in the wrapper section, current wrapper section, job or platform in that order.
        # Some variables are calculated in further functions, like num_processors and wallclock.
        # These variables can only be present in the wrapper itself
        super(JobPackageThread, self).__init__(jobs)
        if jobs_resources is None:
            jobs_resources = {}
        if wrapper_info is None:
            wrapper_info = []
        if len(wrapper_info) > 0:
            self.wrapper_type = wrapper_info[0]
            self.wrapper_policy = wrapper_info[1]
            self.wrapper_method = wrapper_info[2]
            self.jobs_in_wrapper = wrapper_info[3]
            self.extensible_wallclock = wrapper_info[4]
        else:
            self.wrapper_type = None
            self.wrapper_policy = None
            self.wrapper_method = None
            self.jobs_in_wrapper = None
            self.extensible_wallclock = 0
        self._job_scripts = {}
        # Seems like this one is not used at all in the class
        self._job_dependency = dependency
        self._common_script = None
        self.executable = None

        self._wallclock = '00:00'
        # depends on the type of wrapper

        self._jobs_resources = jobs_resources
        self._wrapper_factory = self.platform.wrapper
        self.current_wrapper_section = wrapper_section
        self.inner_retrials = 0
        if not hasattr(self, "_num_processors"):
            self._num_processors = '0'
        self.parameters = dict()
        self.nodes = jobs[0].nodes if not self.nodes else self.nodes
        self.queue = jobs[0].queue
        self.parameters["CURRENT_QUEUE"] = self.queue
        self.partition = jobs[0].partition
        self.tasks = jobs[0].tasks
        self.exclusive = jobs[0].exclusive
        self.custom_directives = jobs[0].custom_directives
        self.wallclock = '00:00'
        self.reservation = jobs[0].reservation
        if configuration is not None:
            self.inner_retrials = configuration.experiment_data["WRAPPERS"].get(self.current_wrapper_section,
                                                                                {}).get("RETRIALS",
                                                                                        self.jobs[0].retrials)
            if self.inner_retrials == 0:
                self.inner_retrials = configuration.experiment_data["WRAPPERS"].get(self.current_wrapper_section,
                                                                                    {}).get("INNER_RETRIALS",
                                                                                            self.jobs[0].retrials)
            for job in self.jobs:
                job.retrials = self.inner_retrials
            self.export = configuration.get_wrapper_export(
                configuration.experiment_data["WRAPPERS"][self.current_wrapper_section])
            if self.export.lower() != "none" and len(self.export) > 0:
                for job in self.jobs:
                    if job.export.lower() not in "none" and len(job.export) > 0:
                        self.export = job.export
                        break
            self.executable = configuration.experiment_data["WRAPPERS"].get(self.current_wrapper_section, {}).get(
                "EXECUTABLE", self.executable)
            if jobs[0].het.get("HETSIZE", 1) <= 1:
                wr_queue = configuration.get_wrapper_queue(
                    configuration.experiment_data["WRAPPERS"][self.current_wrapper_section])
                if wr_queue is not None and len(str(wr_queue)) > 0:
                    self.queue = wr_queue
                    self.parameters["CURRENT_QUEUE"] = wr_queue
                wr_partition = configuration.get_wrapper_partition(
                    configuration.experiment_data["WRAPPERS"][self.current_wrapper_section])
                if wr_partition and len(str(wr_partition)) > 0:
                    self.partition = wr_partition
                self.exclusive = configuration.experiment_data["WRAPPERS"].get(self.current_wrapper_section, {}).get(
                    "EXCLUSIVE", self.exclusive)
                wr_custom_directives = configuration.experiment_data["WRAPPERS"].get(self.current_wrapper_section,
                                                                                     {}).get("CUSTOM_DIRECTIVES", [])
                # parse custom_directives
                if type(wr_custom_directives) is list:
                    wr_custom_directives = json.dumps(wr_custom_directives)
                wr_custom_directives = wr_custom_directives.replace("\'", "\"").strip("[]").strip(", ")
                if wr_custom_directives == '':
                    if jobs[0].custom_directives is None:
                        jobs[0].custom_directives = ''
                    wr_custom_directives = jobs[0].custom_directives
                    if type(wr_custom_directives) is list:
                        wr_custom_directives = json.dumps(wr_custom_directives)
                    wr_custom_directives = wr_custom_directives.replace("\'", "\"").strip("[]").strip(", ")
                if wr_custom_directives != '':
                    if wr_custom_directives[0] != "\"":
                        wr_custom_directives = "\"" + wr_custom_directives
                    if wr_custom_directives[-1] != "\"":
                        wr_custom_directives = wr_custom_directives + "\""
                    wr_custom_directives = "[" + wr_custom_directives + "]"
                    wr_custom_directives = json.loads(wr_custom_directives)
                else:
                    wr_custom_directives = []
                if len(str(wr_custom_directives)) > 0:
                    self.custom_directives = wr_custom_directives
                self.tasks = configuration.experiment_data["WRAPPERS"].get(self.current_wrapper_section, {}).get(
                    "TASKS", self.tasks)
                self.nodes = configuration.experiment_data["WRAPPERS"].get(self.current_wrapper_section, {}).get(
                    "NODES", self.nodes)
                self.reservation = configuration.experiment_data["WRAPPERS"].get(self.current_wrapper_section, {}).get(
                    "RESERVATION", self.reservation)
                wr_threads = configuration.experiment_data["WRAPPERS"].get(self.current_wrapper_section, {}).get(
                    "THREADS", None)
                if wr_threads:
                    self.threads = wr_threads
                else:
                    self.threads = jobs[0].threads
        self.parameters["CURRENT_PROJ"] = self._project
        self.het = jobs[0].het

        # Memory needs more work outside this branch
        self.parameters["MEMORY"] = jobs[0].memory
        self.memory = jobs[0].memory
        self.parameters["MEMORY_PER_TASK"] = jobs[0].memory_per_task
        self.memory_per_task = jobs[0].memory_per_task
        self.parameters["NODES"] = self.nodes
        self.processors = self._num_processors
        self.parameters["RESERVATION"] = self.reservation  # have to look
        self.parameters['TASKS'] = self.tasks
        self.parameters["EXECUTABLE"] = self.executable  # have to look
        self.method = method

    @property
    def name(self):
        # FIXME: Calling this raises an exception at runtime?
        return self._name

    @property
    def _jobs_scripts(self):
        self._jobs_resources['PROCESSORS_PER_NODE'] = self.platform.processors_per_node
        jobs_scripts = []
        for job in self.jobs:
            if job.section not in self._jobs_resources:
                self._jobs_resources[job.section] = dict()
                self._jobs_resources[job.section]['PROCESSORS'] = job.processors
                self._jobs_resources[job.section]['TASKS'] = job.tasks
            with suppress(Exception):
                jobs_scripts.append(self._job_scripts[job.name])
        return jobs_scripts

    @property
    def queue(self):
        if (not str(self.nodes).isdigit() or (str(self.nodes).isdigit() and int(self.nodes) < 1)) and (
                not str(self._num_processors).isdigit() or (
                str(self._num_processors).isdigit() and int(self._num_processors) <= 1)):
            return self.platform.serial_platform.serial_queue
        else:
            return self._queue

    @queue.setter
    def queue(self, value):
        self._queue = value

    @property
    def _project(self):
        return self._platform.project

    def set_job_dependency(self, dependency):
        self._job_dependency = dependency

    def _create_scripts(self, configuration: 'AutosubmitConfig'):
        for i in range(0, len(self.jobs)):
            self._job_scripts[self.jobs[i].name] = self.jobs[i].create_script(configuration)
        self._common_script = self._create_common_script()

    def _create_common_script(self, filename: str = ""):
        lang = locale.getlocale()[1]
        if lang is None:
            lang = locale.getdefaultlocale()[1]
            if lang is None:
                lang = 'UTF-8'
        script_content = self._common_script_content()
        script_file = self.name + '.cmd'
        open(os.path.join(self._tmp_path, script_file), 'wb').write(script_content.encode(lang))
        os.chmod(os.path.join(self._tmp_path, script_file), 0o755)
        return script_file

    def _send_files(self):
        Log.debug("Check remote dir")
        self.platform.check_remote_log_dir()
        compress_type = "w"
        output_filepath = 'wrapper_scripts.tar'
        if callable(getattr(self.platform, 'remove_multiple_files')):
            filenames = str()
            for job in self.jobs:
                filenames += " " + self.platform.remote_log_dir + "/" + job.name + ".cmd"
            self.platform.remove_multiple_files(filenames)
        tar_path = os.path.join(self._tmp_path, output_filepath)
        Log.debug("Compressing multiple_files")
        with tarfile.open(tar_path, compress_type) as tar:
            for job in self.jobs:
                jfile = os.path.join(self._tmp_path, self._job_scripts[job.name])
                with open(jfile, 'rb') as f:
                    info = tar.gettarinfo(jfile, self._job_scripts[job.name])
                    tar.addfile(info, f)
        tar.close()
        os.chmod(tar_path, 0o755)
        self.platform.send_file(tar_path, check=False)
        Log.debug("Uncompress - send_command")
        self.platform.send_command(f"cd {self.platform.get_files_path()}; tar -xvf {output_filepath}")
        Log.debug("Send_file: common_script")
        self.platform.send_file(self._common_script)
        for job in self.jobs:
            for f in job.additional_files:
                real_name = job.construct_real_additional_file_name(f)
                self.platform.send_file(real_name)

    def _do_submission(self, job_scripts: dict[str, str] = None, hold: bool = False) -> None:
        """
        Submits jobs to the platform, cleans previous run logs, and updates job status.

        :param job_scripts: Dictionary of job scripts, defaults to None.
        :param hold: If True, the job won't start immediately, defaults to False.
        """
        if callable(getattr(self.platform, 'remove_multiple_files')):
            filenames = str()
            for job in self.jobs:
                job.update_local_logs()
                filenames += " " + self.platform.remote_log_dir + "/" + job.name + "_STAT " + \
                             self.platform.remote_log_dir + "/" + job.name + "_COMPLETED"
            self.platform.remove_multiple_files(filenames)

        else:
            for job in self.jobs:
                job.update_local_logs()
                self.platform.remove_stat_file(job)
                self.platform.remove_completed_file(job.name)
                if hold:
                    job.hold = hold

        package_id = self.platform.submit_job(None, self._common_script, hold=hold, export=self.export)

        if package_id is None or not package_id:
            return
        for i in range(0, len(self.jobs)):
            Log.info(f"{self.jobs[i].name} submitted")
            self.jobs[i].id = str(package_id)
            self.jobs[i].status = Status.SUBMITTED
            self.jobs[i].wrapper_name = self.name

    def _common_script_content(self) -> str:
        pass  # pragma: no cover


class JobPackageThreadWrapped(JobPackageThread):
    """
    Class to manage a thread-based package of jobs to be submitted by autosubmit
    """
    FILE_PREFIX = 'ASThread'

    def __init__(self, jobs: list[Job], dependency=None, configuration: Optional['AutosubmitConfig'] = None,
                 wrapper_section="WRAPPERS"):
        super(JobPackageThreadWrapped, self).__init__(jobs, configuration)
        self._name = None
        self._job_scripts = {}
        self._job_dependency = dependency
        self._common_script = None
        self._wallclock = '00:00'
        self._num_processors = '0'
        self._threads = '1'
        self.current_wrapper_section = wrapper_section

    @property
    def name(self):
        return self._name

    @property
    def _jobs_scripts(self):
        jobs_scripts = []
        for job in self.jobs:
            jobs_scripts.append(self._job_scripts[job.name])
        return jobs_scripts

    @property
    def queue(self):
        if str(self._num_processors) == '1' or str(self._num_processors) == '0':
            return self.platform.serial_platform.serial_queue
        else:
            return self.platform.queue

    @queue.setter
    def queue(self, value):
        self._queue = value

    @property
    def _project(self):
        return self._platform.project

    def _create_scripts(self, configuration: 'AutosubmitConfig'):
        for i in range(0, len(self.jobs)):
            self._job_scripts[self.jobs[i].name] = self.jobs[i].create_script(configuration)
        self._common_script = self._create_common_script()

    def _create_common_script(self, filename: str = ""):
        script_content = self._common_script_content()
        script_file = self.name + '.cmd'
        open(os.path.join(self._tmp_path, script_file), 'wb').write(script_content)
        os.chmod(os.path.join(self._tmp_path, script_file), 0o755)
        return script_file

    def _send_files(self):
        for job in self.jobs:
            self.platform.send_file(self._job_scripts[job.name])
        self.platform.send_file(self._common_script)

    def _do_submission(self, job_scripts: dict[str, str] = None, hold: bool = False) -> None:
        """
        Submits jobs to the platform, cleans previous run logs, and updates job status.

        :param job_scripts: Dictionary of job scripts, defaults to None.
        :param hold: If True, the job won't start immediately, defaults to False.
        :type hold: bool
        """
        for job in self.jobs:
            job.update_local_logs()
            self.platform.remove_stat_file(job)
            self.platform.remove_completed_file(job.name)
            if hold:
                job.hold = hold

        package_id = self.platform.submit_job(None, self._common_script, hold=hold, export=self.export)

        if package_id is None or not package_id:
            raise Exception('Submission failed')
        for i in range(0, len(self.jobs)):
            Log.info(f"{self.jobs[i].name} submitted")
            self.jobs[i].id = str(package_id)
            self.jobs[i].status = Status.SUBMITTED
            self.jobs[i].wrapper_name = self.name


class JobPackageVertical(JobPackageThread):
    """
    Class to manage a vertical thread-based package of jobs to be submitted by autosubmit
    :param jobs:
    :type jobs:
    :param: dependency:
    """

    def __init__(self, jobs: list[Job], dependency=None, configuration: Optional['AutosubmitConfig'] = None,
                 wrapper_section: str = "WRAPPERS", wrapper_info: Optional[list] = None):
        if wrapper_info is None:
            wrapper_info = []
        super(JobPackageVertical, self).__init__(jobs, dependency, configuration=configuration,
                                                 wrapper_section=wrapper_section, wrapper_info=wrapper_info)
        for job in jobs:
            if int(job.processors) >= int(self._num_processors):
                self._num_processors = job.processors
                self._threads = job.threads
        self._threads = configuration.experiment_data["WRAPPERS"].get(self.current_wrapper_section, {}).get("THREADS",
                                                                                                            self._threads)
        for job in jobs:
            self._wallclock = sum_str_hours(self._wallclock, job.wallclock)
        self._name = f"{self._expid}_{self.FILE_PREFIX}_{jobs_in_wrapper_str(configuration, self.current_wrapper_section)}_{str(int(time.time())) + str(random.randint(1, 10000))}_{self._num_processors}_{len(self._jobs)}"

    def parse_time(self):
        # TODO: Remove this function and use the one in the Job class or move the one in the job class into utils
        # noinspection Annotator
        regex = re.compile(r'(((?P<hours>\d+):)(?P<minutes>\d+))(:(?P<seconds>\d+))?')
        parts = regex.match(self._wallclock)
        if not parts:
            return None
        parts = parts.groupdict()
        if int(parts['hours']) > 0:
            format_ = "hour"
        else:
            format_ = "minute"
        time_params = {}
        for name, param in parts.items():
            if param:
                time_params[name] = int(param)
        return timedelta(**time_params), format_

    def _common_script_content(self) -> str:
        # TODO: normalize this logic to be the same as the one in the Job class
        if self.jobs[0].wrapper_type == "vertical":
            wallclock, format_ = self.parse_time()
            original_wallclock_to_seconds = wallclock.days * 86400.0 + wallclock.seconds

            if format_ == "hour":
                total = wallclock.days * 24 + wallclock.seconds / 60 / 60
            else:
                total = wallclock.days * 24 + wallclock.seconds / 60

            if format_ == "hour":
                hour = int(total)
                minute = int((total - int(total)) * 60.0)
                second = int(((total - int(total)) * 60 -
                              int((total - int(total)) * 60.0)) * 60.0)
            else:
                hour = 0
                minute = int(total)
                second = int((total - int(total)) * 60.0)
            wallclock_delta = datetime.timedelta(hours=hour, minutes=minute, seconds=second)
            wallclock_seconds = wallclock_delta.days * 24 * 60 * 60 + wallclock_delta.seconds
            wallclock_by_level = wallclock_seconds / (self.jobs[-1].level + 1)
            if self.extensible_wallclock > 0:
                wallclock_seconds = int(original_wallclock_to_seconds + wallclock_by_level * self.extensible_wallclock)
                wallclock_delta = datetime.timedelta(hours=0, minutes=0, seconds=wallclock_seconds)
                total = wallclock_delta.days * 24 + wallclock_delta.seconds / 60 / 60
                hh = int(total)
                mm = int((total - int(total)) * 60.0)
                if hh < 10:
                    hh_str = '0' + str(hh)
                else:
                    hh_str = str(hh)
                if mm < 10:
                    mm_str = '0' + str(mm)
                else:
                    mm_str = str(mm)
                self._wallclock = f"{hh_str}:{mm_str}"
                Log.info(f"Submitting {self._name} with wallclock {hh_str}:{mm_str}")
        else:
            wallclock_by_level = 0

        return self._wrapper_factory.get_wrapper(self._wrapper_factory.vertical_wrapper, name=self._name,
                                                 queue=self._queue, project=self._project, wallclock=self._wallclock,
                                                 num_processors=self._num_processors, jobs_scripts=self._jobs_scripts,
                                                 dependency=self._job_dependency, jobs_resources=self._jobs_resources,
                                                 expid=self._expid, rootdir=self.platform.root_dir,
                                                 directives=self._custom_directives, threads=self._threads,
                                                 method=self.method.lower(), retrials=self.inner_retrials,
                                                 wallclock_by_level=wallclock_by_level, partition=self.partition,
                                                 wrapper_data=self, num_processors_value=self._num_processors)


class JobPackageHorizontal(JobPackageThread):
    """
    Class to manage a horizontal thread-based package of jobs to be submitted by autosubmit
    """
    def __init__(self, jobs: list[Job], dependency=None, jobs_resources: dict = None, method: str = 'ASThread',
                 configuration: Optional['AutosubmitConfig'] = None, wrapper_section="WRAPPERS"):
        super(JobPackageHorizontal, self).__init__(jobs, dependency, jobs_resources, configuration=configuration,
                                                   wrapper_section=wrapper_section)
        self.method = method
        self._queue = self.queue
        for job in jobs:
            if job.wallclock > self._wallclock:
                self._wallclock = job.wallclock
            self._num_processors = str(int(self._num_processors) + int(job.processors))
            self._threads = job.threads
        self._threads = configuration.experiment_data["WRAPPERS"].get(self.current_wrapper_section, {}).get("THREADS",
                                                                                                            self._threads)
        self._name = f"{self._expid}_{self.FILE_PREFIX}_{jobs_in_wrapper_str(configuration, self.current_wrapper_section)}_{str(int(time.time())) + str(random.randint(1, 10000))}_{self._num_processors}_{len(self._jobs)}"

        self._jobs_resources = jobs_resources

    def _common_script_content(self) -> str:
        return self._wrapper_factory.get_wrapper(self._wrapper_factory.horizontal_wrapper, name=self._name,
                                                 queue=self._queue, project=self._project, wallclock=self._wallclock,
                                                 num_processors=self._num_processors, jobs_scripts=self._jobs_scripts,
                                                 dependency=self._job_dependency, jobs_resources=self._jobs_resources,
                                                 expid=self._expid, rootdir=self.platform.root_dir,
                                                 directives=self._custom_directives, threads=self._threads,
                                                 method=self.method.lower(), partition=self.partition,
                                                 wrapper_data=self, num_processors_value=self._num_processors)


class JobPackageHybrid(JobPackageThread):
    """
        Class to manage a hybrid (horizontal and vertical) thread-based package of jobs to be submitted by autosubmit
        """

    def __init__(self, jobs: list[list[Job]], num_processors: str, total_wallclock, dependency=None,
                 jobs_resources: Optional[dict] = None, method: str = "ASThread",
                 configuration: Optional['AutosubmitConfig'] = None, wrapper_section="WRAPPERS"):
        all_jobs = [item for sublist in jobs for item in sublist]  # flatten list
        if jobs_resources is None:
            jobs_resources = {}
        super(JobPackageHybrid, self).__init__(all_jobs, dependency, jobs_resources, method,
                                               configuration=configuration, wrapper_section=wrapper_section)
        self.jobs_lists = jobs
        self.method = method
        self._num_processors = int(num_processors)
        self._threads = all_jobs[0].threads
        self._wallclock = total_wallclock
        self._name = f"{self._expid}_{self.FILE_PREFIX}_{jobs_in_wrapper_str(configuration, self.current_wrapper_section)}_{str(int(time.time())) + str(random.randint(1, 10000))}_{self._num_processors}_{len(self._jobs)}"

    @property
    def _jobs_scripts(self):
        self._jobs_resources['PROCESSORS_PER_NODE'] = self.platform.processors_per_node

        jobs_scripts = []
        for job_list in self.jobs_lists:
            inner_jobs = list()
            for job in job_list:
                inner_jobs.append(job.name + '.cmd')
                if job.section not in self._jobs_resources:
                    self._jobs_resources[job.section] = dict()
                    self._jobs_resources[job.section]['PROCESSORS'] = job.processors
                    self._jobs_resources[job.section]['TASKS'] = job.tasks
            jobs_scripts.append(inner_jobs)
        return jobs_scripts


class JobPackageVerticalHorizontal(JobPackageHybrid):

    def _common_script_content(self) -> str:
        return self._wrapper_factory.get_wrapper(self._wrapper_factory.hybrid_wrapper_vertical_horizontal,
                                                 name=self._name, queue=self._queue, project=self._project,
                                                 wallclock=self._wallclock, num_processors=self._num_processors,
                                                 jobs_scripts=self._jobs_scripts, dependency=self._job_dependency,
                                                 jobs_resources=self._jobs_resources, expid=self._expid,
                                                 rootdir=self.platform.root_dir, directives=self._custom_directives,
                                                 threads=self._threads, method=self.method.lower(),
                                                 partition=self.partition, wrapper_data=self,
                                                 num_processors_value=self._num_processors)


class JobPackageHorizontalVertical(JobPackageHybrid):

    def _common_script_content(self) -> str:
        return self._wrapper_factory.get_wrapper(self._wrapper_factory.hybrid_wrapper_horizontal_vertical,
                                                 name=self._name, queue=self._queue, project=self._project,
                                                 wallclock=self._wallclock, num_processors=self._num_processors,
                                                 jobs_scripts=self._jobs_scripts, dependency=self._job_dependency,
                                                 jobs_resources=self._jobs_resources, expid=self._expid,
                                                 rootdir=self.platform.root_dir, directives=self._custom_directives,
                                                 threads=self._threads, method=self.method.lower(),
                                                 partition=self.partition, wrapper_data=self,
                                                 num_processors_value=self._num_processors)
