"""Module for array job submission.
"""
from hpc_tools import (
    __version__,
    __name__ as PKG_NAME,
    common
)
from pyaddons import utils
from shutil import copyfile, rmtree
from datetime import datetime
from stat import (
    S_IXGRP,
    S_IXUSR,
    S_IXOTH,
    ST_MODE,
    S_IRWXO,
    S_IRWXG
)
from terminaltables import AsciiTable
import argparse
import configparser
import re
import sys
import os
import csv
import time
import tempfile
import subprocess
import tarfile
import math
import shutil

# Argument, Default, Required, quote
QSUB_ARGS = [('step', 1, True, False),
             ('batch', None, False, False),
             ('mem', '1G', True, False),
             ('shell', '/bin/bash', True, True),
             ('time', None, False, True),
             ('nodes', None, False, True),
             ('tmp_dir', None, True, True),
             ('scratch_size', None, False, False),
             ('tmp_size', None, False, False),
             ('project', None, False, True),
             ('paid', None, False, False)]

LOG_ARGS = [('log_dir', None, True, False)]

JOB_ARRAY_ARGS = [('ja_file', None, True, False),
                  ('task_script', None, True, False),
                  ('infiles', [], False, False),
                  ('outfiles', [], False, False)]

SCRIPT_ARGS = [
    ('verbose', False, True, False),
    ('whole_steps', False, True, False)
]

# Mappings between arguments in this script and qsub arguments
CLUSTER_OPTION_MAP = {'mem': [''], 'time': ''}
QSUB_PYTHON_MAPPING = 'QSUB_PYTHON_MAPPING'


# @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
class JobNotRunningError(Exception):
    """
    Called when a job has been initialised but not run
    """
    pass


# @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
class JobIsRunningError(Exception):
    """
    Called when a job is running and is attempted to be run again
    """
    pass


# @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
class JobNotInitialisedError(Exception):
    """
    Called when a job has not been initialised and is attempted to be run
    """
    pass


# @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
class JobIsInitialisedError(Exception):
    """
    Called when a job has been initialised and is trying to be initialised
    again
    """
    pass


# @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
class JobCompleteError(Exception):
    """
    Called when a job has been completed
    """
    pass


# @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
class JobArray(object):
    """
    A class for managing and submitting a job array. This will workout what
    tasks have been run and what tasks stll have to be run and will adjust the
    length of the job array accordingly
    """
    # The delimiter for the run log
    DELIMITER = "\t"

    # The header columns for the run log
    RUN_LOG_HEADER = ['run_time', 'log_dir', 'run_dir', 'step', 'batch', 'mem',
                      'shell', 'time', 'nodes', 'scratch_dir', 'tmp_dir',
                      'scratch_size', 'temp_size', 'total_tasks',
                      'no_of_tasks_submitted', 'infile_exist', 'infile_missing',
                      'outfile_exist', 'outfile_missing', 'ja_file_path',
                      'ja_file_md5', 'task_script_path', 'task_script_md5',
                      'job_id']

    # A shortened job header for outputting as a summary on the command line
    RUN_LOG_HEADER_SMALL = ['run_time', 'run_dir', 'step', 'batch', 'mem',
                            'time', 'total_tasks', 'no_of_tasks_submitted',
                            'infile_exist', 'infile_missing', 'outfile_exist',
                            'outfile_missing', 'job_id']

    # The file basename for the run log that will be output into the current
    # run directory
    RUN_LOG_NAME = 'run.log'

    # Columns that can't be in the job array file header, there are not any at
    # the moment but they can be added here
    RESERVED_COLS = []

    # The name of the binary call
    BINARY = 'qsub'

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    @classmethod
    def parse_run_log(cls, run_log_path):
        """
        Parse the run log into a `list` of `OrderedDict` where each dict
        represents a row in the `run.log` file. The rows in the `list` are in
        the same order as they were written to the file. The `keys` of the
        `OrderedDict` are the following column names:

        1. `run_time` - The date and time the run was started
        2. `log_dir` - The root of the directory where all the logs are written
        3. `run_dir` - The directory where the current run files are written
        4. `step` - The step size of the job array. length - step = #tasks
        5. `batch` - The max number of tasks that can be run at the same time
        6. `mem` - The memory size for each task in of the job array
        7. `shell` - The shell for each task in the job array
        8. `time` - The time for each task in the job array
        9. `nodes` - Any specific node set that it is being run on
        10. `scratch_dir` - The location of the scratch directory if set
        11. `tmp_dir` - The location of the tmp directory if set
        12. `scratch_size` - The max size of all the files written to scratch
        13. `temp_size` - The max size of all the files written to tmp
        14. `total_tasks` - The total number of tasks in the job array file
        15. `no_of_tasks_submitted` - The number of tasks in the job array file
                                      that will be submitted in the current run
        16. 'infile_exist` - The number of input files that exist. Note that
                             this is not always the same as total_tasks
        17. `infile_missing` - The number of input files that are missing
        18. `outfile_exist` - The number of output files that exist, note that
                              this does not always equal the number of
                              completed tasks
        19. `outfile_missing` - The number of output files that are not present
        20. `ja_file_path` - The path to the job array file
        21. `ja_file_md5` - The MD5 hash for the job array file
        22. `task_script_path` - The path to the task script that will be
                                 called by pyjasub
        23. `task_script_md5` - The MD5 hash of the task script
        24. `job_id` - The job ID that will be returned by qsub, will be
                       `NOT_SUBMITTED` if the job has not been run

        Parameters
        ----------
        run_log_path : str
            The path to the run log file

        Returns
        -------
        run_log : list of OrderedDict
            Each element in the `list` is a row in the run log file. Each key
            in a dict is a column name and each value is the row/column value
        """
        run_log = []
        with open(run_log_path) as csvin:
            reader = csv.reader(csvin, delimiter=cls.DELIMITER)
            header = next(reader)

            for row in reader:
                run_log.append(dict([(header[idx], row[idx])
                                     for idx in range(len(header))]))

        return run_log

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    @classmethod
    def print_run_log(cls, run_log_path, small=False):
        """
        Output the run log in a terminal table

        Parameters
        ----------
        run_log_path : str
            The path to the run log file
        small : bool, optional
            Shall a full width table be output, ir just the columns with the
            most relevant information:
            `run_time`, `run_dir`, `step`, `batch`, `mem`, `time`,
            `total_tasks`, `no_of_tasks_submitted`, `infile_exist`,
            `infile_missing`, `outfile_exist`, `outfile_missing`, `job_id`
        """
        run_log = cls.parse_run_log(run_log_path)

        data = []

        if small is False:
            data = [cls.RUN_LOG_HEADER]

            for i in run_log:
                data.append([i[j] for j in cls.RUN_LOG_HEADER])
        elif small is True:
            data = [cls.RUN_LOG_HEADER_SMALL]
            for i in run_log:
                row = [i['run_time'], os.path.basename(i['run_dir'])]
                row.extend([i[j] for j in data[0][2:]])
                data.append(row)

        # print with terminal tables
        table = AsciiTable(data)
        print(table.table)

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def __init__(self, **kwargs):
        """
        Parameters
        ----------
        log_dir : str
            The location of the run log directory
        tmp_dir : str
            The temp location, this will be passed to the task script so
            intermediate files can be placed there. Although, a keyword
            argument, this _must_ be supplied.
        ja_file : str
            The location of the job array file
        task_script : str
            The location of the task script
        step : int, optional
            The step size for the job array (default=1)
        batch : int, optional
            The batch size for the job array, i.e. how many jobs will run
            at once (default=None (no limit))
        mem : str, optional
            The memory for each task in the job array (default=`1G`)
        shell : str, optional
            The shell to run the task in (defualt=`/bin/bash`)
        time : str, optional
            The max time limit for the job (default=`NoneType`) no specified
            time limit but the schedulaer may apply one of it's own
        nodes : str, optional
            Restrict to specific hostname names (default=`NoneType` (not set))
        scratch_size : str, optional
            The size of any scratch/temp space (default=`NoneType` (not set))
        infiles : list of str, optional
            The names of column in the job array file that contain input files
            any columns in here must exist in the job array file and must
            contain a file that is present and readable. Note that the input
            files in any of the infiles columns can be pipe `|` delimited.
            i.e. >1 input file in each column (default=`[]`)
        outfiles : list of str, optional
            The location os columns in the job array file that contain output
            files. If all the output files are present in these columns then
             the task is not scheduled for running. Note that the input
            files in any of the outfiles columns can be pipe `|` delimited.
            i.e. >1 output file in each column (default=`[]`)
        verbose : bool
            Should progress be output? (default=`False`)
        whole_steps : bool, optional, default: False
            If step size is > 1 should all the steps (rows) be treated as a
            logical unit
        """
        # Has the log directory been created during this run
        self._log_dir_created = False

        # Does the run directory already exist. This is a safety feature that
        # makes sure that if the run directory exists and fails when it is
        # created, then it is not deleted in the abort sequence
        self._run_dir_exists = False

        # Will hold the path to the log files for the current run, this will
        # be created during initialisation and will be deleted if the
        # submission run fails for any reason
        self._run_log_path = ""

        # This will hold all the entries from the run log when it has been
        # parsed, the run log is in a table format
        self._run_log = []

        # log the current time the submission run started and also the epoch
        # time
        self._cur_run_time = datetime.now()
        self._cur_run_epoch = int(time.time())

        # We initialise the current run number to 1 and the previous run
        # number to 0. This will be updated when the run log is parsed during
        # initialisation
        self._cur_run_no = 1
        self._prev_run_no = self._cur_run_no - 1

        # This holds the run log entry for the current run
        self._cur_run = {}

        # Several boolean check points that will become True as we go through
        # the submission process
        self._is_submitted = False
        self._is_running = False
        self._is_initialised = False
        self._is_complete = False
        self._using_context_manager = False

        # Create a msg outputter
        self.msg = common.Msg(verbose=False, prefix='[info]')

        # now parse the script arguments
        self._parse_args(SCRIPT_ARGS, kwargs)
        self.msg.verbose = self.verbose

        # Initialise all the object variables
        self._parse_args(QSUB_ARGS, kwargs)
        self._parse_args(LOG_ARGS, kwargs)
        self._parse_args(JOB_ARRAY_ARGS, kwargs)

        # Now get the mappings
        self._mappings = kwargs.pop('qsub_mappings', None)

        # We force whole steps to be False if the step size == 1, this should
        # not actually matter but I wanted to ensure there are no side effects
        # as I have not tested whole_steps to destruction
        if self.step == 1:
            self.whole_steps = False

        # Now check the arguments
        self._check_args()

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def __enter__(self):
        """
        Entry point for the context manager
        """
        self._using_context_manager = True
        return self

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def __exit__(self, exc_type, exc_value, exc_traceback):
        """
        Exit point for the context manager, accepts the error code/traceback
        these will be `NoneType` if we exit cleanly

        Parameters
        ----------
        exc_type
        exc_value
        exc_traceback
        """
        if exc_type is None:
            # Write the logs etc
            self.finish()
        else:
            # We have an error so go through the abort sequence
            self.abort()

        # If we ever get here we will not be using the context manager anymore
        self._using_context_manager = False
        self._reset()

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def initialise(self):
        """
        Run the checks on the job array file

        Raises
        ------
        FileExistsError
            If the log_dir exists but is not a directory
        PermissionError
            If the task script is not executable in any way
        KeyError
            If any of the checking columns are not in the header. If any of
            the reserved columns are in the header
        IndexError
            If the length of any of the rows in the job array does not match
            the length of the header
        FileNotFoundError
            If any of the input files are not present
        RuntimeError
            If there are no tasks to submit, the inference is that all the
            tasks are complete
        JobIsInitialisedError
            If initialise has already been called
        """
        if self._is_initialised is True:
            raise JobIsInitialisedError("already initialised")

        # Make sure the job log directory exists, if not then create it
        # if possible
        self._init_log_dir()

        # Parse the run log and determine the run number
        self._parse_run_log()

        # Now set up a run directory and directories to hold out and err files
        self._set_run_dir()

        # Initialise the job array file, this checks it exists and can be
        # opened and makes sure it has not been changed
        self._init_ja_file()

        # Now make sure the task script is ok, create a backup copy of it
        # and issue a warning if it has changed
        self._init_task_script()

        try:
            # Now perform the required checks on the job array file, so these
            # are task level checks of input/output files
            checks = self._check_jafile()
        except Exception as e:
            raise Exception("problem checking the job array file...") from e

        try:
            self._evaluate_checks(checks)
        except JobCompleteError as e:
            # Now update the run log with all the parameters defined during
            # initialisation
            self._update_log_entry(checks)
            self.msg.msg(str(e), prefix='[complete]')
            self._is_complete = True
            self._wrap_up_mesg()
            raise

        self._checks = checks

        # Now update the run log with all the parameters defined during
        # initialisation
        self._update_log_entry(checks)

        # If we get here then we have sucessfully initialised
        self._is_initialised = True

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def finish(self):
        """
        Normally called by the __exit__ in the context manager
        """
        if self._is_complete is True:
            # Write the logs
            self.msg.msg('job is complete')
            # raise JobCompleteError("job is complete")
        elif self._is_initialised is False:
            self.msg.msg('job not initialised...will abort')
            self.abort()
            # raise JobNotInitialisedError("job is not initialised")
        elif self._is_running is False:
            self.msg.msg('job not run...will abort')
            self.abort()
            # raise JobNotRunError("job is not initialised")

        # If we every get here we will not be using the context manager
        # anymore
        self._using_context_manager = False
        self.msg.msg("*** job submission finished ***")

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def run(self):
        """
        Submit any outstanding tasks
        """
        if self._is_complete is True:
            raise JobCompleteError("job is already complete")
        if self._is_initialised is False:
            raise JobNotInitialisedError("the job array file has not been"
                                         " initialised, please run "
                                         "obj.initialise()")
        elif self._is_running is True:
            raise JobIsRunningError("job array is already running")

        command = self._map_commands()
        # pp.pprint(command)
        print("[run] You are about to run '{0}' tasks over '{1}' rows in a"
              " job array with a step of '{2}', using the following"
              " command:".format(self._checks['no_tasks_to_submit'],
                                 self._checks['run_file_rows'],
                                 self.step))
        print("[run] {0}".format(" ".join(command)))
        print("[run] if all this looks correct, press [ENTER] to submit or"
              " [CTRL-C] to quit > ")
        try:
            input()
            print('[run] submitting job...')
            self._run(command)
            self._wrap_up_mesg()
            self._is_running = True
        except KeyboardInterrupt:
            print('[abort] user aborted run...')

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def _run(self, command):
        """
        Actually run the job and capture the ID
        """
        with subprocess.Popen(command, stdout=subprocess.PIPE,
                              stderr=subprocess.PIPE) as proc:
            proc.wait()
            if proc.returncode != 0:
                raise RuntimeError("problem submitting job: {0}".format(
                    proc.stderr.read().decode()))
            id_str = proc.stdout.read().decode()
            jid_regexp = re.search(
                r'Your\s+job[ -]array\s+(\d+)\..+submitted$',
                id_str, re.IGNORECASE
            )
            self._cur_run['job_id'] = jid_regexp.group(1)
            print("[run] submitted array job with ID {0}".format(
                self._cur_run['job_id']))

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def abort(self):
        """
        Abort a run
        """
        if self._is_running is True:
            raise JobIsRunningError("job is running can't abort")

        print("[abort] aborting run '{0}'".format(self._cur_run_no),
              file=sys.stderr)
        self.msg.prefix = '[abort]'

        if self._log_dir_created is True:
            try:
                self.msg.msg(
                    "removing job log directory: '{0}'".format(self.log_dir))
                rmtree(self.log_dir)
            except AttributeError:
                pass
        elif self._run_dir_exists is False:
            try:
                self.msg.msg(
                    "removing run log directory: '{0}'".format(self._run_dir))
                rmtree(self._run_dir)
            except AttributeError:
                pass

        # If we are aborting then we reset everything
        self._reset()

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def _wrap_up_mesg(self):
        """

        """
        # final wrappup messages
        self._run_log.append(self._cur_run)
        self._write_run_log()
        try:
            open(self._run_log_path).close()
            self.msg.msg("run log details below...")
            self.__class__.print_run_log(self._run_log_path, small=True)
        except FileNotFoundError:
            pass

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def _update_log_entry(self, checks):
        """
        Generate a log file entry and potentially add some data from checks to
        it

        Parameters
        ----------
        checks : dict, optional
            The results of checks on a job array file. If not supplied then
            `total_tasks`, `no_of_tasks_submitted`, `infile_exist`,
            `infile_missing`, `outfile_exist`, `outfile_missing` are set to
            `NoneType`

        Returns
        -------
        entry : dict
            An dictionary resresenting an entry into the run log file
        """
        # Initialise the log entry before updating with object values
        self._cur_run['run_time'] = self._cur_run_time
        self._cur_run['log_dir'] = self.log_dir
        self._cur_run['run_dir'] = self._run_dir
        self._cur_run['step'] = self.step
        self._cur_run['batch'] = self.batch
        self._cur_run['mem'] = self.mem
        self._cur_run['shell'] = self.shell
        self._cur_run['time'] = self.time
        self._cur_run['nodes'] = self.nodes
        self._cur_run['tmp_dir'] = self.tmp_dir
        self._cur_run['scratch_size'] = self.scratch_size
        self._cur_run['temp_size'] = self.tmp_size
        self._cur_run['job_id'] = 'NOT_SUBMITTED'
        self._cur_run['total_tasks'] = checks['total_tasks']
        self._cur_run['no_of_tasks_submitted'] = checks['no_tasks_to_submit']
        self._cur_run['infile_exist'] = checks['infiles_present']
        self._cur_run['infile_missing'] = checks['infiles_absent']
        self._cur_run['outfile_exist'] = checks['outfiles_present']
        self._cur_run['outfile_missing'] = checks['outfiles_absent']

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def _reset(self):
        """
        Reset all the flags
        """
        # Reset all the flags
        self._is_submitted = False
        self._is_running = False
        self._is_initialised = False
        self._is_complete = False
        self._checks = None

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def _parse_args(self, arg_list, kwargs):
        """
        Set variables in self from the arg_list and the kwargs

        Parameters
        ----------
        arg_list : list of tuple
            Each tuple has the argument name at [0], the default argument value
            at [1] and the is required value at [2]
        kwargs : dict
            Arguments and values that are parsed to the class initialisation

        Raises
        ------
        ValueError
            If any of the requried arguments are NoneType
        """

        # Loop through all the arguments
        for arg, default, is_required, quote in arg_list:
            try:
                # Attempt to set the argument
                setattr(self, arg, kwargs[arg])
            except KeyError:
                # not in the kwargs, so give the default
                setattr(self, arg, default)

            # Now we error out if the argument is None and is_required
            if getattr(self, arg) is None and is_required is True:
                raise ValueError("'{0}' is required but is set to "
                                 "'NoneType'".format(arg))

            # Output the options is verbose is True
            self.msg.msg('{0}: {1}'.format(arg, getattr(self, arg)))

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def _check_args(self):
        """
        Make sure that all the QSUB arguments are valid
        """
        # These should be integers
        try:
            self.step = int(self.step)
        except TypeError:
            self.step = 1

        try:
            self.batch = int(self.batch)
        except TypeError:
            self.batch = None

        self.mem = self._check_mem_size(self.mem)
        self.scratch_size = self._check_mem_size(self.scratch_size)
        self.tmp_size = self._check_mem_size(self.tmp_size)

        # Make sure the temp dir is valid
        self._check_tmp_dir()

        # Now make sure the time argument is valid
        try:
            if not re.match(r'^\d{2,}:\d{2}:\d{2}$', self.time):
                raise ValueError("unknown time format '{0}'".format(self.time))
        except TypeError:
            # We do allow time to be None
            pass

        # Now make sure we have some qsub mappings
        if self._mappings is None or len(self._mappings) == 0 \
           or not isinstance(self._mappings, dict):
            raise ValueError("qsub_mappings not set correctly")

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def _check_mem_size(self, arg):
        """

        """
        # make sure a qsub "memory" type argument is valid
        # first see if it is an int argument
        try:
            int(arg)
            return arg
        except ValueError:
            pass
        except TypeError:
            # NoneType
            pass

        # If we get here then we make sure that it has a size suffix
        try:
            if re.match(r'^\d+[GMK]$', arg):
                return arg
        except TypeError:
            # it is NoneType which we allow
            return arg

        # If we get here then we do not know what it is
        raise ValueError("unknown argument value '{0}'".format(arg))

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def _check_tmp_dir(self):
        """
        Make sure the temp directory is represented as an absolute path and
        that it exists and is writable
        """
        self.tmp_dir = os.path.realpath(os.path.expanduser(self.tmp_dir))

        if os.path.isdir(self.tmp_dir):
            # Attempt to write a file
            test_fobj, test_fn = tempfile.mkstemp(dir=self.tmp_dir)
            os.close(test_fobj)
            os.unlink(test_fn)
        else:
            raise FileExistsError("tmp_dir is actually a file")

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def _init_log_dir(self):
        """
        Initialise the log directory, if it does not exist then this will
        create it

        Raises
        ------
        FileExistsError
            If the log_dir exists but is not a directory
        """
        # get the full path to what will be the job log directory
        self.log_dir = os.path.realpath(os.path.expanduser(self.log_dir))
        self.msg.msg("full job log directory '{0}'".format(self.log_dir))

        # If the log directory exists but is not a directory then we raise
        # an error
        if os.path.exists(self.log_dir) and os.path.isdir(self.log_dir) is False:
            raise FileExistsError(
                "the log_dir exists but is not a directory")

        # If it does not exist then we create the log dir
        if not os.path.exists(self.log_dir):
            self.msg.msg("creating log directory '{0}'".format(self.log_dir))
            os.mkdir(self.log_dir)

            # This flag is used in the abort sequence. If the directory is
            # created during this run and we have to abort then the whole log
            # directory is removed, otherwise just the run directory is removed
            self._log_dir_created = True

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def _map_commands(self):
        """
        build the job submission command
        """
        # Now build the qsub command string
        command = [self.__class__.BINARY]
        try:
            self._assign_arg(
                command,
                self._mappings['array'],
                (1, self._checks['run_file_rows'], self.step)
            )

            self._assign_arg(command, self._mappings['err'],
                             (self._run_files_err, ))
            self._assign_arg(command, self._mappings['out'],
                             (self._run_files_out, ))

            for argument, default, required, quote in QSUB_ARGS:
                if argument in ['step', 'tmp_dir']:
                    continue

                if getattr(self, argument) is not None:
                    value = getattr(self, argument)

                    self._assign_arg(
                        command, self._mappings[argument], (value, )
                    )
        except KeyError as e:
            raise KeyError("this is probably caused by bad mappings") from e

        # Now add the task script and the task script arguments
        command.extend([self.task_script, self._run_ja_file,
                        self.tmp_dir, str(self.step)])
        return command

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def _assign_arg(self, command, args, values):
        """
        Assign a parameter with arguments with to the command string

        Parameters
        ----------
        command : list of str
            The existing command string that the parameters and arguments
            will be added to
        args : str
            An argument string
        values : tuple
            Parameters to the argument
        """
        all_args = []
        for a in args:
            for i in a.split(" "):
                all_args.append(i.format(*values))
        command.extend(all_args)

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def _write_run_log(self):
        """
        Write the latest run log to the log file
        """
        # First make sure the current run is updated with all the parameters
        # that are defined post run
        with open(self._run_log_path, 'w') as csvout:
            writer = csv.writer(csvout, delimiter=self.__class__.DELIMITER,
                                lineterminator=os.linesep)

            # if self._cur_run == 1:
            writer.writerow(self.__class__.RUN_LOG_HEADER)

            for i in self._run_log:
                writer.writerow([i[h] for h in self.__class__.RUN_LOG_HEADER])

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def _parse_run_log(self):
        """
        Parse the run log table and determine the current run number
        """
        # Determine the path to the run log file
        self._run_log_path = os.path.join(self.log_dir,
                                          self.__class__.RUN_LOG_NAME)

        try:
            # Attempt to parse it, if this is the first run then it may not be
            # available
            self._run_log = self.__class__.parse_run_log(self._run_log_path)
        except FileNotFoundError:
            self._run_log = []

        # Determine the previous and the current run number
        self._prev_run_no = len(self._run_log)
        self._cur_run_no = self._prev_run_no + 1
        self._cur_run = dict([(h, 'unknown')
                              for h in self.__class__.RUN_LOG_HEADER])
        self._cur_run['run_time'] = self._cur_run_time
        # self._run_log.append(self._cur_run)

        self.msg.msg("run started at: '{0}'".format(self._cur_run_time))
        self.msg.msg("current run number: '{0}'".format(self._cur_run_no))
        self.msg.msg("run log path: '{0}'".format(self._run_log_path))

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def _set_run_dir(self):
        """
        Initialise the directries that will hold the data for the current run
        """
        # The run directory will be a concatination of the run number and the
        # epoch time that the object was initialised
        run_dir_basename = '{0}_{1}'.format(self._cur_run_no,
                                            self._cur_run_epoch)

        # Add the run directory to the run log
        self._cur_run['run_dir'] = run_dir_basename

        # Now create the full paths to the run directory and the err/out
        # directories that will hold the job err and out files
        self._run_dir = os.path.join(self.log_dir, run_dir_basename)
        self._run_dir_err = os.path.join(self._run_dir, 'err')
        self._run_dir_out = os.path.join(self._run_dir, 'out')

        # Now create all the run directories
        try:
            os.mkdir(self._run_dir)
            os.mkdir(self._run_dir_out)
            os.mkdir(self._run_dir_err)
        except FileExistsError:
            # This is a safety feature that
            # makes sure that if the run directory exists and fails when it is
            # created, then it is not deleted in the abort sequence
            self._run_dir_exists = True
            raise

        self._run_name = '{0}_{1}'.format(self._cur_run_no,
                                          self._cur_run_epoch)
        # self._log_spec = r'\$TASK_ID_{0}'.format(self._run_name)
        self._log_spec = r"$TASK_ID_{0}".format(self._run_name)
        self._run_files_out = os.path.join(self._run_dir_out,
                                           "{0}.out".format(self._log_spec))
        # self._run_files_out = "'{0}'".format(self._run_files_out)
        self._run_files_err = os.path.join(self._run_dir_err,
                                           "{0}.err".format(self._log_spec))
        # self._run_files_err = "'{0}'".format(self._run_files_err)
        self.msg.msg("run directory: '{0}'".format(self._run_dir))
        self.msg.msg("run out files: '{0}'".format(self._run_files_out))
        self.msg.msg("run err files: '{0}'".format(self._run_files_err))

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def _init_ja_file(self):
        """
        initialise the job array file, this makes sure that the file exists
        and is readable. It also make sure that the md5sum of the jobarray file
        has not changed between runs
        """
        self.ja_file = os.path.realpath(os.path.expanduser(self.ja_file))

        # Will raise a FileNotFound error or a permission error
        open(self.ja_file).close()

        # Now get the MD5 of the job array
        ja_md5 = utils.get_file_md5(self.ja_file)

        # Now if we are not on the first run make sure the MD5 of the jobarray
        # file has not changed
        if self._cur_run_no > 1 and ja_md5 != self._run_log[-1]['ja_file_md5']:
            raise ValueError("the job array file has changed")

        # Store the job array attributes
        self._cur_run['ja_file_path'] = self.ja_file
        self._cur_run['ja_file_md5'] = ja_md5

        self.msg.msg("full job array path: '{0}'".format(self.ja_file))
        self.msg.msg("job array md5: '{0}'".format(ja_md5))

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def _init_task_script(self):
        """
        Initialise the task script. This makes sure it exists and is executable
        it also looks to see if it has changed and issues a warning message if
        it has (not a python warning but is written to stderr)
        """
        self.task_script = os.path.realpath(
            os.path.expanduser(self.task_script))
        self.msg.msg("full task script path: '{0}'".format(self.task_script))

        # Will raise a FileNotFound error or a permission error
        open(self.task_script).close()

        # Now get the MD5 of the job array
        task_script_md5 = utils.get_file_md5(self.task_script)
        self.msg.msg("task script md5: '{0}'".format(task_script_md5))

        # If we are not on the first run thencheck to see if the task script
        # MD5 has changed, if so issue a warning (if verbose)
        if self._cur_run_no > 1:
            self.msg.msg("previous task script md5: '{0}'".format(
                self._run_log[-1]['task_script_md5']))

            if task_script_md5 != self._run_log[-1]['task_script_md5']:
                self.msg.msg("the task script has change since the previous "
                             "run!", prefix='[warning]')

        self._cur_run['task_script_path'] = self.task_script
        self._cur_run['task_script_md5'] = task_script_md5

        # Now see if the file is executable by owner, group and other
        # if none are executable then we issue an error
        own_exe = S_IXUSR & os.stat(self.task_script)[ST_MODE]
        grp_exe = S_IXGRP & os.stat(self.task_script)[ST_MODE]
        oth_exe = S_IXOTH & os.stat(self.task_script)[ST_MODE]

        # Now issue warnings and errors about it being executable
        if sum([own_exe, own_exe, own_exe]) == 0:
            raise PermissionError("task script not executable")

        if own_exe is False:
            self.msg.msg("task script not executable by owner",
                         prefix='[warning]')
        if grp_exe is False:
            self.msg.msg("task script not executable by group",
                         prefix='[warning]')
        if oth_exe is False:
            self.msg.msg("task script not executable by others",
                         prefix='[warning]')

        # Make a backup copy of the task script
        task_backup = os.path.join(self._run_dir,
                                   '{0}.task'.format(task_script_md5))
        self.msg.msg("copying task script: '{0}'".format(task_backup))
        copyfile(self.task_script, task_backup)

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def _check_jafile(self):
        """
        Check the job array file and produce a run file of all the tasks that
        will be run. This will not contain all the tasks that have output files

        Rasies
        ------
        ValueError
            If the first value in each data row is not a sequential counter
            that starts from 1 (after the header)
        KeyError
            If any of the checking columns are not in the header. If any of
            the reserved columns are in the header
        IndexError
            If the length of any of the rows in the job array does not match
            the length of the header. If the total rows in the job array file
            is not a multiple of the step size
        """
        # The run file will be the file that is actually passed to the task
        # script. So the job array file will serve as a template for the
        # run file
        self._run_ja_file = os.path.join(self._run_dir, 'run_job_array.job')
        self.msg.msg("run job array path: '{0}'".format(self._run_ja_file))

        # We will log all these statistics
        checks = {'total_rows': 0, 'total_tasks': 0, 'no_tasks_to_submit': 0,
                  'run_file_rows': 0, 'infiles_present': 0,
                  'infiles_absent': 0, 'outfiles_present': 0,
                  'outfiles_absent': 0}

        check_steps = self.step
        if self.whole_steps is False:
            check_steps = 1

        # Open the run file make sure we have the correct line endings
        with open(self._run_ja_file, 'w') as csvout:
            writer = csv.writer(csvout, delimiter="\t",
                                lineterminator=os.linesep)
            # Now open up the job array file
            with open(self.ja_file) as csvin:
                reader = csv.reader(csvin, delimiter="\t")
                header = next(reader)

                # Make sure the header is valid. This checks that the column
                # 'rowidx' is at column 0 and any defined input/output file
                # columns are present
                self._check_ja_file_header(header)

                # copy the header into the run file
                writer.writerow(header)

                # So here it can get tricky. We have to take into account the
                # step size when evaluating the output files. So, we have to
                # have all output files present for all steps in the task. So
                # often the step size will be 1, so each row in the job array
                # file will be a task. However, if the step size is 2 then
                # every two rows in a job array file will be a task and if both
                # rows have an output file then both output files will have
                # to exist in order for that task to eb marked completed

                # These flags keep track of the presence/absence of
                # input/output files within all steps of a task, so we can
                # evaluate at the task level not the row level
                step_batch_all_in_present = True
                step_batch_any_out_absent = True

                # This will buffer all of the rows for all steps in a task.
                # When, we come to the last step then we evaluate, output
                # if necessary, clear theis buffer and move on
                step_batch_rows = []

                runidx = 0
                for row in reader:
                    # If the length of the current row does not equal the
                    # length of the header then it is an error
                    if len(row) != len(header):
                        raise IndexError("row length '{0}' != header length"
                                         " '{1}' at row '{2}'".format(
                                             len(row), len(header),
                                             checks['total_tasks']))

                    # Map the row back to the header, this makes the code
                    # clearer below
                    mapped_row = dict([(h, row[c])
                                       for c, h in enumerate(header)])

                    # Here we should be able to predict what the first column
                    # value in each row is. If it is not what we expect then
                    # we will error out
                    if checks['total_rows']+1 != int(mapped_row['rowidx']):
                        raise ValueError("the first column of the job array"
                                         " file should be the rowidx, this"
                                         " should be an incremental counter"
                                         " of the number of data rows in "
                                         "the job arrayfile")

                    # These flags indicate if across th einput files in a
                    # row (not necessarily a task) are present and are any
                    # of the output files absent. They are defaulting to True
                    # as when they are both in that state the task will be run
                    # We want this to be the default state just in case the
                    # user does not pass any specific input/output columns
                    # to check, in which case, all tasks will always be run
                    all_in_present = True
                    any_out_absent = True

                    # If the user has passed some input file columns to check
                    # there could be 1 or more and each or these could be
                    # a pipe separated list of input files
                    if len(self.infiles) > 0:
                        # 1. boolean are all the files in all the input file
                        # columns present?
                        # 2. The total number of input files checked for the
                        # current row
                        # 3. The total number of present input files in the
                        # current row
                        all_in_present, total_infiles, present_infiles = \
                            self._infile_check(mapped_row)

                        # Increament the global counters
                        checks['infiles_present'] += present_infiles
                        checks['infiles_absent'] += \
                            (total_infiles - present_infiles)

                        # update the step flags with the data from the row flag
                        step_batch_all_in_present &= all_in_present

                    # If the user has passed some output file columns as with
                    # the input file columns there could be 1 or more and these
                    # could be pipe | separated output file names. Remember ALL
                    # of the output files will have to exist in order for the
                    # task to be skipped
                    if len(self.outfiles) > 0:
                        # 1. Boolean - are any output files missing, if so we
                        # will end up running the task
                        # 2. The total output files tested
                        # 3. The total number of output files that were present
                        any_out_absent, total_outfiles, present_outfiles = \
                            self._outfile_check(mapped_row)

                        # Increment the global totals
                        checks['outfiles_present'] += present_outfiles
                        checks['outfiles_absent'] += \
                            (total_outfiles - present_outfiles)

                        # Update the the step batch flags. As soon as this
                        # becomes False then the task will not be run
                        step_batch_any_out_absent &= any_out_absent

                    # Buffer the row with any other row steps that make up
                    # the task
                    step_batch_rows.append(row)
                    checks['total_rows'] += 1
                    # If we are at the end of the step batch, i.e. the
                    # last row for the task. Ten we need to evaluate
                    # if we need to include all the step rows into the
                    # run file or exclude them all
                    # if (checks['total_rows'] % self.step) == 0:
                    if (checks['total_rows'] % check_steps) == 0:
                        # if all the input files for the step batch are
                        # present and not all the output files are present
                        # i.e. an incompletly finished task. Then we want
                        # to run the task
                        if (step_batch_all_in_present &
                            step_batch_any_out_absent) is True:
                            checks['no_tasks_to_submit'] += 1
                            for r in step_batch_rows:
                                runidx += 1
                                row[0] = runidx
                                writer.writerow(row)

                        # This is the total tasks in the job array file, not
                        # necessarily the number we are running
                        checks['total_tasks'] += 1

                        # reset the data for the next step batch
                        step_batch_all_in_present = True
                        step_batch_any_out_absent = True
                        step_batch_rows = []

                # The number of rows in the run file (minus the header)
                # (not necessarily the number of tasks)
                checks['run_file_rows'] = runidx

                # Now make sure that there are no rows left
                if len(step_batch_rows) > 0:
                    raise IndexError("total rows in job array '{0}' is not a"
                                     " multiple of the step size '{1}'".format(
                                         checks['total_rows'], self.step))

                # This is a bit of a fudge to account for whole steps = False
                # whole steps = True used to be the default
                # (but not an actual option) but it is not always needed so I
                # made it an option. in doing so I had to force the checking
                # of steps to think that the step size is 1 (see check_step).
                # This buggers the task count so this if statement is required
                # to correct the counts. Really, I need to re-write this method
                if check_steps == 1 and self.step > 1:
                    checks['total_tasks'] = math.ceil(
                        checks['total_rows'] / self.step
                    )
                    checks['no_tasks_to_submit'] = math.ceil(
                        checks['run_file_rows'] / self.step
                    )
        return checks

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def _check_ja_file_header(self, header):
        """
        This checks the job array file header. It makes sure that any columns
        that have things that we want to check within them do actually exist
        and that we do not have a rowidx column which is added by this class
        and can't be present

        Parameters
        ----------
        header : list of str
            The header from the job array file

        Raises
        ------
        KeyError
            If any required columns are missing or any reserved columns are
            present
        """
        # Make sure the first column is the rowidx
        if header[0] != 'rowidx':
            raise KeyError("column '0' in the job array file should be "
                           "rowidx not '{0}'".format(header[0]))

        for o in self.outfiles:
            if o not in header:
                raise KeyError("column '{0}' is not in the job array file "
                               "header".format(o))

        for i in self.infiles:
            if i not in header:
                raise KeyError("column '{0}' is not in the job array file "
                               "header".format(i))

        for r in self.__class__.RESERVED_COLS:
            if r in header:
                raise KeyError("reserved column '{0}' is in the job array "
                               "file header".format(i))

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def _infile_check(self, row):
        """
        Check that all the input files are present. in order for the task to be
        run then all the input files must exist

        Parameters
        ----------
        row : dict
            A task row, where the keys are column names and the values are
            task values

        Returns
        -------
        all_in_present : bool
            Are all the input files present if so then the task needs to be run
        total_outfiles : int
            The total number of input files tested
        present_infiles : int
            The number of input files that are present
        """
        all_in_present = False
        total_infiles = 0
        present_infiles = 0

        # Loop through all the input file columns that need to be checked
        for c in self.infiles:
            # Get all the infiles in the current column we are checking. We
            # allow for the possibility that the input file column is pipe |
            # delimited
            infiles = [i.strip() for i in row[c].split('|')]

            # increment the total number of input files
            total_infiles += len(infiles)

            # Loop through all the input files in the current row/column
            for i in infiles:
                try:
                    # if we can open and close it without a file or permission
                    # error then it is able to be used
                    open(i).close()
                    present_infiles += 1
                except (FileNotFoundError, PermissionError):
                    # infile is not accessable for whatever reason
                    pass

        # if we have some input files and and all of them are present then
        # the task can be run
        if total_infiles > 0 and total_infiles == present_infiles:
            all_in_present = True

        return all_in_present, total_infiles, present_infiles

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def _outfile_check(self, row):
        """
        Check that all the output files in all the columns that we want to
        check are absent. If they are all present then we do not want to run
        the job. For example, if there are 10 output files and 10 are present
        then the task is already complete so does not need to be run. However,
        if there are 10 output files but only 9 are present then the task is
        not complete so will need to be (re)run.

        Parameters
        ----------
        row :dict
            A task row, where the keys are column names and the values are
            task values

        Returns
        -------
        any_out_absent : bool
            Are any of the output files absent if so then the task needs
            to be run
        total_outfiles :i nt
            The total number of output files tested
        present_outfiles : int
            The number of output files that are present
        """
        any_out_absent = False
        total_outfiles = 0
        present_outfiles = 0
        for c in self.outfiles:
            # Get all the output files, we allow for the possibility of the
            # output files being pipe '|' delimited in the job array file
            outfiles = [o.strip() for o in row[c].split('|')]

            # increment the total output files
            total_outfiles += len(outfiles)

            # Now loop through the output files
            for o in outfiles:
                try:
                    # if we can open and close it without a FileNotFound error
                    # then it is deemed present
                    open(o).close()
                    present_outfiles += 1
                except FileNotFoundError:
                    # if we throw a file not found error then it is not there
                    pass

        # If we have some output files to check and they are not all presnt
        # then the task needs to be run
        if total_outfiles > 0 and present_outfiles < total_outfiles:
            any_out_absent = True

        return any_out_absent, total_outfiles, present_outfiles

    # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    def _evaluate_checks(self, checks):
        """
        Evaluate the job array file checks to determine how to proceed

        Parameters
        ----------
        checks :dict
            The results from the job array file checks

        Raises
        ------
        StopIteration
            If the there are no task rows in the job array file
        IndexError
            If the total rows in the job array file is not a multiple of the
            step size
        FileNotFoundError
            If any of the input files are not present
        JobCompleteError
            If there are no tasks to submit, the inference is that all the
            tasks are complete
        """

        # Make sure that we actually have some rows
        if checks['total_rows'] == 0:
            raise StopIteration("no task rows in the job array file")

        # Now we check that the stepping is correct, if the total
        # number of rows in the job array file should be an integer
        # multiple of the step size, i.e. every task should have the
        # same number of rows in the job array file. If not it is a
        # problem. The vast majority of the time the step size will
        # be 1 but maybe not in all cases
        if self.whole_steps is True and \
           (checks['total_rows'] % self.step) != 0:
            raise IndexError("total rows in job array '{0}' is not a multiple"
                             " of the step size '{1}'".format(
                                 checks['total_rows'], self.step))

        self.msg.msg("check results below:")

        # Now we process the check results
        if len(self.infiles) > 0:
            self.msg.msg("input file present: '{0}'".format(
                checks['infiles_present']))
            self.msg.msg("input file absent: '{0}'".format(
                checks['infiles_absent']))
        else:
            self.msg.msg("no input file checks!")

        if len(self.outfiles) > 0:
            self.msg.msg("output file present: '{0}'".format(
                checks['outfiles_present']))
            self.msg.msg("output file absent: '{0}'".format(
                checks['outfiles_absent']))
        else:
            self.msg.msg("no output file checks!")

        self.msg.msg("total tasks: '{0}'".format(
                checks['total_tasks']))

        # If there are missing input files, this is potentially an issue
        # but it depends on if input files have been deleted as part of the
        # job run. So, if the run number is > 1, then we check that the
        # input files/output files are modulus 0 that is to say the absent
        # input files are accounted by the present output files, if so we just
        # warn instead of erroring out
        if checks['infiles_absent'] > 0:
            if self._cur_run_no > 1 and \
               max(checks['infiles_absent'], checks['outfiles_present']) % \
               min(checks['infiles_absent'], checks['outfiles_present']) == 0:
                self.msg.msg(
                    "*** WARNING *** '{0}' missing input files but these are"
                    " accounted for by present output files '{1}', if you "
                    "do not expect this then you should abort".format(
                        checks['infiles_absent'], checks['outfiles_present']
                    )
                )
            else:
                self.msg.msg("tasks to be submitted: '0'")
                raise FileNotFoundError(
                    "unable to submit due to missing input files"
                )
        else:
            self.msg.msg("tasks to be submitted: '{0}'".format(
                checks['no_tasks_to_submit']))

        if checks['no_tasks_to_submit'] == 0:
            raise JobCompleteError("no tasks to submit - job is complete")


# @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
def _init_cmd_args(description=None):
    parser = None
    if description is None:
        description = ("Submit a qsub job array accordint to the options and"
                       " configuration that has been supplied")

    parser = argparse.ArgumentParser(description=description)

    parser.add_argument('config', type=str, nargs='?',
                        help="The job config file, options in the config file"
                        " are overwritten by options on the command line")
    parser.add_argument('--project', type=str,
                        help="Use specific project nodes and potentially a "
                        "different policy")
    parser.add_argument('--paid', type=str,
                        help="Force the tasks to use any paid resources "
                        "specified by project")
    parser.add_argument('--step', type=int,
                        help="the step size for the array job")
    parser.add_argument('--batch', type=str,
                        help="the number of concurrent jobs that can be ran"
                        " simultaniously")
    parser.add_argument('--mem', type=str,
                        help="The memory for the job")
    parser.add_argument('--shell', type=str,
                        help="The shell to use for the job")
    parser.add_argument('--time', type=str,
                        help="The max time for the job")
    parser.add_argument('--nodes', type=str, help="restrict the job to "
                        "running on certain nodes")
    parser.add_argument('--scratch_dir', type=str,
                        help="The scratch location")
    parser.add_argument('--tmp_dir', type=str,
                        help="The location or tmp")
    parser.add_argument('--scratch_size', type=str,
                        help="The scratch space that is required")
    parser.add_argument('--tmp_size', type=str,
                        help="The tmp space that is required")
    parser.add_argument('--log_dir', type=str,
                        help="The space to log all the job parameters and "
                        "err/out files")
    parser.add_argument('--ja_file', type=str,
                        help="The location of the job array file")
    parser.add_argument('--task_script', type=str,
                        help="The location of the job array task script")
    parser.add_argument('--infiles', type=str, nargs='+',
                        default=None,
                        help="space separated list of column names in the job"
                        " array file that contain input files to check they"
                        " are present")
    parser.add_argument('--outfiles', type=str, nargs='+',
                        default=None,
                        help="space separated list of column names in the job"
                        " array file that contain output files to check they"
                        " are present")
    parser.add_argument('-v', '--verbose', action='store_true',
                        help="Print out progress")
    parser.add_argument(
        '-w', '--whole-steps', action='store_true',
        help="If step size is > 1 then this ensures that the step size is a"
        " multiple of total tasks and will throw an error if not. This treats"
        " all the rows steps in the job array file as a logical unit and "
        "ensures that all the output files for the steps in the task exists"
        " if not they are tasked for re-running again. If this is not set "
        "then the rows are not treated as logical blocks"
    )

    return parser


# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
def make_python_type(arg):
    """
    convert text based arguments that are none, true or false into a python
    None, True, False

    Parameters
    ----------
    arg :str
        The argument to test and convert if needed

    Returns
    -------
    converted_arg :`any`
        The argment that has been converted if needed
    """
    if re.match('none', arg, re.IGNORECASE):
        return None

    if re.match('true', arg, re.IGNORECASE):
        return True

    if re.match('false', arg, re.IGNORECASE):
        return False

    return arg


# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
def parse_config(config_file):
    """
    Parse the config file options into a dictionary

    Parameters
    ----------
    config_file :str
        The path to a job array config file

    Returns
    -------
    config_options :dict
        The configuration options
    """
    config_options = {}
    config = configparser.ConfigParser()
    config_file = os.path.realpath(os.path.expanduser(config_file))

    # To generate an error if the file does not exist or is None
    open(config_file).close()

    # If we get here then we can read in
    config.read(config_file)

    # Loop through the section headings in the config files, and all the
    # variables that can be present
    for section, section_args in [('qsub', QSUB_ARGS),
                                  ('logs', LOG_ARGS),
                                  ('job_array', JOB_ARRAY_ARGS),
                                  ('script', SCRIPT_ARGS)]:
        for arg, default, required, quote in section_args:
            try:
                # attempt to process the argument and make none, true/false
                # into python types
                config_options[arg] = make_python_type(config[section][arg])
            except KeyError:
                # if we get a KeyError then it is not in the config file
                pass

    # Now process any infiles/outfiles options, these are comma separated lists
    # of column headings in the job array file that will be checked for the
    # presence of input/output files. These are processed into a list
    for i in ['infiles', 'outfiles']:
        try:
            # This strange suntax is to allow for the possibility of internal
            # commas to be protexted by quotes ""
            config_options[i] = [c.strip() for c in next(csv.reader(
                [config_options[i]], delimiter=','))]
        except KeyError:
            # option not present
            config_options[i] = []

    return config_options


# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
def parse_args(args):
    """
    Parse the command line arguments and the config file arguments (if a
    config file is present). Command line arguments overide config file
    arguments

    Parameters
    ----------
    parser : :obj:`argparse.ArgumentParser`
        A parser with all the arguments in it

    Returns
    -------
    config_options : dict
        The composite of any command line arguments and configuration options
    """
    try:
        # Attempt to parse the config file
        config_options = parse_config(args.config)
    except TypeError:
        # No config supplied
        config_options = {}

    try:
        config_options['verbose']
    except KeyError:
        config_options['verbose'] = False

    try:
        config_options['whole_steps']
    except KeyError:
        config_options['whole_steps'] = False

    config_options['verbose'] = config_options['verbose'] | args.verbose

    try:
        config_options['whole_steps'] = \
            config_options['whole_steps'] | args.whole_steps
    except AttributeError:
        # the args could be incomming from a script that does not support
        # whole steps so in this case we will set to False
        config_options['whole_steps'] = False

    # Loop through the options in the command line args and merge them into
    # the arguments parsed out of the config file
    for section, section_args in [('qsub', QSUB_ARGS),
                                  ('logs', LOG_ARGS),
                                  ('job_array', JOB_ARRAY_ARGS)]:
        for arg, default, required, quote in section_args:
            try:
                # Now see if we have supplied an overriding option on the
                # command line
                cmd_arg = getattr(args, arg)
            except AttributeError:
                # Here for arguments that are in the config file but not on the
                # command line, there are not any at the moment though
                continue

            # If the command line argument has been set then overide from
            # configuration file options
            if cmd_arg is not None:
                config_options[arg] = cmd_arg

    # pass the verbose argument through to the config options as it is not
    # covered in the config file
    # if config_options['verbose'] is False and config_verbose is True:
    #     config_options['verbose'] = True

    return config_options


# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
def parse_mappings(mapping_file):
    """
    parse the mapping file
    """
    # Make sure we get the full path
    mapping_file = os.path.realpath(os.path.expanduser(mapping_file))

    # To generate an error if the file does not exist or is None
    open(mapping_file).close()

    if check_mapping_permissions(mapping_file) is False:
        raise PermissionError("mapping file '{0}' should only be readable "
                              "by you".format(os.environ[QSUB_PYTHON_MAPPING]))

    config = configparser.ConfigParser()

    # If we get here then we can read in
    config.read(mapping_file)
    mappings = {}

    for k, v in config['mappings'].items():
        # there could be multiple comma separated arguments
        qsub_opts = [c.strip() for c in next(csv.reader([v], delimiter=','))]
        mappings[k] = qsub_opts

    return mappings


# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
def check_mapping_permissions(mapping_file):
    """
    Check the mappings file is only user accessible.

    Parameters
    ----------

    cache_file :`str`
        The file name of the cache file

    Returns
    -------

    ok :`bool`
        True if the permissions are ok False if not
    """
    st = os.stat(mapping_file)
    return not bool(st.st_mode & (S_IRWXO | S_IRWXG))


# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
def init_job_run(config_options):
    """
    """
    msg = common.Msg(verbose=config_options['verbose'], file=sys.stderr)

    try:
        mappings = parse_mappings(os.environ[QSUB_PYTHON_MAPPING])
        config_options['qsub_mappings'] = mappings
    except KeyError as e:
        raise KeyError("no mapping config available, please set the "
                       "ENVIRONMENT variable '{0}' in your"
                       " ~/.bashrc with the location of your mapping"
                       " configuration".format(QSUB_PYTHON_MAPPING)) from e

    # Now set up the job array, initialise and run
    archive = False
    with JobArray(**config_options) as job_array:
        try:
            job_array.initialise()
            job_array.run()
        except JobCompleteError:
            # Offer archive
            archive = True

    if archive is True:
        try:
            print("[archive] do you want to archive the run files, press [ENTER] to archive or"
                  " [CTRL-C] to quit > ")
            input()
            outtar = '{0}.tar.gz'.format(job_array.log_dir)
            msg.msg("[archive] archiving to '{0}'".format(outtar))
            with tarfile.open(outtar, "w:gz") as tar:
                tar.add(job_array.log_dir,
                        arcname=os.path.basename(job_array.log_dir))
            shutil.rmtree(job_array.log_dir)
        except KeyboardInterrupt:
            pass

    # Final end message
    msg.msg("*** END ***")


# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
def main():
    """
    The main entry point for the script
    """
    # Initialise all the command line arguments
    parser = _init_cmd_args()

    # Now parse any command line argumanets and any supplied config file
    args = parser.parse_args()
    config_options = parse_args(args)
    msg = common.Msg(verbose=config_options['verbose'], file=sys.stderr)
    msg.msg('=== pyjasub ({0} v{1}) ==='.format(PKG_NAME, __version__))
    init_job_run(config_options)


# @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
if __name__ == '__main__':
    main()
