#!/usr/bin/env python

# Copyright (C) 2015 UCSC Computational Genomics Lab
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import absolute_import
import os
import sys
import importlib
from argparse import ArgumentParser
import xml.etree.cElementTree as ET
from abc import ABCMeta, abstractmethod
import tempfile
import uuid
import time
import copy_reg
import cPickle
import logging

from bd2k.util.humanize import human2bytes
from io import BytesIO

from toil.resource import ModuleDescriptor
from toil.common import loadJobStore

logger = logging.getLogger( __name__ )

from toil.lib.bioio import (setLoggingFromOptions,
                               getTotalCpuTimeAndMemoryUsage, getTotalCpuTime)
from toil.common import setupToil, addOptions
from toil.leader import mainLoop

class JobException( Exception ):
    def __init__( self, message ):
        super( JobException, self ).__init__( message )

class Job(object):
    """
    Represents a unit of work in toil. Jobs are composed into graphs
    which make up a workflow. 
    
    This public functions of this class and its  nested classes are the API 
    to toil.
    """
    def __init__(self, memory=None, cores=None, disk=None):
        """
        This method must be called by any overiding constructor.
        
        Memory is the maximum number of bytes of memory the job will
        require to run. Cores is the number of CPU cores required.
        """
        self.cores = cores
        self.memory = human2bytes(str(memory)) if memory is not None else memory
        self.disk = human2bytes(str(disk)) if disk is not None else disk
        #Private class variables

        #See Job.addChild
        self._children = []
        #See Job.addFollowOn
        self._followOns = []
        #See Job.addService
        self._services = []
        #A follow-on, service or child of a job A, is a "successor" of A, if B
        #is a successor of A, then A is a predecessor of B.
        self._predecessors = set()
        # Note that self.__module__ is not necessarily this module, i.e. job.py. It is the module
        # defining the class self is an instance of, which may be a subclass of Job that may be
        # defined in a different module.
        self.userModule = ModuleDescriptor.forModule(self.__module__)
        #See Job.rv()
        self._rvs = {}

    def run(self, fileStore):
        """
        Do user stuff here, including creating any follow on jobs.
        
        The fileStore argument is an instance of Job.FileStore, and can
        be used to create temporary files which can be shared between jobs.
        
        The return values of the function can be passed to other jobs
        by means of the rv() function.  
        
        Note: We disallow return values to be PromisedJobReturnValue instances
        (generated by the Job.rv() function - see below).
        A check is made that will result in a runtime error if you attempt to do this.
        Allowing PromisedJobReturnValue instances to be returned does not work because
        the mechanism to pass the promise uses a jobStoreFileID that will be deleted once
        the current job and its descendants have been completed. This is similar to
        scope rules in a language like C, where returning a reference to memory allocated
        on the stack within a function will produce an undefined reference. 
        Disallowing this also avoids nested promises (PromisedJobReturnValue
        instances that contain other PromisedJobReturnValue).
        """
        pass

    def addChild(self, childJob):
        """
        Adds the child job to be run as child of this job. Returns childJob.
        Child jobs are run after the Job.run method has completed.
        
        See Job.checkJobGraphAcylic for formal definition of allowed forms of
        job graph.
        """
        self._children.append(childJob)
        childJob._addPredecessor(self)
        return childJob

    def addService(self, service):
        """
        Add a service of type Job.Service. The Job.Service.start() method
        will be called after the run method has completed but before any successors 
        are run. It's Job.Service.stop() method will be called once the
        successors of the job have been run.
        
        :rtype : An instance of PromisedJobReturnValue which will be replaced
        with the return value from the service.start() in any successor of the job.
        """
        jobService = ServiceJob(service)
        self._services.append(jobService)
        return jobService.rv()

    def addFollowOn(self, followOnJob):
        """
        Adds a follow-on job, follow-on jobs will be run
        after the child jobs and their descendants have been run.
        Returns followOnJob.
        
        See Job.checkJobGraphAcylic for formal definition of allowed forms of
        job graph.
        """
        self._followOns.append(followOnJob)
        followOnJob._addPredecessor(self)
        return followOnJob

    ##Convenience functions for creating jobs

    def addChildFn(self, fn, *args, **kwargs):
        """
        Adds a child fn. See FunctionWrappingJob.
        Returns the new child Job.
        """
        return self.addChild(FunctionWrappingJob(fn, *args, **kwargs))

    def addChildJobFn(self, fn, *args, **kwargs):
        """
        Adds a child job fn. See JobFunctionWrappingJob.
        Returns the new child Job.
        """
        return self.addChild(JobFunctionWrappingJob(fn, *args, **kwargs))

    def addFollowOnFn(self, fn, *args, **kwargs):
        """
        Adds a follow-on fn. See FunctionWrappingJob.
        Returns the new follow-on Job.
        """
        return self.addFollowOn(FunctionWrappingJob(fn, *args, **kwargs))

    def addFollowOnJobFn(self, fn, *args, **kwargs):
        """
        Add a follow-on job fn. See JobFunctionWrappingJob.
        Returns the new follow-on Job.
        """
        return self.addFollowOn(JobFunctionWrappingJob(fn, *args, **kwargs))

    @staticmethod
    def wrapJobFn(fn, *args, **kwargs):
        """
        Makes a Job out of a job function.
        
        Convenience function for constructor of JobFunctionWrappingJob
        """
        return JobFunctionWrappingJob(fn, *args, **kwargs)

    @staticmethod
    def wrapFn(fn, *args, **kwargs):
        """
        Makes a Job out of a function.
        
        Convenience function for constructor of FunctionWrappingJob
        """
        return FunctionWrappingJob(fn, *args, **kwargs)

    def encapsulate(self):
        """
        See EncapsulatedJob.
        
        :rtype : A new EncapsulatedJob for this job.
        """
        return EncapsulatedJob(self)

    ####################################################
    #The following function is used for passing return values between
    #job run functions
    ####################################################

    def rv(self, argIndex=None):
        """
        Gets a PromisedJobReturnValue, representing the argIndex return
        value of the run function (see run method for description).
        This PromisedJobReturnValue, if a class attribute of a Job instance,
        call it T, will be replaced by the actual return value when the 
        T is loaded. The function rv therefore allows the output 
        from one Job to be wired as input to another Job before either
        is actually run.  
        
        :param argIndex: If None the complete return value will be returned, if argIndex
        is an integer it is used to refer to the return value as indexable 
        (tuple/list/dictionary, or in general object that implements __getitem__), 
        hence rv(i) would refer to the ith (indexed from 0) member of return value.
        """
        #Check if the return value has already been promised and if it has
        #return it
        if argIndex in self._rvs:
            return self._rvs[argIndex]
        #Create, store, return new PromisedJobReturnValue
        self._rvs[argIndex] = PromisedJobReturnValue()
        return self._rvs[argIndex]

    ####################################################
    #Cycle/connectivity checking
    ####################################################

    def checkJobGraphForDeadlocks(self):
        """
        Raises a JobGraphDeadlockException exception if the job graph
        is cyclic or contains multiple roots.
        """
        self.checkJobGraphConnected()
        self.checkJobGraphAcylic()

    def getRootJobs(self):
        """
        A root is a job with no predecessors.
        :rtype : set, the roots of the connected component of jobs that
        contains this job.
        """
        roots = set()
        visited = set()
        #Function to get the roots of a job
        def getRoots(job):
            if job not in visited:
                visited.add(job)
                if len(job._predecessors) > 0:
                    map(lambda p : getRoots(p), job._predecessors)
                else:
                    roots.add(job)
                #The following call ensures we explore all successor edges.
                map(lambda c : getRoots(c), job._children +
                    job._followOns + job._services)
        getRoots(self)
        return roots

    def checkJobGraphConnected(self):
        """
        Raises a JobGraphDeadlockException exception if getRootJobs() does not
        contain exactly one root job.
        As execution always starts from one root job, having multiple root jobs will
        cause a deadlock to occur.
        """
        rootJobs = self.getRootJobs()
        if len(rootJobs) != 1:
            raise JobGraphDeadlockException("Graph does not contain exactly one root job: %s" % rootJobs)

    def checkJobGraphAcylic(self):
        """
        Raises a JobGraphDeadlockException exception if the connected component
        of jobs containing this job contains any cycles of child/followOn dependencies
        in the augmented job graph (see below). Such cycles are not allowed
        in valid job graphs. This function is run during execution.
        
        A job B that is on a directed path of child/followOn edges from a
        job A in the job graph is a descendant of A,
        similarly A is an ancestor of B.
        
        A follow-on edge (A, B) between two jobs A and B is equivalent
        to adding a child edge to B from (1) A, (2) from each child of A, 
        and (3) from the descendants of each child of A. We
        call such an edge an "implied" edge. The augmented job graph is a
        job graph including all the implied edges.

        For a job (V, E) the algorithm is O(|V|^2). It is O(|V| + |E|) for
        a graph with no follow-ons. The former follow on case could be improved!
        """
        #Get the root jobs
        roots = self.getRootJobs()
        if len(roots) == 0:
            raise JobGraphDeadlockException("Graph contains no root jobs due to cycles")

        #Get implied edges
        extraEdges = self._getImpliedEdges(roots)

        #Check for directed cycles in the augmented graph
        visited = set()
        for root in roots:
            root._checkJobGraphAcylicDFS([], visited, extraEdges)

    ####################################################
    #The following nested classes are used for
    #creating jobtrees (Job.Runner),
    #managing temporary files (Job.FileStore),
    #and defining a service (Job.Service)
    ####################################################

    class Runner(object):
        """
        Used to setup and run a graph of jobs.
        """

        @staticmethod
        def getDefaultOptions(jobStore):
            """
            Returns an optparse.Values object of the 
            options used by a toil.
            """
            parser = ArgumentParser()
            Job.Runner.addToilOptions(parser)
            options = parser.parse_args(args=[jobStore])
            return options

        @staticmethod
        def addToilOptions(parser):
            """
            Adds the default toil options to an optparse or argparse
            parser object.
            """
            addOptions(parser)

        @staticmethod
        def startToil(job, options):
            """
            Runs the toil workflow using the given options 
            (see Job.Runner.getDefaultOptions and Job.Runner.addToilOptions) 
            starting with this job. 
            
            :raises: toil.leader.FailedJobsException if at the end of function their remain
    failed jobs
            """
            setLoggingFromOptions(options)
            with setupToil(options, userScript=job.getUserScript()) as (config, batchSystem, jobStore):
                if options.restart:
                    jobStore.clean() #This cleans up any half written jobs after a restart
                    rootJob = job._loadRootJob(jobStore)
                else:
                    #Setup the first wrapper.
                    rootJob = job._serialiseFirstJob(jobStore)
                return mainLoop(config, batchSystem, jobStore, rootJob)

    class FileStore:
        """
        Class used to manage temporary files and log messages, 
        passed as argument to the Job.run method.
        """

        def __init__(self, jobStore, jobWrapper, localTempDir):
            """
            This constructor should not be called by the user, 
            FileStore instances are only provided as arguments 
            to the run function.
            """
            self.jobStore = jobStore
            self.jobWrapper = jobWrapper
            self.localTempDir = localTempDir
            self.loggingMessages = []

        def writeGlobalFile(self, localFileName):
            """
            Takes a file (as a path) and uploads it to to the global file store, returns
            an ID that can be used to retrieve the file. 
            """
            return self.jobStore.writeFile(self.jobWrapper.jobStoreID, localFileName)

        def updateGlobalFile(self, fileStoreID, localFileName):
            """
            Replaces the existing version of a file in the global file store, 
            keyed by the fileStoreID. 
            Throws an exception if the file does not exist.
            """
            self.jobStore.updateFile(fileStoreID, localFileName)

        def readGlobalFile(self, fileStoreID, localFilePath=None):
            """
            Returns a path to a local copy of the file keyed by fileStoreID. 
            The version will be consistent with the last copy of the file 
            written/updated to the global file store. If localFilePath is not None, 
            the returned file path will be localFilePath.
            """
            if localFilePath is None:
                fd, localFilePath = tempfile.mkstemp(dir=self.getLocalTempDir())
                self.jobStore.readFile(fileStoreID, localFilePath)
                os.close(fd)
            else:
                self.jobStore.readFile(fileStoreID, localFilePath)
            return localFilePath

        def deleteGlobalFile(self, fileStoreID):
            """
            Deletes a global file with the given fileStoreID. Returns true if 
            file exists, else false.
            """
            return self.jobStore.deleteFile(fileStoreID)

        def writeGlobalFileStream(self):
            """
            Similar to writeGlobalFile, but returns a context manager yielding a 
            tuple of 1) a file handle which can be written to and 2) the ID of 
            the resulting file in the job store. The yielded file handle does
            not need to and should not be closed explicitly.
            """
            return self.jobStore.writeFileStream(self.jobWrapper.jobStoreID)

        def updateGlobalFileStream(self, fileStoreID):
            """
            Similar to updateGlobalFile, but returns a context manager yielding 
            a file handle which can be written to. The yielded file handle does 
            not need to and should not be closed explicitly.
            """
            return self.jobStore.updateFileStream(fileStoreID)

        def getEmptyFileStoreID(self):
            """
            Returns the ID of a new, empty file.
            """
            return self.jobStore.getEmptyFileStoreID(self.jobWrapper.jobStoreID)

        def globalFileExists(self, fileStoreID):
            """
            :rtype : True if and only if the jobStore contains the given fileStoreID, else
            false.
            """
            return self.jobStore.fileExists(fileStoreID)

        def readGlobalFileStream(self, fileStoreID):
            """
            Similar to readGlobalFile, but returns a context manager yielding a 
            file handle which can be read from. The yielded file handle does not 
            need to and should not be closed explicitly.
            """
            return self.jobStore.readFileStream(fileStoreID)

        def getLocalTempDir(self):
            """
            Get the local temporary directory. This directory will exist for the 
            duration of the job only, and is guaranteed to be deleted once
            the job terminates.
            """
            return self.localTempDir

        def logToMaster(self, string):
            """
            Send a logging message to the leader. Will only ne reported if logging 
            is set to INFO level (or lower) in the leader.
            """
            self.loggingMessages.append(str(string))

    class Service:
        """
        Abstract class used to define the interface to a service.
        """
        __metaclass__ = ABCMeta
        def __init__(self, memory=None, cores=None):
            """
            Memory and core requirements are specified identically to the Job
            constructor.
            """
            self.memory = memory
            self.cores = cores

        @abstractmethod
        def start(self):
            """
            Start the service.
            
            :rtype : An object describing how to access the service. Must be 
            pickleable. Will be used by a job to access the service.
            """
            pass

        @abstractmethod
        def stop(self):
            """
            Stops the service.
            
            Function can block until complete. 
            """
            pass

    ####################################################
    #Private functions
    ####################################################

    def _addPredecessor(self, predecessorJob):
        """
        Adds a predecessor job to the set of predecessor jobs. Raises a
        RuntimeError is the job is already a predecessor.
        """
        if predecessorJob in self._predecessors:
            raise RuntimeError("The given job is already a predecessor of this job")
        self._predecessors.add(predecessorJob)

    ####################################################
    #The following functions are used to serialise
    #a job graph to the jobStore
    ####################################################

    def _getHashOfJobsToUUIDs(self, jobsToUUIDs):
        """
        Creates a map of the jobs in the graph to randomly selected UUIDs.
        Excludes the root job.
        """
        #Call recursively
        for successor in self._children + self._followOns:
            successor._getHashOfJobsToUUIDs2(jobsToUUIDs)
        return jobsToUUIDs

    def getUserScript(self):
        return self.userModule

    def _getHashOfJobsToUUIDs2(self, jobsToUUIDs):
        if self not in jobsToUUIDs:
            jobsToUUIDs[self] = str(uuid.uuid1())
            self._getHashOfJobsToUUIDs(jobsToUUIDs)

    def _createEmptyJobForJob(self, jobStore, updateID=None, command=None,
                                 predecessorNumber=0):
        """
        Create an empty job for the job.
        """
        return jobStore.create(command=command,
                               memory=(self.memory if self.memory is not None
                                       else jobStore.config.defaultMemory),
                               cores=(self.cores if self.cores is not None
                                    else float(jobStore.config.defaultCores)),
                               disk=(self.disk if self.disk is not None
                                    else float(jobStore.config.defaultDisk)),
                               updateID=updateID, predecessorNumber=predecessorNumber)

    def _makeJobWrappers(self, jobStore, jobsToUUIDs, jobsToJobs, predecessor, rootJob):
        """
        Creates a job for each job in the job graph, recursively.
        """
        if self not in jobsToJobs:
            #The job for the job
            assert predecessor in self._predecessors
            jobWrapper = self._createEmptyJobForJob(jobStore, jobsToUUIDs[self],
                                                predecessorNumber=len(self._predecessors))
            jobsToJobs[self] = jobWrapper

            #Add followOns/children to be run after the current job.
            for successors in (self._followOns, self._children):
                jobs = map(lambda successor:
                    successor._makeJobWrappers(jobStore, jobsToUUIDs,
                                               jobsToJobs, self, rootJob), successors)
                if len(jobs) > 0:
                    jobWrapper.stack.append(jobs)

            #Pickle the job so that its run method can be run at a later time.
            #Drop out the children/followOns/predecessors/services - which are
            #all recored within the jobStore and do not need to be stored within
            #the job
            self._children = []
            self._followOns = []
            self._services = []
            self._predecessors = set()
            #The pickled job is "run" as the command of the job, see worker
            #for the mechanism which unpickles the job and executes the Job.run
            #method.
            fileStoreID = jobStore.getEmptyFileStoreID(rootJob.jobStoreID)
            with jobStore.writeFileStream(rootJob.jobStoreID) as (fileHandle, fileStoreID):
                cPickle.dump(self, fileHandle, cPickle.HIGHEST_PROTOCOL)
            jobWrapper.command = ' '.join( ('_toil', fileStoreID) + self.userModule.globalize())
            #Update the status of the job on disk
            jobStore.update(jobWrapper)
        else:
            #Lookup the already created job
            jobWrapper = jobsToJobs[self]
            assert jobWrapper.predecessorNumber > 1

        #The return is a tuple stored within the job.stack of the jobs to run.
        #The tuple is jobStoreID, memory, cores, disk, predecessorID
        #The predecessorID is used to establish which predecessors have been
        #completed before running the given Job - it is just a unique ID
        #per predecessor
        return (jobWrapper.jobStoreID, jobWrapper.memory, jobWrapper.cores, jobWrapper.disk,
                None if jobWrapper.predecessorNumber <= 1 else str(uuid.uuid4()))

    def _serialiseJobGraph(self, jobWrapper, jobStore):
        """
        Serialises the graph of jobs rooted at this job,
        storing them in the jobStore.
        Assumes the root job is already in the jobStore.
        """
        #Create jobIDs as UUIDs
        jobsToUUIDs = self._getHashOfJobsToUUIDs({})
        #Set the jobs to delete
        jobWrapper.jobsToDelete = list(jobsToUUIDs.values())
        #Update the job on disk. The jobs to delete is a record of what to
        #remove if the update goes wrong
        jobStore.update(jobWrapper)
        #Create the jobs for followOns/children
        jobsToJobs = {}
        for successors in (self._followOns, self._children):
            jobs = map(lambda successor:
                successor._makeJobWrappers(jobStore, jobsToUUIDs,
                                           jobsToJobs, self, jobWrapper), successors)
            if len(jobs) > 0:
                jobWrapper.stack.append(jobs)
        #Remove the jobs to delete list and remove the old command finishing the update
        jobWrapper.jobsToDelete = []
        jobWrapper.command = None
        jobStore.update(jobWrapper)

    def _serialiseFirstJob(self, jobStore):
        """
        Serialises the root job. Returns the wrapping job.
        """
        #Pickles the job within a shared file in the jobStore called
        #"firstJob"
        sharedJobFile = "firstJob"
        with jobStore.writeSharedFileStream(sharedJobFile) as f:
            cPickle.dump(self, f, cPickle.HIGHEST_PROTOCOL)
        #Make the first job
        command = ' '.join(('_toil',sharedJobFile) + self.userModule.globalize())
        jobWrapper = self._createEmptyJobForJob(jobStore, command=command)
        #Store the name of the first job in a file in case of restart
        with jobStore.writeSharedFileStream("rootJobStoreID") as f:
            f.write(jobWrapper.jobStoreID)
        #Return the first job wrapper
        return jobWrapper

    @staticmethod
    def _loadRootJob(jobStore):
        """
        Loads the root job.
        :throws JobException: If root job is not in the job store. 
        """
        with jobStore.readSharedFileStream("rootJobStoreID") as f: #Load the root job
            rootJobID = f.read()
        if not jobStore.exists(rootJobID):
            raise JobException("No root job (%s) left in toil workflow (workflow has finished successfully?)" % rootJobID)
        return jobStore.load(rootJobID)

    @classmethod
    def _loadUserModule(cls, userModule):
        """
        Imports and returns the module object represented by the given module descriptor.

        :type userModule: ModuleDescriptor
        """
        if not userModule.belongsToToil:
            userModule = userModule.localize()
        if userModule.dirPath not in sys.path:
            sys.path.append(userModule.dirPath)
        return importlib.import_module(userModule.name)

    @classmethod
    def _loadJob(cls, command, jobStore):
        """
        Unpickles a job.Job instance by decoding the command. See job.Job._serialiseFirstJob and
        job.Job._makeJobWrappers to see how the Job is encoded in the command. Essentially the
        command is a reference to a jobStoreFileID containing the pickle file for the job and a
        list of modules which must be imported so that the Job can be successfully unpickled.
        """
        commandTokens = command.split()
        assert "_toil" == commandTokens[0]
        userModule = ModuleDescriptor(*(commandTokens[2:]))
        userModule = cls._loadUserModule(userModule)
        pickleFile = commandTokens[1]
        if pickleFile == "firstJob":
            openFileStream = jobStore.readSharedFileStream(pickleFile)
        else:
            openFileStream = jobStore.readFileStream(pickleFile)
        with openFileStream as fileHandle:
            return cls._unpickle(userModule, fileHandle)

    @classmethod
    def _unpickle(cls, userModule, fileHandle):
        """
        Unpickles an object graph from the given file handle while loading symbols referencing
        the __main__ module from the given userModule instead.

        :param userModule:
        :param fileHandle:
        :return:
        """
        unpickler = cPickle.Unpickler(fileHandle)

        def filter_main(module_name, class_name):
            if module_name == '__main__':
                return getattr(userModule, class_name)
            else:
                return getattr(importlib.import_module(module_name), class_name)

        unpickler.find_global = filter_main
        return unpickler.load()

    ####################################################
    #Functions to pass Job.run return values to the
    #input arguments of other Job instances
    ####################################################

    def _setFileIDsForPromisedValues(self, jobStore, jobStoreID, visited):
        """
        Sets the jobStoreFileID for each PromisedJobReturnValue in the
        graph of jobs created.
        """
        #Replace None references with valid jobStoreFileIDs. We
        #do this here, rather than within the original constructor of the
        #promised value because we don't necessarily have access to the jobStore when
        #the PromisedJobReturnValue instances are created.
        if self not in visited:
            visited.add(self)
            for i in self._rvs.keys():
                promisedJobReturnValue = self._rvs[i]
                #Instances of PromisedJobReturnValue are replaced with jobStoreFileIDs
                if isinstance(promisedJobReturnValue, PromisedJobReturnValue):
                    promisedJobReturnValue.jobStoreFileID = jobStore.getEmptyFileStoreID(jobStoreID)
                    promisedJobReturnValue.jobStoreString = jobStore.config.jobStore
                    self._rvs[i] = promisedJobReturnValue.jobStoreFileID
            #Now recursively do the same for the children and follow ons.
            for successorJob in self._children + self._followOns + self._services:
                successorJob._setFileIDsForPromisedValues(jobStore, jobStoreID, visited)

    @staticmethod
    def _setReturnValuesForPromises(job, returnValues, jobStore):
        """
        Sets the values for promises using the return values from the job's
        run function.
        """
        for i in job._rvs.keys():
            if i == None:
                argToStore = returnValues
            else:
                argToStore = returnValues[i]
            with jobStore.updateFileStream(job._rvs[i]) as fileHandle:
                if isinstance(argToStore, PromisedJobReturnValue):
                    raise RuntimeError("A nested PromisedJobReturnValue has been found.") #We do not allow the return of PromisedJobReturnValue instance from the run function
                cPickle.dump(argToStore, fileHandle, cPickle.HIGHEST_PROTOCOL)

    ####################################################
    #Functions associated with Job.checkJobGraphAcyclic to establish
    #that the job graph does not contain any cycles of dependencies.
    ####################################################

    def _dfs(self, visited):
        """Adds the job and all jobs reachable on a directed path from current
        node to the set 'visited'.
        """
        if self not in visited:
            visited.add(self)
            for successor in self._children + self._followOns:
                successor._dfs(visited)

    def _checkJobGraphAcylicDFS(self, stack, visited, extraEdges):
        """
        DFS traversal to detect cycles in augmented job graph.
        """
        if self not in visited:
            visited.add(self)
            stack.append(self)
            for successor in self._children + self._followOns + extraEdges[self]:
                successor._checkJobGraphAcylicDFS(stack, visited, extraEdges)
            assert stack.pop() == self
        if self in stack:
            stack.append(self)
            raise JobGraphDeadlockException("A cycle of job dependencies has been detected '%s'" % stack)

    @staticmethod
    def _getImpliedEdges(roots):
        """
        Gets the set of implied edges. See Job.checkJobGraphAcylic
        """
        #Get nodes in job graph
        nodes = set()
        for root in roots:
            root._dfs(nodes)

        ##For each follow-on edge calculate the extra implied edges
        #Adjacency list of implied edges, i.e. map of jobs to lists of jobs
        #connected by an implied edge
        extraEdges = dict(map(lambda n : (n, []), nodes))
        for job in nodes:
            if len(job._followOns) > 0:
                #Get set of jobs connected by a directed path to job, starting
                #with a child edge
                reacheable = set()
                for child in job._children:
                    child._dfs(reacheable)
                #Now add extra edges
                for descendant in reacheable:
                    extraEdges[descendant] += job._followOns[:]
        return extraEdges

    def _modifyJobGraphForServices(self, fileStore):
        """
        Modifies the job graph to correctly schedule any services
        defined for this job.
        """
        if len(self._services) > 0:
            #Set the start/stop jobStore fileIDs for each service
            for service in self._services:
                service.startFileStoreID = fileStore.getEmptyFileStoreID()
                assert fileStore.globalFileExists(service.startFileStoreID)
                service.stopFileStoreID = fileStore.getEmptyFileStoreID()
                assert fileStore.globalFileExists(service.stopFileStoreID)

            def removePredecessor(job):
                assert self in job._predecessors
                job._predecessors.remove(self)

            #t1 and t2 are used to run the children and followOns of the job
            #after the services of the job are started
            startFileStoreIDs = map(lambda i : i.startFileStoreID, self._services)
            t1, t2 = Job.wrapJobFn(blockUntilDeleted, startFileStoreIDs), Job()
            #t1 runs the children of the job
            for child in self._children:
                removePredecessor(child)
                t1.addChild(child)
            self._children = []
            #t2 runs the followOns of the job
            for followOn in self._followOns:
                removePredecessor(followOn)
                t2.addChild(followOn)
            self._followOns = []
            #Wire up the self, t1 and t2
            self.addChild(t1)
            t1.addFollowOn(t2)
            #Now make the services children of the job
            for service in self._services:
                self.addChild(service)
                assert service._predecessors == set((self,))
            #The final task once t1 and t2 have finished is to stop the services
            #this is achieved by deleting the stopFileStoreIDs.
            t2.addFollowOnJobFn(deleteFileStoreIDs, map(lambda i : i.stopFileStoreID, self._services))
            self._services = [] #Defensive

    ####################################################
    #Function which worker calls to ultimately invoke
    #a jobs Job.run method, and then handle created
    #children/followOn jobs
    ####################################################

    def _execute(self, jobWrapper, stats, localTempDir, jobStore):
        """This is the core method for running the job within a worker.
        """
        if stats != None:
            startTime = time.time()
            startClock = getTotalCpuTime()

        baseDir = os.getcwd()
        #Run the job, first cleanup then run.
        fileStore = Job.FileStore(jobStore, jobWrapper, localTempDir)
        returnValues = self.run(fileStore)
        #Modify job graph to run any services correctly
        self._modifyJobGraphForServices(fileStore)
        #Check if the job graph has created
        #any cycles of dependencies or has multiple roots
        self.checkJobGraphForDeadlocks()
        #Set the promised value jobStoreFileIDs
        self._setFileIDsForPromisedValues(jobStore, jobWrapper.jobStoreID, set())
        #Store the return values for any promised return value
        self._setReturnValuesForPromises(self, returnValues, jobStore)
        #Turn the graph into a graph of jobs in the jobStore
        self._serialiseJobGraph(jobWrapper, jobStore)
        #Change dir back to cwd dir, if changed by job (this is a safety issue)
        if os.getcwd() != baseDir:
            os.chdir(baseDir)
        #Finish up the stats
        if stats != None:
            stats = ET.SubElement(stats, "job")
            stats.attrib["time"] = str(time.time() - startTime)
            totalCpuTime, totalMemoryUsage = getTotalCpuTimeAndMemoryUsage()
            stats.attrib["clock"] = str(totalCpuTime - startClock)
            stats.attrib["class"] = self._jobName()
            stats.attrib["memory"] = str(totalMemoryUsage)
        #Return any logToMaster logging messages
        return fileStore.loggingMessages

    ####################################################
    #Method used to resolve the module in which an inherited job instances
    #class is defined
    ####################################################

    @staticmethod
    def _resolveMainModule( moduleName ):
        """
        Returns a tuple of two elements, the first element being the path 
        to the directory containing the given
        module and the second element being the name of the module. 
        If the given module name is "__main__",
        then that is translated to the actual file name of the top-level 
        script without .py or .pyc extensions. The
        caller can then add the first element of the returned tuple to 
        sys.path and load the module from there. See also worker.loadJob().
        """
        # looks up corresponding module in sys.modules, gets base name, drops .py or .pyc
        moduleDirPath, moduleName = os.path.split(os.path.abspath(sys.modules[moduleName].__file__))
        if moduleName.endswith('.py'):
            moduleName = moduleName[:-3]
        elif moduleName.endswith('.pyc'):
            moduleName = moduleName[:-4]
        else:
            raise RuntimeError(
                "Can only handle main modules loaded from .py or .pyc files, but not '%s'" %
                moduleName)
        return moduleDirPath, moduleName

    def _jobName(self):
        """
        :rtype : string, used as identifier of the job class in the stats report. 
        """
        return self.__class__.__name__

class JobGraphDeadlockException( Exception ):
    def __init__( self, string ):
        super( JobGraphDeadlockException, self ).__init__( string )

class FunctionWrappingJob(Job):
    """
    Job used to wrap a function.
    
    If dill is installed 
    Function can be nested function class function, currently.
    *args and **kwargs are used as the arguments to the function.
    """
    def __init__(self, userFunction, *args, **kwargs):
        # FIXME: I'd rather not duplicate the defaults here, unless absolutely necessary
        cores = kwargs.pop("cores") if "cores" in kwargs else None
        disk = kwargs.pop("disk") if "disk" in kwargs else None
        memory = kwargs.pop("memory") if "memory" in kwargs else None
        Job.__init__(self, memory=memory, cores=cores, disk=disk)
        #If dill is installed pickle the user function directly

        #else use indirect method
        self.userFunctionModule = ModuleDescriptor.forModule(userFunction.__module__).globalize()
        self.userFunctionName = str(userFunction.__name__)
        self._args=args
        self._kwargs=kwargs

    def _getUserFunction(self):
        userFunctionModule = self._loadUserModule(self.userFunctionModule)
        return getattr(userFunctionModule, self.userFunctionName)


    def run(self,fileStore):
        userFunction = self._getUserFunction( )
        return userFunction(*self._args, **self._kwargs)

    def getUserScript(self):
        return self.userFunctionModule

    def _jobName(self):
        return ".".join((self.__class__.__name__,self.userFunctionModule.name,self.userFunctionName))

class JobFunctionWrappingJob(FunctionWrappingJob):
    """
    Job used to wrap a function.
    A job function is a function which takes as its first argument a reference
    to the wrapping job.
    
    To enable the job function to get access to the Job.FileStore
    instance (see Job.Run), it is made a variable of the wrapping job, so in the wrapped
    job function the attribute "fileStore" of the first argument (the job) is
    an instance of the Job.FileStore class.
    """

    def __init__(self, userFunction, *args, **kwargs):
        super(JobFunctionWrappingJob, self).__init__(userFunction, *args, **kwargs)
        self.fileStore = None

    def run(self, fileStore):
        userFunction = self._getUserFunction()
        self.fileStore = fileStore
        return userFunction(*((self,) + tuple(self._args)), **self._kwargs)

class ServiceJob(Job):
    """
    Job used to wrap a Job.Service instance. This constructor should not be called by a user.
    """
    def __init__(self, service):
        """
        :type service: Job.Service
        """
        Job.__init__(self, memory=service.memory, cores=service.cores)
        # service.__module__ is the module defining the class service is an instance of.
        self.serviceModule = ModuleDescriptor.forModule(service.__module__).globalize()
        #The service to run, pickled
        self.pickledService = cPickle.dumps(service)
        #An empty file in the jobStore which when deleted is used to signal
        #that the service should cease, is initialised in
        #Job._modifyJobGraphForServices
        self.stopFileStoreID = None
        #Similarly a empty file which when deleted is used to signal that the
        #service is established
        self.startFileStoreID = None

    def run(self, fileStore):
        #Unpickle the service
        userModule = self._loadUserModule(self.serviceModule)
        service = self._unpickle( userModule, BytesIO( self.pickledService ) )
        #Start the service
        startCredentials = service.start()
        #The start credentials  must be communicated to processes connecting to
        #the service, to do this while the run method is running we
        #cheat and set the return value promise within the run method
        self._setReturnValuesForPromises(self, startCredentials, fileStore.jobStore)
        self._rvs = {}  # Set this to avoid the return values being updated after the
        #run method has completed!
        #Now flag that the service is running jobs can connect to it
        assert self.startFileStoreID != None
        assert fileStore.globalFileExists(self.startFileStoreID)
        fileStore.deleteGlobalFile(self.startFileStoreID)
        assert not fileStore.globalFileExists(self.startFileStoreID)
        #Now block until we are told to stop, which is indicated by the removal
        #of a file
        assert self.stopFileStoreID != None
        while fileStore.globalFileExists(self.stopFileStoreID):
            time.sleep(1) #Avoid excessive polling
        #Now kill the service
        service.stop()

    def getUserScript(self):
        return self.serviceModule


class EncapsulatedJob(Job):
    """
    An convenience Job class used to make a job subgraph appear to
    be a single job.
    
    Let A be a root job potentially with children and follow-ons.
    Without an encapsulated job the simplest way to specify a job B which
    runs after A and all its successors is to create a parent of A' and then make B 
    a follow-on of A'. In turn if we wish to run C after B and its successors then we 
    repeat the process to create B', a parent of B, creating a graph in which A' is run,
    then A as a child of A', then the successors of A, then B' as a follow on of A', 
    then B as a child of B', then the successors of B, then finally C as follow on of B', 
    e.g.
    
    A, B, C = A(), B(), C() #Functions to create job graphs
    A' = Job()
    B' = Job()
    A'.addChild(A)
    A'.addFollowOn(B')
    B'.addChild(B)
    B'.addFollowOn(C)
    
    An encapsulated job of E(A) of A saves making A' and B', instead we can write:
    
    A, B, C = A().encapsulate(), B(), C() #Functions to create job graphs
    A.addChild(B)
    A.addFollowOn(C)
    
    Note the call to encapsulate creates the EncapsulatedJob.
    """
    def __init__(self, job):
        """
        job is the job to encapsulate.
        """
        Job.__init__(self)
        Job.addChild(self, job)
        self.followOn = Job()
        Job.addFollowOn(self, self.followOn)

    def addChild(self, childJob):
        return Job.addChild(self.followOn, childJob)

    def addService(self, service):
        return Job.addService(self.followOn, service)

    def addFollowOn(self, followOnJob):
        return Job.addFollowOn(self.followOn, followOnJob)

    def rv(self, argIndex=0):
        return self.followOn.rv(argIndex)

class PromisedJobReturnValue(object):
    """
    References a return value from a Job's run function. Let T be a job.
    Instances of PromisedJobReturnValue are created by
    T.rv(), which is used to reference the return value of T's run function.
    When passed to the constructor of a different, successor Job the PromisedJobReturnValue
    will be replaced by the actual referenced return value.
    This mechanism allows a return values from one Job's run method to be input
    argument to Job before the former Job's run function has been executed.
    """
    def __init__(self):
        #The None values are replaced with a actual values in Job._setFileIDsForPromisedValues
        self.jobStoreFileID = None #A file containing the actual pickled return value
        self.jobStoreString = None #This is the string used to load the jobStore.

def promisedJobReturnValuePickleFunction(promise):
    """
    This function and promisedJobReturnValueUnpickleFunction are used as custom pickle/unpickle 
    functions to ensure that when the PromisedJobReturnValue instance p is unpickled it is replaced with 
    the object pickled in p.jobStoreFileID
    """
    return promisedJobReturnValueUnpickleFunction, (promise.jobStoreString, promise.jobStoreFileID)

def promisedJobReturnValueUnpickleFunction(jobStoreString, jobStoreFileID):
    """The PromisedJobReturnValue custom unpickle function.
    """
    #If the attributes jobStoreFileID and jobStoreString are None then we return
    #a new empty PromisedJobReturnValue (this can happen if Promise is serialised between its creation
    #and the running of Job._setFileIDsForPromisedValues.
    if jobStoreString == None:
        assert jobStoreFileID == None
        return PromisedJobReturnValue()
    jobStore = loadJobStore(jobStoreString)
    with jobStore.readFileStream(jobStoreFileID) as fileHandle:
        value = cPickle.load(fileHandle) #If this doesn't work then the file containing the promise may not exist or be corrupted.
        return value

#This sets up the custom magic for pickling/unpickling a PromisedJobReturnValue
copy_reg.pickle(PromisedJobReturnValue,
                promisedJobReturnValuePickleFunction,
                promisedJobReturnValueUnpickleFunction)

def deleteFileStoreIDs(job, jobStoreFileIDsToDelete):
    """
    Job function that deletes a bunch of files using their jobStoreFileIDs
    """
    map(lambda i : job.fileStore.deleteGlobalFile(i), jobStoreFileIDsToDelete)

def blockUntilDeleted(job, jobStoreFileIDs):
    """
    Function will not terminate until all the fileStoreIDs in jobStoreFileIDs
    cease to exist.
    """
    while True:
        jobStoreFileIDs = [ i for i in jobStoreFileIDs
                           if job.fileStore.globalFileExists(i) ]
        if len(jobStoreFileIDs) == 0:
            break
        time.sleep(1)

