def processFinishedJob(self, batchSystemID, resultStatus, wallTime=None):
        """
        Function reads a processed jobGraph file and updates its state.
        """
        jobNode = self.removeJob(batchSystemID)
        jobStoreID = jobNode.jobStoreID
        if wallTime is not None and self.clusterScaler is not None:
            self.clusterScaler.addCompletedJob(jobNode, wallTime)
        if self.jobStore.exists(jobStoreID):
            logger.debug("Job %s continues to exist (i.e. has more to do)", jobNode)
            try:
                jobGraph = self.jobStore.load(jobStoreID)
            except NoSuchJobException:
                # Avoid importing AWSJobStore as the corresponding extra might be missing
                if self.jobStore.__class__.__name__ == 'AWSJobStore':
                    # We have a ghost job - the job has been deleted but a stale read from
                    # SDB gave us a false positive when we checked for its existence.
                    # Process the job from here as any other job removed from the job store.
                    # This is a temporary work around until https://github.com/BD2KGenomics/toil/issues/1091
                    # is completed
                    logger.warn('Got a stale read from SDB for job %s', jobNode)
                    self.processRemovedJob(jobNode, resultStatus)
                    return
                else:
                    raise
            if jobGraph.logJobStoreFileID is not None:
                with jobGraph.getLogFileHandle(self.jobStore) as logFileStream:
                    # more memory efficient than read().striplines() while leaving off the
                    # trailing \n left when using readlines()
                    # http://stackoverflow.com/a/15233739
                    StatsAndLogging.logWithFormatting(jobStoreID, logFileStream, method=logger.warn,
                                                      message='The job seems to have left a log file, indicating failure: %s' % jobGraph)
                if self.config.writeLogs or self.config.writeLogsGzip:
                    with jobGraph.getLogFileHandle(self.jobStore) as logFileStream:
                        StatsAndLogging.writeLogFiles(jobGraph.chainedJobs, logFileStream, self.config)
            if resultStatus != 0:
                # If the batch system returned a non-zero exit code then the worker
                # is assumed not to have captured the failure of the job, so we
                # reduce the retry count here.
                if jobGraph.logJobStoreFileID is None:
                    logger.warn("No log file is present, despite job failing: %s", jobNode)
                jobGraph.setupJobAfterFailure(self.config)
                self.jobStore.update(jobGraph)
            elif jobStoreID in self.toilState.hasFailedSuccessors:
                # If the job has completed okay, we can remove it from the list of jobs with failed successors
                self.toilState.hasFailedSuccessors.remove(jobStoreID)

            self.toilState.updatedJobs.add((jobGraph, resultStatus)) #Now we know the
            #jobGraph is done we can add it to the list of updated jobGraph files
            logger.debug("Added job: %s to active jobs", jobGraph)
        else:  #The jobGraph is done
            self.processRemovedJob(jobNode, resultStatus)
def processFinishedJob ( self , batchSystemID , resultStatus , wallTime = None ) : jobNode = self . removeJob ( batchSystemID ) jobStoreID = jobNode . jobStoreID if wallTime is not None and self . clusterScaler is not None : self . clusterScaler . addCompletedJob ( jobNode , wallTime ) if self . jobStore . exists ( jobStoreID ) : logger . debug ( "Job %s continues to exist (i.e. has more to do)" , jobNode ) try : jobGraph = self . jobStore . load ( jobStoreID ) except NoSuchJobException : if self . jobStore . __class__ . __name__ == 'AWSJobStore' : logger . warn ( 'Got a stale read from SDB for job %s' , jobNode ) self . processRemovedJob ( jobNode , resultStatus ) return else : raise if jobGraph . logJobStoreFileID is not None : with jobGraph . getLogFileHandle ( self . jobStore ) as logFileStream : StatsAndLogging . logWithFormatting ( jobStoreID , logFileStream , method = logger . warn , message = 'The job seems to have left a log file, indicating failure: %s' % jobGraph ) if self . config . writeLogs or self . config . writeLogsGzip : with jobGraph . getLogFileHandle ( self . jobStore ) as logFileStream : StatsAndLogging . writeLogFiles ( jobGraph . chainedJobs , logFileStream , self . config ) if resultStatus != 0 : if jobGraph . logJobStoreFileID is None : logger . warn ( "No log file is present, despite job failing: %s" , jobNode ) jobGraph . setupJobAfterFailure ( self . config ) self . jobStore . update ( jobGraph ) elif jobStoreID in self . toilState . hasFailedSuccessors : self . toilState . hasFailedSuccessors . remove ( jobStoreID ) self . toilState . updatedJobs . add ( ( jobGraph , resultStatus ) ) logger . debug ( "Added job: %s to active jobs" , jobGraph ) else : self . processRemovedJob ( jobNode , resultStatus )
Function reads a processed jobGraph file and updates its state.