agave-jobs/jobs-core/src/main/java/org/iplantc/service/jobs/queue/actions/SubmissionAction.java
/**
     * This method attempts to stage application assets and launch a job on the remote {@link ExecutionSystem}.
     * If submission is not possible, the job is failed and the remote job work directory is removed.
     *
     * @throws SystemUnavailableException
     * @throws SystemUnknownException
     * @throws JobException
     * @throws JobDependencyException 
     */
This method attempts to stage application assets and launch a job on the remote  ExecutionSystem.
If submission is not possible, the job is failed and the remote job work directory is removed.


This method attempts to stage application assets and launch a job on the remote ExecutionSystem . If submission is not possible the job is failed and the remote job work directory is removed .
public void run() throws SystemUnavailableException, SystemUnknownException, JobException,
            ClosedByInterruptException, JobDependencyException
    {
        boolean submitted = false;
        int attempts = getJob().getRetries();
        try 
        {
            while (!submitted && attempts <= Settings.MAX_SUBMISSION_RETRIES)
            {
                checkStopped();
                getJob().setRetries(attempts);
                attempts++;
                log.debug("Attempt " + attempts + " to submit job " + getJob().getUuid() +
                          " (max retries = " + Settings.MAX_SUBMISSION_RETRIES + ").");
                updateJobStatus(JobStatusType.SUBMITTING, "Attempt " + attempts + " to submit job");
                try 
                {
                    setJobLauncher(new JobLauncherFactory().getInstance(getJob()));
                    getJobLauncher().launch();
                    submitted = true;
                    log.info("Successfully submitted job " + getJob().getUuid() + " to " + getJob().getSystem());
                }
                catch (ClosedByInterruptException e) {
                    log.debug("Job submission cancelled by outside process for job " + getJob().getUuid());
                    throw e;
                } 
                catch (UnknownSoftwareException|SystemUnknownException e) {
                    log.error("Failed to submit job " + getJob().getUuid() + " on " + getJob().getSystem() +
                                    ". " + e.getMessage());
                    updateJobStatus(JobStatusType.FAILED, "Unable to submit job " + getJob().getUuid() + ". " +
                            e.getMessage() + " No further attempts will be made.");
                    throw new JobDependencyException(e);
                }
                catch (SystemUnavailableException e) {
                    log.debug("One or more dependent systems for job " + getJob().getUuid() + " is currently unavailable. " + e.getMessage());
                    updateJobStatus(JobStatusType.STAGED,
                        "Remote execution of  job " + getJob().getUuid() + " is currently paused waiting for " + getJob().getSystem() +
                        " to become available. If the system becomes available again within 7 days, this job " +
                        "will resume staging. After 7 days it will be killed. Original error message: " + e.getMessage());
                    break;
                }
                catch (SoftwareUnavailableException e) {
                    log.debug("Software for job " + getJob().getUuid() + " is currently unavailable. " + e.getMessage());
                    updateJobStatus(JobStatusType.STAGED,
                            "Remote execution of  job " + getJob().getUuid() + " is currently paused waiting for app " + getJob().getSoftwareName() +
                                    " to become available. If the app becomes available again within 30 days of job creation, this job " +
                                    "will resume staging. After that time, it will be killed.");
                    break;
                }
                catch (SchedulerException e) {
                    log.debug("Failed to submit job " + getJob().getUuid() + " on " + getJob().getSystem() +
                            " due to scheduler exception: " + e.getMessage());
                    updateJobStatus(JobStatusType.STAGED, "Attempt "
                        + attempts + " failed to submit job due to scheduler exception. " + e.getMessage());
                }
                catch (IOException e) {
                    log.error("Failed to submit job " + getJob().getUuid() +
                            ". Unable to connect to remote system " + e.getMessage(),e);
                    updateJobStatus(JobStatusType.STAGED, e.getMessage() +
                            " The service was unable to connect to the target execution system " +
                            "for this application, " + getJob().getSystem() + ". This job will " +
                            "remain in queue until the system becomes available. Original error message: " + e.getMessage());
                }
                catch (Throwable e)
                {   
                    if (e.getCause() instanceof UnresolvableObjectException ||
                            e.getCause() instanceof ObjectNotFoundException) {
                        log.error("Race condition was just avoided for job " + getJob().getUuid(), e);
                    }
                    else if (attempts >= Settings.MAX_SUBMISSION_RETRIES ) 
                    {
                        log.error("Failed to submit job " + getJob().getUuid() +
                            " after " + attempts + " attempts.", e);
                        updateJobStatus(getJob().getStatus(), "Attempt "
                                + attempts + " failed to submit job. " + e.getMessage());
                        try {
                            if (getJob().isArchived()) {
                                setJob(getJobManager().deleteStagedData(job));
                            }
                        } catch (Exception e1) {
                            log.error("Failed to remove remote work directory for job " + getJob().getUuid(), e1);
                            updateJobStatus(getJob().getStatus(),
                                    "Failed to remove remote work directory.");
                        }
                        log.error("Unable to submit job " + getJob().getUuid() +
                                " after " + attempts + " attempts. Job cancelled.");
                        updateJobStatus(JobStatusType.FAILED,
                                "Unable to submit job after " + attempts + " attempts. Job cancelled.");
                        break;
                    } 
                    else 
                    {
                        try 
                        {
                            String msg = "Attempt " + attempts + " failed to submit job. " + e.getMessage();
                            log.info(msg, e);
                            updateJobStatus(getJob().getStatus(), msg);
                        }
                        catch (Exception e1) {
                            log.error("Failed to update job " + getJob().getUuid() + " status to " + getJob().getStatus(), e1);
                        }
                    }
                }
            }
        }
        finally
        {
            if (getJobLauncher() != null) {
                FileUtils.deleteQuietly(getJobLauncher().getTempAppDir());
            }
        }
    }
public void run ( ) throws SystemUnavailableException , SystemUnknownException , JobException , ClosedByInterruptException , JobDependencyException { boolean submitted = false ; int attempts = getJob ( ) . getRetries ( ) ; try { while ( ! submitted && attempts <= Settings . MAX_SUBMISSION_RETRIES ) { checkStopped ( ) ; getJob ( ) . setRetries ( attempts ) ; attempts ++ ; log . debug ( "Attempt " + attempts + " to submit job " + getJob ( ) . getUuid ( ) + " (max retries = " + Settings . MAX_SUBMISSION_RETRIES + ")." ) ; updateJobStatus ( JobStatusType . SUBMITTING , "Attempt " + attempts + " to submit job" ) ; try { setJobLauncher ( new JobLauncherFactory ( ) . getInstance ( getJob ( ) ) ) ; getJobLauncher ( ) . launch ( ) ; submitted = true ; log . info ( "Successfully submitted job " + getJob ( ) . getUuid ( ) + " to " + getJob ( ) . getSystem ( ) ) ; } catch ( ClosedByInterruptException e ) { log . debug ( "Job submission cancelled by outside process for job " + getJob ( ) . getUuid ( ) ) ; throw e ; } catch ( UnknownSoftwareException | SystemUnknownException e ) { log . error ( "Failed to submit job " + getJob ( ) . getUuid ( ) + " on " + getJob ( ) . getSystem ( ) + ". " + e . getMessage ( ) ) ; updateJobStatus ( JobStatusType . FAILED , "Unable to submit job " + getJob ( ) . getUuid ( ) + ". " + e . getMessage ( ) + " No further attempts will be made." ) ; throw new JobDependencyException ( e ) ; } catch ( SystemUnavailableException e ) { log . debug ( "One or more dependent systems for job " + getJob ( ) . getUuid ( ) + " is currently unavailable. " + e . getMessage ( ) ) ; updateJobStatus ( JobStatusType . STAGED , "Remote execution of  job " + getJob ( ) . getUuid ( ) + " is currently paused waiting for " + getJob ( ) . getSystem ( ) + " to become available. If the system becomes available again within 7 days, this job " + "will resume staging. After 7 days it will be killed. Original error message: " + e . getMessage ( ) ) ; break ; } catch ( SoftwareUnavailableException e ) { log . debug ( "Software for job " + getJob ( ) . getUuid ( ) + " is currently unavailable. " + e . getMessage ( ) ) ; updateJobStatus ( JobStatusType . STAGED , "Remote execution of  job " + getJob ( ) . getUuid ( ) + " is currently paused waiting for app " + getJob ( ) . getSoftwareName ( ) + " to become available. If the app becomes available again within 30 days of job creation, this job " + "will resume staging. After that time, it will be killed." ) ; break ; } catch ( SchedulerException e ) { log . debug ( "Failed to submit job " + getJob ( ) . getUuid ( ) + " on " + getJob ( ) . getSystem ( ) + " due to scheduler exception: " + e . getMessage ( ) ) ; updateJobStatus ( JobStatusType . STAGED , "Attempt " + attempts + " failed to submit job due to scheduler exception. " + e . getMessage ( ) ) ; } catch ( IOException e ) { log . error ( "Failed to submit job " + getJob ( ) . getUuid ( ) + ". Unable to connect to remote system " + e . getMessage ( ) , e ) ; updateJobStatus ( JobStatusType . STAGED , e . getMessage ( ) + " The service was unable to connect to the target execution system " + "for this application, " + getJob ( ) . getSystem ( ) + ". This job will " + "remain in queue until the system becomes available. Original error message: " + e . getMessage ( ) ) ; } catch ( Throwable e ) { if ( e . getCause ( ) instanceof UnresolvableObjectException || e . getCause ( ) instanceof ObjectNotFoundException ) { log . error ( "Race condition was just avoided for job " + getJob ( ) . getUuid ( ) , e ) ; } else if ( attempts >= Settings . MAX_SUBMISSION_RETRIES ) { log . error ( "Failed to submit job " + getJob ( ) . getUuid ( ) + " after " + attempts + " attempts." , e ) ; updateJobStatus ( getJob ( ) . getStatus ( ) , "Attempt " + attempts + " failed to submit job. " + e . getMessage ( ) ) ; try { if ( getJob ( ) . isArchived ( ) ) { setJob ( getJobManager ( ) . deleteStagedData ( job ) ) ; } } catch ( Exception e1 ) { log . error ( "Failed to remove remote work directory for job " + getJob ( ) . getUuid ( ) , e1 ) ; updateJobStatus ( getJob ( ) . getStatus ( ) , "Failed to remove remote work directory." ) ; } log . error ( "Unable to submit job " + getJob ( ) . getUuid ( ) + " after " + attempts + " attempts. Job cancelled." ) ; updateJobStatus ( JobStatusType . FAILED , "Unable to submit job after " + attempts + " attempts. Job cancelled." ) ; break ; } else { try { String msg = "Attempt " + attempts + " failed to submit job. " + e . getMessage ( ) ; log . info ( msg , e ) ; updateJobStatus ( getJob ( ) . getStatus ( ) , msg ) ; } catch ( Exception e1 ) { log . error ( "Failed to update job " + getJob ( ) . getUuid ( ) + " status to " + getJob ( ) . getStatus ( ) , e1 ) ; } } } } } finally { if ( getJobLauncher ( ) != null ) { FileUtils . deleteQuietly ( getJobLauncher ( ) . getTempAppDir ( ) ) ; } } }
This method attempts to stage application assets and launch a job on the remote {@link ExecutionSystem}.
This method attempts to stage application assets and launch a job on the remote { @link ExecutionSystem } .
// clean up the job directory now that we're either done or failed