agave-jobs/jobs-core/src/main/java/org/iplantc/service/jobs/queue/ZombieJobWatch.java
/**
	 * Rolls back a {@link Job} status to the last stable status prior to the
	 * current one. This is akin to release a job back to the queue from which
	 * it was previously taken. This is helpful in situations where a job is
	 * stuck due to a failed worker or other abandoned process which left the
	 * job is a zombie state from which it would not otherwise recover.
	 *
	 * @param job the job that will be rolled back
	 * @param callingUsername the principal requesting the rollback
	 * @throws JobException if the job cannot be rolled back due to invalid status
	 * @throws JobDependencyException if job was inactive when attempting to roll back
	 */
Rolls back a  Job status to the last stable status prior to the
current one. This is akin to release a job back to the queue from which
it was previously taken. This is helpful in situations where a job is
stuck due to a failed worker or other abandoned process which left the
job is a zombie state from which it would not otherwise recover.

@param job the job that will be rolled back
@param callingUsername the principal requesting the rollback
@throws JobException if the job cannot be rolled back due to invalid status
@throws JobDependencyException if job was inactive when attempting to roll back
Rolls back a Job status to the last stable status prior to the current one . This is akin to release a job back to the queue from which it was previously taken . This is helpful in situations where a job is stuck due to a failed worker or other abandoned process which left the job is a zombie state from which it would not otherwise recover . @param job the job that will be rolled back @param callingUsername the principal requesting the rollback @throws JobException if the job cannot be rolled back due to invalid status @throws JobDependencyException if job was inactive when attempting to roll back
public Job rollbackJob(Job job, String callingUsername)
	throws JobException, JobDependencyException
	{
		try
		{
			JobStatusType rollbackJobStatus = job.getStatus().rollbackState();
			RemoteSystem executionSystem = new SystemDao().findBySystemId(job.getSystem());
			if (executionSystem == null)
			{
				if (job.getStatus() == ARCHIVING)
				{
				    job = JobManager.updateStatus(job, JobStatusType.ARCHIVING_FAILED,
		    				"Execution system is no longer present.");
				    job = JobManager.updateStatus(job, JobStatusType.FINISHED,
		    				"Job completed, but failed to archive.");
				}
				else
				{
				    job = JobManager.updateStatus(job, JobStatusType.FAILED,
							"Job failed. Execution system is no longer present.");
				}
				log.debug("Zombie reaper task is setting status of job " + job.getUuid() +
						" to " + job.getStatus() + " because the execution system " +
						job.getSystem() + " is no longer present.");
			}
			else if (job.getArchiveSystem() == null)
			{
				if (job.getStatus() == ARCHIVING)
				{
				    job = JobManager.updateStatus(job, JobStatusType.ARCHIVING_FAILED,
		    				"Archive system is no longer present.");
					job = JobManager.updateStatus(job, JobStatusType.FINISHED,
		    				"Job completed, but failed to archive.");
		    	}
				else
				{
				    job = JobManager.updateStatus(job, JobStatusType.FAILED,
							"Job failed. Archive system is no longer present.");
				}
				log.debug("Zombie reaper task is setting status of job " + job.getUuid() +
						" to " + job.getStatus() + " because the archive system "
						+ "is no longer present.");
			}
			cancelCurrentTransfers(job, callingUsername);
			if (!isFinished(job.getStatus()))
			{
				log.debug("Zombie reaper task is rolling back status of job " + job.getUuid() +
						" from " + job.getStatus() + " to " + rollbackJobStatus);
				if (rollbackJobStatus == CLEANING_UP)
				{
		        	if (executionSystem.isAvailable())
		        	{
		        		if (job.getArchiveSystem().isAvailable())
		            	{
		        		    job = JobManager.updateStatus(job, CLEANING_UP, "Archiving task for this job was "
									+ "found in a zombie state. Job will be rolled back to the previous state and "
									+ "archiving to " + job.getArchiveSystem().getSystemId()  + " will resume.");
		            	}
		        		else
		        		{
		        		    job = JobManager.updateStatus(job, CLEANING_UP, "Archiving task for this job was "
									+ "found in a zombie state. Job will be rolled back to the previous state and "
		            				+ "archiving will resume when the " + job.getArchiveSystem().getSystemId()
		            				+ " becomes available.");
		        		}
		        	}
		        	else
		        	{
		        	    job = JobManager.updateStatus(job, JobStatusType.CLEANING_UP, "Archiving task for this job was "
								+ "found in a zombie state. Job will be rolled back to the previous state and "
		        				+ "archiving will resume when " + job.getArchiveSystem().getSystemId()
		        				+ " the execution system becomes available.");
		        	}
				}
				else if (rollbackJobStatus == STAGED)
				{
					if (executionSystem.isAvailable())
		        	{
					    job = JobManager.updateStatus(job, STAGED, "Submission task for this job was "
								+ "found in a zombie state. Job will be rolled back to the previous state and "
								+ "submission to " + job.getSystem() + " will resume.");
		        	}
					else
					{
					    job = JobManager.updateStatus(job, STAGED, "Submission task for this job was "
								+ "found in a zombie state. Job will be rolled back to the previous state and "
		        				+ "submission to " + job.getSystem() + " will resume when the system becomes "
		        				+ "available.");
					}
				}
				else if (rollbackJobStatus == PENDING)
				{
					if (executionSystem.isAvailable())
		        	{
					    job = JobManager.updateStatus(job, PENDING, "Input data staging for this job was "
								+ "found in a zombie state. Job will be rolled back to the previous state and "
								+ "input staging to " + job.getSystem() + " will resume.");
		        	}
					else
					{
					    job = JobManager.updateStatus(job, PENDING, "Input data staging for this job was "
								+ "found in a zombie state. Job will be rolled back to the previous state and "
		        				+ "input staging to " + job.getSystem() + " will resume when the system becomes "
		        				+ "available.");
					}
				}
				else
				{
					if (executionSystem.isAvailable())
		        	{
					    job = JobManager.updateStatus(job, rollbackJobStatus,
		    					StringUtils.capitalize(job.getStatus().name())
		    					+ " for this job was found in a zombie state. Job will be rolled back "
		    					+ "to the previous state and and resume.");
		        	}
					else
					{
						job = JobManager.updateStatus(job, rollbackJobStatus,
								StringUtils.capitalize(job.getStatus().name())
		    					+ " for this job was found in a zombie state. Job will be rolled back "
								+ "to the previous state and and resume when "
		    					+ job.getSystem() + " becomes available.");
					}
				}
			}
			else {
				throw new JobDependencyException("Unabled to roll back status of inactive jobs.");
			}
		}
		catch (JobDependencyException e) {
			throw e;
		}
		catch (JobException e) {
			throw e;
		}
		catch (UnresolvableObjectException e) {
			throw new JobException("Just avoided a job archive race condition");
		}
		catch (StaleObjectStateException e) {
			throw new JobException("Just avoided a job archive race condition");
		}
		catch (HibernateException e) {
			throw new JobException("Failed to retrieve job information from db", e);
		}
		catch (Throwable e)
		{
			throw new JobException("Failed to roll back job " + job.getUuid() + " to previous state", e);
		}
		return job;
	}
public Job rollbackJob ( Job job , String callingUsername ) throws JobException , JobDependencyException { try { JobStatusType rollbackJobStatus = job . getStatus ( ) . rollbackState ( ) ; RemoteSystem executionSystem = new SystemDao ( ) . findBySystemId ( job . getSystem ( ) ) ; if ( executionSystem == null ) { if ( job . getStatus ( ) == ARCHIVING ) { job = JobManager . updateStatus ( job , JobStatusType . ARCHIVING_FAILED , "Execution system is no longer present." ) ; job = JobManager . updateStatus ( job , JobStatusType . FINISHED , "Job completed, but failed to archive." ) ; } else { job = JobManager . updateStatus ( job , JobStatusType . FAILED , "Job failed. Execution system is no longer present." ) ; } log . debug ( "Zombie reaper task is setting status of job " + job . getUuid ( ) + " to " + job . getStatus ( ) + " because the execution system " + job . getSystem ( ) + " is no longer present." ) ; } else if ( job . getArchiveSystem ( ) == null ) { if ( job . getStatus ( ) == ARCHIVING ) { job = JobManager . updateStatus ( job , JobStatusType . ARCHIVING_FAILED , "Archive system is no longer present." ) ; job = JobManager . updateStatus ( job , JobStatusType . FINISHED , "Job completed, but failed to archive." ) ; } else { job = JobManager . updateStatus ( job , JobStatusType . FAILED , "Job failed. Archive system is no longer present." ) ; } log . debug ( "Zombie reaper task is setting status of job " + job . getUuid ( ) + " to " + job . getStatus ( ) + " because the archive system " + "is no longer present." ) ; } cancelCurrentTransfers ( job , callingUsername ) ; if ( ! isFinished ( job . getStatus ( ) ) ) { log . debug ( "Zombie reaper task is rolling back status of job " + job . getUuid ( ) + " from " + job . getStatus ( ) + " to " + rollbackJobStatus ) ; if ( rollbackJobStatus == CLEANING_UP ) { if ( executionSystem . isAvailable ( ) ) { if ( job . getArchiveSystem ( ) . isAvailable ( ) ) { job = JobManager . updateStatus ( job , CLEANING_UP , "Archiving task for this job was " + "found in a zombie state. Job will be rolled back to the previous state and " + "archiving to " + job . getArchiveSystem ( ) . getSystemId ( ) + " will resume." ) ; } else { job = JobManager . updateStatus ( job , CLEANING_UP , "Archiving task for this job was " + "found in a zombie state. Job will be rolled back to the previous state and " + "archiving will resume when the " + job . getArchiveSystem ( ) . getSystemId ( ) + " becomes available." ) ; } } else { job = JobManager . updateStatus ( job , JobStatusType . CLEANING_UP , "Archiving task for this job was " + "found in a zombie state. Job will be rolled back to the previous state and " + "archiving will resume when " + job . getArchiveSystem ( ) . getSystemId ( ) + " the execution system becomes available." ) ; } } else if ( rollbackJobStatus == STAGED ) { if ( executionSystem . isAvailable ( ) ) { job = JobManager . updateStatus ( job , STAGED , "Submission task for this job was " + "found in a zombie state. Job will be rolled back to the previous state and " + "submission to " + job . getSystem ( ) + " will resume." ) ; } else { job = JobManager . updateStatus ( job , STAGED , "Submission task for this job was " + "found in a zombie state. Job will be rolled back to the previous state and " + "submission to " + job . getSystem ( ) + " will resume when the system becomes " + "available." ) ; } } else if ( rollbackJobStatus == PENDING ) { if ( executionSystem . isAvailable ( ) ) { job = JobManager . updateStatus ( job , PENDING , "Input data staging for this job was " + "found in a zombie state. Job will be rolled back to the previous state and " + "input staging to " + job . getSystem ( ) + " will resume." ) ; } else { job = JobManager . updateStatus ( job , PENDING , "Input data staging for this job was " + "found in a zombie state. Job will be rolled back to the previous state and " + "input staging to " + job . getSystem ( ) + " will resume when the system becomes " + "available." ) ; } } else { if ( executionSystem . isAvailable ( ) ) { job = JobManager . updateStatus ( job , rollbackJobStatus , StringUtils . capitalize ( job . getStatus ( ) . name ( ) ) + " for this job was found in a zombie state. Job will be rolled back " + "to the previous state and and resume." ) ; } else { job = JobManager . updateStatus ( job , rollbackJobStatus , StringUtils . capitalize ( job . getStatus ( ) . name ( ) ) + " for this job was found in a zombie state. Job will be rolled back " + "to the previous state and and resume when " + job . getSystem ( ) + " becomes available." ) ; } } } else { throw new JobDependencyException ( "Unabled to roll back status of inactive jobs." ) ; } } catch ( JobDependencyException e ) { throw e ; } catch ( JobException e ) { throw e ; } catch ( UnresolvableObjectException e ) { throw new JobException ( "Just avoided a job archive race condition" ) ; } catch ( StaleObjectStateException e ) { throw new JobException ( "Just avoided a job archive race condition" ) ; } catch ( HibernateException e ) { throw new JobException ( "Failed to retrieve job information from db" , e ) ; } catch ( Throwable e ) { throw new JobException ( "Failed to roll back job " + job . getUuid ( ) + " to previous state" , e ) ; } return job ; }
Rolls back a {@link Job} status to the last stable status prior to the
 current one.
Rolls back a { @link Job } status to the last stable status prior to the current one .
// roll back the status so it will be picked back up