def batch_process(de_novo_path, temp_dir, output_path):
    """ sets up a lsf job array
    """
    
    temp_dir = tempfile.mkdtemp(dir=temp_dir)
    count = split_denovos(de_novo_path, temp_dir)
    
    # set up run parameters
    job_name = "denovonear"
    job_id = "{0}[1-{1}]%20".format(job_name, count)
    
    basename = os.path.basename(de_novo_path)
    infile = os.path.join(temp_dir, "tmp.\$LSB_JOBINDEX\.txt")
    outfile = os.path.join(temp_dir, "tmp.\$LSB_JOBINDEX\.output")
    
    command = ["denovonear", "cluster",
        "--in", infile,
        "--out", outfile]
    submit_bsub_job(command, job_id, memory=3500, requeue_code=134, logfile="clustering.bjob")
    time.sleep(2)
    
    # merge the array output after the array finishes
    merge_id = "{}_merge".format(job_name)
    command = ["head", "-n", "1", os.path.join(temp_dir, "tmp.1.output"), ">", output_path, \
        "; tail", "-q", "-n", "+2", os.path.join(temp_dir, "tmp.*.output"), "|", "sort", ">>", output_path]
    submit_bsub_job(command, merge_id, memory=100, dependent_id=job_id, logfile="clustering.bjob")
    time.sleep(2)
    
    # submit a cleanup job to the cluster
    submit_bsub_job(["rm", "-r", temp_dir], job_id="{}_cleanup".format(job_name), \
        memory=100, dependent_id=merge_id, logfile="clustering.bjob")
def batch_process ( de_novo_path , temp_dir , output_path ) : temp_dir = tempfile . mkdtemp ( dir = temp_dir ) count = split_denovos ( de_novo_path , temp_dir ) job_name = "denovonear" job_id = "{0}[1-{1}]%20" . format ( job_name , count ) basename = os . path . basename ( de_novo_path ) infile = os . path . join ( temp_dir , "tmp.\$LSB_JOBINDEX\.txt" ) outfile = os . path . join ( temp_dir , "tmp.\$LSB_JOBINDEX\.output" ) command = [ "denovonear" , "cluster" , "--in" , infile , "--out" , outfile ] submit_bsub_job ( command , job_id , memory = 3500 , requeue_code = 134 , logfile = "clustering.bjob" ) time . sleep ( 2 ) merge_id = "{}_merge" . format ( job_name ) command = [ "head" , "-n" , "1" , os . path . join ( temp_dir , "tmp.1.output" ) , ">" , output_path , "; tail" , "-q" , "-n" , "+2" , os . path . join ( temp_dir , "tmp.*.output" ) , "|" , "sort" , ">>" , output_path ] submit_bsub_job ( command , merge_id , memory = 100 , dependent_id = job_id , logfile = "clustering.bjob" ) time . sleep ( 2 ) submit_bsub_job ( [ "rm" , "-r" , temp_dir ] , job_id = "{}_cleanup" . format ( job_name ) , memory = 100 , dependent_id = merge_id , logfile = "clustering.bjob" )
sets up a lsf job array