#!/usr/bin/env python

import argparse
import sys, subprocess, os
import re
import datetime
import math

# Generate customized SLURM submit scripts to run RAxML-ng.
#   Takes a directory of alignmets, runs the raxml-ng --parse
#   function to get estimates of RAM and CPU needs for the job.
#   Then uses a templates sbatch script to make a customized
#   script for each job.
# 
#   Other aspects of the job script can be customized too, 
#   see the options in --help.
# 
#   Optionally (using the -s/--submit flag) this script will
#   submit your jobs to the scheduler.
#  
# <NAME>
# University of Florida

__version__= 1.0   #  July 3, 2020

def run_raxml_ng_parse(raxml_path, alignment, output_path, model, prefix, verbose):
    """Run raxml_ng --parse to get estimates of job resource parameters"""

    # Define regex for mem and threads output from raxml-ng
    mem_re = re.compile('Estimated memory requirements\s*:\s*(\d+)\s*([MG]B)')
    thread_re = re.compile('Recommended number of threads / MPI processes:\s*(\d+)')

    align_root = os.path.basename(alignment)
    prefix = prefix + align_root
    prefix = os.path.join(output_path, prefix)

    # Run raxml-ng on alignment to get estimates of memory and threads
    try:
        raxml_result = subprocess.run([raxml_path, '--parse', '--msa', alignment,
                        '--threads', '1', '--model', model, '--prefix', prefix],
                        capture_output=True)
    except FileNotFoundError:
        print('raxml-ng does not appear to be installed in your path. Please use the --raxml flag to specify the path to raxml-ng')
        sys.exit(1) 

    stdout = raxml_result.stdout.decode('utf-8')
    stderr = raxml_result.stderr.decode('utf-8')

    # Check that raxml-ng ran. If not check path to raxml-ng.
    if raxml_result.returncode != 0:
        check_raxml = subprocess.run([raxml_path, '-h'])
        if check_raxml.returncode != 0:
            if raxml_path == 'raxml-ng':
                print('raxml-ng does not appear to be installed in your path. Please use the --raxml flag to specify the path to raxml-ng')
                sys.exit(1)
            else:
                print('Could not run raxml-ng from the path provided. Please check your path.')
                print(f'You specified --raxml as: {raxml_path}')
                sys.exit(1)
        else:
            print('RAxML-ng failed, but produced this output:')
            print(stdout)
            print('\n\nRAxML-ng failed with this error:')
            print(stderr)
            sys.exit(1)
           
    else:
        memory = mem_re.search(stdout)
        threads = thread_re.search(stdout)
        if verbose:
            print(f'Estimates: {alignment}\t{memory.group(1)}{memory.group(2)}\t{threads.group(1)}')

    if memory and threads:
        # Not sure if raxml-ng ever returns estimates in GB, but just in case...
        if memory.group(2) == 'GB':
            memory_MB = int(memory.group(1)) * 1024
        else:
            memory_MB = memory.group(1)
        return memory.group(1), threads.group(1)

    else:
        if verbose:
            print('Could not find memory or threads value in raxml-ng output for {alignment}:')
            print(f'\tMemory: {memory}')
            print(f'\tThreads: {threads}')


def replace_all(text, dict):
    """Replace all using dictionary of find:replace"""
    for k, v in dict.items():
        text = text.replace(k, v)
    return text


def make_output_dir(path):
    try:
        os.makedirs(path)
    except FileExistsError:
        pass
    except:
        raise
        sys.exit(1)


def generate_sbatch(alignment, output_path, log_dir, memory, threads, template, prefix, model, no_use_rba):
    """ Generate the individual job script files. """
    try:
        template_fh = open(template, 'r')
    except:
        print(f'Cannot open {template} for reading.')
        sys.exit(1)

    align_root = os.path.basename(alignment)
    prefix = prefix + align_root

    if no_use_rba:
        alignment=os.path.relpath(alignment,output_path)
    else:
        alignment= prefix + '.raxml.rba'

    sbatch_filename= prefix + '.sbatch'
    sbatch_file = os.path.join(output_path, sbatch_filename)

    out_file = prefix + '.out'
    job_out = os.path.join(log_dir, out_file)

    error_file= prefix + '.err'
    job_error = os.path.join(log_dir, error_file)

    make_output_dir(os.path.join(output_path, log_dir))

    info_str = f"This sbatch file autogenerated using the script {__file__} v. {__version__}\n# at {datetime.datetime.now()} from the template at {template}"
    repl_dict = {'%%THREADS%%' : str(threads),
                '%%MEMORY%%MB' : str(memory),
                '%%JOBNAME%%' : str(prefix),
                '%%JOB_OUT%%' : str(job_out),
                '%%JOB_ERROR%%' : str(job_error),
                '%%ALIGNMENT%%' : str(alignment),
                '%%MODEL%%' : str(model),
                '%%PREFIX%%' : str(prefix),
                '%%INFO%%' : info_str}

    try:
        out_fh = open(sbatch_file, 'w')
    except:
        print(f'Cannot open {sbatch_file} for writing.')

    # Replace the %% keys in template file using replace_all function and repl_dict.
    # Then write out the customized sbatch file for th
    for line in template_fh:
        out_line = replace_all(line,repl_dict)
        out_fh.write(out_line)
    
    return sbatch_file

def main():
    parser = argparse.ArgumentParser(prog='raxml_job_generator.py')
    parser.add_argument('-d', '--directory', help='Directory of alignments needing jobs', required=True)
    parser.add_argument('-o', '--output_dir', help='Path to write job scripts files', required=True)
    parser.add_argument('-l', '--log_dir', help='Path to write job logs', default='logs')
    parser.add_argument('-j', '--job_name_prefix', help='prefix for job names', default='')
    parser.add_argument('-m', '--model', help='Model of evolution to use', default='GTR+G')
    parser.add_argument('-t', '--template', help='Job script template to use', default='./RAxML_Gene_Tree.sbatch')
    parser.add_argument('-c', '--max_cpu', help='Max number of CPUs to request', default=32, type=int)
    parser.add_argument('-b', '--mem_buffer', help='Memory buffer to raxml-ng estimate. Should be decimal value 0.0-1.0. Default is 0.15 or 15 percent', default=0.15, type=float)
    parser.add_argument('-r', '--raxml', help='Path to raxml-ng. Required if not in your path.', default='raxml-ng')
    parser.add_argument('-n', '--no_use_rba', help='Do not use the binary MSA file created during parsing in analysis?', action='store_true', default=False)
    parser.add_argument('-s', '--submit', help='Submit jobs after creating job script?', action='store_true', default=False)
    parser.add_argument('-v', '--verbose',action='store_true', default=False)
    parser.add_argument('--version', action='version', version=f'%(prog)s  Version: {__version__}:')

    args = parser.parse_args()

    make_output_dir(args.output_dir)

    # Go through each alignment file (ends in .phy) in input directory.
    # Could be extended to add other accepted file extensions.
    for file in os.listdir(args.directory):
        if file.endswith('.phy'):
            file_full_path = os.path.join(args.directory, file)

            # Get memory and thread estimate for this dataset from raxml-ng --parse
            memory, threads = run_raxml_ng_parse(args.raxml, file_full_path,
                              args.output_dir, args.model, args.job_name_prefix, 
                              args.verbose)

            # Add the memory buffer as % over estimate from raxml-ng
            memory = float(memory) + (float(memory) * float(args.mem_buffer))
            memory = int(math.ceil(memory))
            # If memory is under 100MB, set to 100MB as a minimum request
            if memory < 100:
                memory = 100

            # If threads > max_cpu, set to max_cpu
            if int(threads) > int(args.max_cpu):
                threads = int(args.max_cpu)

            # Generate the sbatch file for this alignment
            script_file = generate_sbatch(file_full_path,args.output_dir, 
                          args.log_dir, memory, threads, args.template, 
                          args.job_name_prefix, args.model, args.no_use_rba)

            # If --submit is passed, submit the job after creating the job script.
            if args.submit:
                submit_result = subprocess.Popen(['sbatch', '--chdir', args.output_dir, script_file],
                                                  stdout=subprocess.PIPE, stderr=subprocess.PIPE)
                if args.verbose:
                    stdout=submit_result.communicate()[0].decode('utf-8')
                    stdout=stdout.strip()
                    print(f'{stdout}\t{file}')
    
    if not args.submit:
        if args.verbose:
            print('Job scripts created, but not submitted.')

if __name__ == '__main__':
    main()

#!/usr/bin/env python

import argparse
import sys, subprocess, os
import re
import datetime
import math

# Generate customized SLURM submit scripts to run RAxML-ng.
#   Takes a directory of alignmets, runs the raxml-ng --parse
#   function to get estimates of RAM and CPU needs for the job.
#   Then uses a templates sbatch script to make a customized
#   script for each job.
# 
#   Other aspects of the job script can be customized too, 
#   see the options in --help.
# 
#   Optionally (using the -s/--submit flag) this script will
#   submit your jobs to the scheduler.
#  
# <NAME>
# University of Florida

__version__= 1.0   #  July 3, 2020

def run_raxml_ng_parse(raxml_path, alignment, output_path, model, prefix, verbose):
    """Run raxml_ng --parse to get estimates of job resource parameters"""

    # Define regex for mem and threads output from raxml-ng
    mem_re = re.compile('Estimated memory requirements\s*:\s*(\d+)\s*([MG]B)')
    thread_re = re.compile('Recommended number of threads / MPI processes:\s*(\d+)')

    align_root = os.path.basename(alignment)
    prefix = prefix + align_root
    prefix = os.path.join(output_path, prefix)

    # Run raxml-ng on alignment to get estimates of memory and threads
    try:
        raxml_result = subprocess.run([raxml_path, '--parse', '--msa', alignment,
                        '--threads', '1', '--model', model, '--prefix', prefix],
                        capture_output=True)
    except FileNotFoundError:
        print('raxml-ng does not appear to be installed in your path. Please use the --raxml flag to specify the path to raxml-ng')
        sys.exit(1) 

    stdout = raxml_result.stdout.decode('utf-8')
    stderr = raxml_result.stderr.decode('utf-8')

    # Check that raxml-ng ran. If not check path to raxml-ng.
    if raxml_result.returncode != 0:
        check_raxml = subprocess.run([raxml_path, '-h'])
        if check_raxml.returncode != 0:
            if raxml_path == 'raxml-ng':
                print('raxml-ng does not appear to be installed in your path. Please use the --raxml flag to specify the path to raxml-ng')
                sys.exit(1)
            else:
                print('Could not run raxml-ng from the path provided. Please check your path.')
                print(f'You specified --raxml as: {raxml_path}')
                sys.exit(1)
        else:
            print('RAxML-ng failed, but produced this output:')
            print(stdout)
            print('\n\nRAxML-ng failed with this error:')
            print(stderr)
            sys.exit(1)
           
    else:
        memory = mem_re.search(stdout)
        threads = thread_re.search(stdout)
        if verbose:
            print(f'Estimates: {alignment}\t{memory.group(1)}{memory.group(2)}\t{threads.group(1)}')

    if memory and threads:
        # Not sure if raxml-ng ever returns estimates in GB, but just in case...
        if memory.group(2) == 'GB':
            memory_MB = int(memory.group(1)) * 1024
        else:
            memory_MB = memory.group(1)
        return memory.group(1), threads.group(1)

    else:
        if verbose:
            print('Could not find memory or threads value in raxml-ng output for {alignment}:')
            print(f'\tMemory: {memory}')
            print(f'\tThreads: {threads}')


def replace_all(text, dict):
    """Replace all using dictionary of find:replace"""
    for k, v in dict.items():
        text = text.replace(k, v)
    return text


def make_output_dir(path):
    try:
        os.makedirs(path)
    except FileExistsError:
        pass
    except:
        raise
        sys.exit(1)


def generate_sbatch(alignment, output_path, log_dir, memory, threads, template, prefix, model, no_use_rba):
    """ Generate the individual job script files. """
    try:
        template_fh = open(template, 'r')
    except:
        print(f'Cannot open {template} for reading.')
        sys.exit(1)

    align_root = os.path.basename(alignment)
    prefix = prefix + align_root

    if no_use_rba:
        alignment=os.path.relpath(alignment,output_path)
    else:
        alignment= prefix + '.raxml.rba'

    sbatch_filename= prefix + '.sbatch'
    sbatch_file = os.path.join(output_path, sbatch_filename)

    out_file = prefix + '.out'
    job_out = os.path.join(log_dir, out_file)

    error_file= prefix + '.err'
    job_error = os.path.join(log_dir, error_file)

    make_output_dir(os.path.join(output_path, log_dir))

    info_str = f"This sbatch file autogenerated using the script {__file__} v. {__version__}\n# at {datetime.datetime.now()} from the template at {template}"
    repl_dict = {'%%THREADS%%' : str(threads),
                '%%MEMORY%%MB' : str(memory),
                '%%JOBNAME%%' : str(prefix),
                '%%JOB_OUT%%' : str(job_out),
                '%%JOB_ERROR%%' : str(job_error),
                '%%ALIGNMENT%%' : str(alignment),
                '%%MODEL%%' : str(model),
                '%%PREFIX%%' : str(prefix),
                '%%INFO%%' : info_str}

    try:
        out_fh = open(sbatch_file, 'w')
    except:
        print(f'Cannot open {sbatch_file} for writing.')

    # Replace the %% keys in template file using replace_all function and repl_dict.
    # Then write out the customized sbatch file for th
    for line in template_fh:
        out_line = replace_all(line,repl_dict)
        out_fh.write(out_line)
    
    return sbatch_file

def main():
    parser = argparse.ArgumentParser(prog='raxml_job_generator.py')
    parser.add_argument('-d', '--directory', help='Directory of alignments needing jobs', required=True)
    parser.add_argument('-o', '--output_dir', help='Path to write job scripts files', required=True)
    parser.add_argument('-l', '--log_dir', help='Path to write job logs', default='logs')
    parser.add_argument('-j', '--job_name_prefix', help='prefix for job names', default='')
    parser.add_argument('-m', '--model', help='Model of evolution to use', default='GTR+G')
    parser.add_argument('-t', '--template', help='Job script template to use', default='./RAxML_Gene_Tree.sbatch')
    parser.add_argument('-c', '--max_cpu', help='Max number of CPUs to request', default=32, type=int)
    parser.add_argument('-b', '--mem_buffer', help='Memory buffer to raxml-ng estimate. Should be decimal value 0.0-1.0. Default is 0.15 or 15 percent', default=0.15, type=float)
    parser.add_argument('-r', '--raxml', help='Path to raxml-ng. Required if not in your path.', default='raxml-ng')
    parser.add_argument('-n', '--no_use_rba', help='Do not use the binary MSA file created during parsing in analysis?', action='store_true', default=False)
    parser.add_argument('-s', '--submit', help='Submit jobs after creating job script?', action='store_true', default=False)
    parser.add_argument('-v', '--verbose',action='store_true', default=False)
    parser.add_argument('--version', action='version', version=f'%(prog)s  Version: {__version__}:')

    args = parser.parse_args()

    make_output_dir(args.output_dir)

    # Go through each alignment file (ends in .phy) in input directory.
    # Could be extended to add other accepted file extensions.
    for file in os.listdir(args.directory):
        if file.endswith('.phy'):
            file_full_path = os.path.join(args.directory, file)

            # Get memory and thread estimate for this dataset from raxml-ng --parse
            memory, threads = run_raxml_ng_parse(args.raxml, file_full_path,
                              args.output_dir, args.model, args.job_name_prefix, 
                              args.verbose)

            # Add the memory buffer as % over estimate from raxml-ng
            memory = float(memory) + (float(memory) * float(args.mem_buffer))
            memory = int(math.ceil(memory))
            # If memory is under 100MB, set to 100MB as a minimum request
            if memory < 100:
                memory = 100

            # If threads > max_cpu, set to max_cpu
            if int(threads) > int(args.max_cpu):
                threads = int(args.max_cpu)

            # Generate the sbatch file for this alignment
            script_file = generate_sbatch(file_full_path,args.output_dir, 
                          args.log_dir, memory, threads, args.template, 
                          args.job_name_prefix, args.model, args.no_use_rba)

            # If --submit is passed, submit the job after creating the job script.
            if args.submit:
                submit_result = subprocess.Popen(['sbatch', '--chdir', args.output_dir, script_file],
                                                  stdout=subprocess.PIPE, stderr=subprocess.PIPE)
                if args.verbose:
                    stdout=submit_result.communicate()[0].decode('utf-8')
                    stdout=stdout.strip()
                    print(f'{stdout}\t{file}')
    
    if not args.submit:
        if args.verbose:
            print('Job scripts created, but not submitted.')

if __name__ == '__main__':
    main()

#!/usr/bin/env python # Generate customized SLURM submit scripts to run RAxML-ng. #   Takes a directory of alignmets, runs the raxml-ng --parse #   function to get estimates of RAM and CPU needs for the job. #   Then uses a templates sbatch script to make a customized #   script for each job. # #   Other aspects of the job script can be customized too, #   see the options in --help. # #   Optionally (using the -s/--submit flag) this script will #   submit your jobs to the scheduler. # # <NAME> # University of Florida #  July 3, 2020 Run raxml_ng --parse to get estimates of job resource parameters # Define regex for mem and threads output from raxml-ng # Run raxml-ng on alignment to get estimates of memory and threads # Check that raxml-ng ran. If not check path to raxml-ng. # Not sure if raxml-ng ever returns estimates in GB, but just in case... Replace all using dictionary of find:replace Generate the individual job script files. # at {datetime.datetime.now()} from the template at {template}" # Replace the %% keys in template file using replace_all function and repl_dict. # Then write out the customized sbatch file for th # Go through each alignment file (ends in .phy) in input directory. # Could be extended to add other accepted file extensions. # Get memory and thread estimate for this dataset from raxml-ng --parse # Add the memory buffer as % over estimate from raxml-ng # If memory is under 100MB, set to 100MB as a minimum request # If threads > max_cpu, set to max_cpu # Generate the sbatch file for this alignment # If --submit is passed, submit the job after creating the job script.