def create_env_def_file(env_filename, runjson, language):
    # I have to use these variables after this script finishes running.
    # I didn't use os.environ + os.system('bash') because that would remove the other
    # env variables set before this script started running.
    app = runjson.Job.App
    with open(env_filename, 'w') as f_env:
        f_env.write("export LANGUAGE={}\n".format(app.language))
        if language in ['wdl', 'wdl_v1', 'wdl_draft2']:
            f_env.write("export WDL_URL={}\n".format(app.wdl_url))
            f_env.write("export MAIN_WDL={}\n".format(app.main_wdl))
            f_env.write("export WORKFLOW_ENGINE={}\n".format(app.workflow_engine))
            f_env.write("export WDL_FILES=\"{}\"\n".format(' '.join(app.other_wdl_files.split(','))))
        elif language == 'snakemake':
            f_env.write("export SNAKEMAKE_URL={}\n".format(app.snakemake_url))
            f_env.write("export MAIN_SNAKEMAKE={}\n".format(app.main_snakemake))
            f_env.write("export SNAKEMAKE_FILES=\"{}\"\n".format(' '.join(app.other_snakemake_files.split(','))))
            f_env.write("export COMMAND=\"{}\"\n".format(app.command.replace("\"", "\\\"")))
            f_env.write("export CONTAINER_IMAGE={}\n".format(app.container_image))
        elif language == 'shell':
            f_env.write("export COMMAND=\"{}\"\n".format(app.command.replace("\"", "\\\"")))
            f_env.write("export CONTAINER_IMAGE={}\n".format(app.container_image))
        else:  # cwl
            f_env.write("export CWL_URL={}\n".format(app.cwl_url))
            f_env.write("export MAIN_CWL={}\n".format(app.main_cwl))
            f_env.write("export CWL_FILES=\"{}\"\n".format(' '.join(app.other_cwl_files.split(','))))
        # other env variables
        env_preserv_str = ''
        docker_env_str = ''
        if runjson.Job.Input.Env:
            for ev, val in sorted(runjson.Job.Input.Env.items()):
                f_env.write("export {}={}\n".format(ev, val))
                env_preserv_str = env_preserv_str + "--preserve-environment " + ev + " "
                docker_env_str = docker_env_str + "-e " + ev + " "
        f_env.write("export PRESERVED_ENV_OPTION=\"{}\"\n".format(env_preserv_str))
        f_env.write("export DOCKER_ENV_OPTION=\"{}\"\n".format(docker_env_str)) def create_mount_command_list(mountlist_filename, runjson_input,
                              kms_key_id=None):
    """ This function creates a mount point directory and starts goofys
        with some default arguments.
        Note that KMS key arguments are needed for the mount if encryption
        is enabled.
    """
    buckets_to_be_mounted = set()
    for category in ["Input_files_data", "Secondary_files_data"]:
        for inkey, v in getattr(runjson_input, category).items():
            if v.mount:
                buckets_to_be_mounted.add(v.dir_)
    with open(mountlist_filename, 'w') as f:
        for b in sorted(buckets_to_be_mounted):
            f.write("mkdir -p %s\n" % (INPUT_MOUNT_DIR_PREFIX + b))
            if kms_key_id:
                f.write("goofys --sse-kms %s -f %s %s &\n" % (kms_key_id, b, INPUT_MOUNT_DIR_PREFIX + b))
            else:
                f.write("goofys -f %s %s &\n" % (b, INPUT_MOUNT_DIR_PREFIX + b)) def create_download_command_list(downloadlist_filename, runjson_input):
    """create a download command list file from the information in json"""
    with open(downloadlist_filename, 'w') as f:
        for category in ["Input_files_data", "Secondary_files_data"]:
            for inkey, v in getattr(runjson_input, category).items():
                if v.mount:  # do not download if it will be mounted
                    continue
                if inkey.startswith('file://'):
                    target = inkey.replace('file://', '')
                    print("key %s will be downloaded to target %s" % (v.path, inkey))
                    run_on_nested_arrays2(v.path, target, add_download_cmd, data_bucket=v.dir_,
                                          profile=v.profile, f=f, unzip=v.unzip)
                else:
                    target_template = INPUT_DIR + "/%s"
                    if not v.rename or len(flatten(v.rename)) == 0:
                        rename = create_dim(v.path, empty=True)
                    else:
                        rename = v.rename
                    run_on_nested_arrays2(v.path, rename, add_download_cmd, data_bucket=v.dir_,
                                          profile=v.profile, f=f, unzip=v.unzip, target_template=target_template) def create_download_cmd(data_bucket, data_file, target, profile, unzip=''):
    profile_flag = ' --profile ' + profile if profile else ''
    format_list = [data_bucket, data_file, target, profile_flag]
    key_type = determine_key_type(data_bucket, data_file, profile)
    if key_type == 'Does not exist':
        raise Exception("Cannot download file s3://%s/%s - file does not exist." % (data_bucket, data_file))
    elif key_type == 'File':
        download_cmd = 'aws s3 cp s3://{0}/{1} {2}{3}'.format(*format_list)
        if unzip == 'gz':
            unzip_cmd = 'gunzip {2}'
        elif unzip == 'bz2':
            unzip_cmd = 'bzip2 -d {2}'
        else:
            unzip_cmd = ''
        cmd = download_cmd + '; ' + unzip_cmd
        return cmd.format(*format_list)
    else: # key_type == 'Folder':
        download_cmd = 'aws s3 cp --recursive s3://{0}/{1} {2}{3}'.format(*format_list)
        if unzip == 'gz':
            unzip_cmd = 'for f in `find {2} -type f`; do if [[ $f =~ \\.gz$ ]]; then gunzip $f; fi; done;'
        elif unzip == 'bz2':
            unzip_cmd = 'for f in `find {2} -type f`; do if [[ $f =~ \\.bz2$ ]]; then bzip2 -d $f; fi; done;'
        else:
            unzip_cmd = ''
        cmd = download_cmd + '; ' + unzip_cmd
        return cmd.format(*format_list) def add_download_cmd(data_file, rename, data_bucket, profile, f, unzip, target_template='%s'):
    if data_file:
        if not rename:
            rename = data_file
        target = target_template % rename
        cmd = create_download_cmd(data_bucket, data_file, target, profile, unzip)
        f.write(cmd + '\n') def determine_key_type(bucket, key, profile):
    """Return values : 'File', 'Folder' or 'Does not exist'"""
    if profile:
        s3 = boto3.session(profile_name=profile).client('s3')
    else:
        s3 = boto3.client('s3')
    if not key:
        raise Exception("Cannot determine key type - no key is specified")
    if not bucket:
        raise Exception("Cannot determine key type - no bucket is specified")
    if key.endswith('/'):
        key = key.rstrip('/')
    res = s3.list_objects_v2(Bucket=bucket, Prefix=key + '/')
    if not 'KeyCount' in res:
        raise Exception("Cannot determine key type - no response from S3")
    if res['KeyCount'] == 0:
        res2 = s3.list_objects_v2(Bucket=bucket, Prefix=key)
        if not 'KeyCount' in res2:
            raise Exception("Cannot determine key type - no response from S3")
        elif res2['KeyCount'] == 0:
            return 'Does not exist'  # key does not exist
        # The file itself may be a prefix of another file (e.v. abc.vcf.gz vs abc.vcf.gz.tbi)
        # but it doesn't matter.
        else:
            return 'File'
    else:
        # data_file is a folder
        return 'Folder' def create_output_files_dict(language='cwl', execution_metadata=None, md5dict=None, strict=True):
    """create a dictionary that contains 'path', 'secondaryFiles', 'md5sum' with argnames as keys.
    For snakemake and shell, returns an empty dictionary (execution_metadata not required).
    secondaryFiles is added only if the language is cwl.
    execution_metadata is a dictionary read from wdl/cwl execution log json file.
    md5dict is a dictionary with key=file path, value=md5sum (optional).
    if strict is set False, then it does not check executio_metadata exists for cwl/wdl."""
    if language in ['cwl', 'cwl_v1', 'wdl', 'wdl_v1', 'wdl_draft2'] and not execution_metadata and strict:
        raise Exception("execution_metadata is required for cwl/wdl.")
    out_meta = dict()
    if language in ['wdl', 'wdl_v1', 'wdl_draft2']:
        for argname, outfile in execution_metadata['outputs'].items():
            if outfile:
                out_meta[argname] = {'path': outfile}
    elif language == 'snakemake' or language == 'shell':
        out_meta = {}
    else:  # cwl, cwl_v1
        # read cwl output json file
        out_meta = execution_metadata

    # add md5
    if not md5dict:
        md5dict = {}
    for of, ofv in out_meta.items():
        if ofv['path'] in md5dict:
            ofv['md5sum'] = md5dict[ofv['path']]
        if 'secondaryFiles' in ofv:
            for sf in ofv['secondaryFiles']:
                if sf['path'] in md5dict:
                    sf['md5sum'] = md5dict[sf['path']]

    return out_meta def read_md5file(md5file):
    with open(md5file, 'r') as md5_f:
        md5dict = dict()
        for line in md5_f:
            a = line.split()
            path = a[1]
            md5sum = a[0]
            md5dict[path] = md5sum
    return md5dict def update_postrun_json_init(json_old, json_new):
    """Update postrun json with just instance ID, filesystem and availability zone"""
    # read old json file
    prj = read_postrun_json(json_old)

    # simply add instance ID, file system and availability zone
    prj.Job.instance_id = os.getenv('INSTANCE_ID')
    prj.Job.filesystem = os.getenv('EBS_DEVICE')
    prj.Job.instance_availablity_zone = os.getenv('INSTANCE_AVAILABILITY_ZONE')

    # write to new json file
    write_postrun_json(json_new, prj) def update_postrun_json_upload_output(json_old, execution_metadata_file, md5file, json_new,
                                      language='cwl_v1', strict=True, upload=True, endpoint_url=None):
    """Update postrun json with output files.
    if strict is set false, it does not check execution metadata is required for cwl/wdl."""
    # read old json file and prepare postrunjson skeleton
    prj = read_postrun_json(json_old)

    # read md5 file
    md5dict = read_md5file(md5file)

    # read execution metadata file
    if execution_metadata_file:
        with open(execution_metadata_file, 'r') as f:
            execution_metadata = json.load(f)
    else:
        execution_metadata = None
    output_files = create_output_files_dict(language, execution_metadata, md5dict, strict=strict)

    # create output files for postrun json
    prj.Job.Output.add_output_files(output_files)

    # upload output to S3 (this also updates postrun json)
    if upload:
        upload_output(prj, endpoint_url=endpoint_url)

    # write to new json file
    write_postrun_json(json_new, prj) def postrun_json_final(prj, logfile=None):
    # add commands
    if logfile:
        print("parsing commands from log file...")
        log_content = log.read_logfile_by_line(logfile)
        prj.update(commands=log.parse_commands(log_content))
    # add end time, status, instance_id
    prj_job = prj.Job
    prj_job.update(end_time=time.strftime("%Y%m%d-%H:%M:%S-%Z"))
    prj_job.update(status=os.getenv('JOB_STATUS'))
    prj_job.update(total_input_size=os.getenv('INPUTSIZE'))
    prj_job.update(total_tmp_size=os.getenv('TEMPSIZE'))
    prj_job.update(total_output_size=os.getenv('OUTPUTSIZE')) def upload_postrun_json(jsonfile):
    prj = read_postrun_json(jsonfile)
    bucket = prj.Job.Log.log_bucket_directory
    dest = prj.Job.JOBID + '.postrun.json'
    if '/' in bucket:
        bucket_dirs = bucket.split('/')
        bucket = bucket_dirs.pop(0)
        prefix = '/'.join(bucket_dirs)
        dest = prefix + '/' + dest
    if prj.config.public_postrun_json:
        acl = 'public-read'
    else:
        acl = 'private'
    s3 = boto3.client('s3')
    upload_arg = {
        "Body": format_postrun_json(prj).encode('utf-8'),
        "Bucket": bucket,
        "Key": dest,
        "ACL": acl
    }
    if prj.config.encrypt_s3_upload:
        upload_arg.update({"ServerSideEncryption": "aws:kms"})
        if prj.config.kms_key_id:
            upload_arg['SSEKMSKeyId'] = prj.config.kms_key_id
    s3.put_object(**upload_arg) def upload_to_output_target(prj_out, encrypt_s3_upload=False, kms_key_id=None, endpoint_url=None):
    # parsing output_target and uploading output files to output target
    output_bucket = prj_out.output_bucket_directory
    output_argnames = prj_out.output_files.keys()
    output_target = prj_out.alt_output_target(output_argnames)

    for k in output_target:
        target = Target(output_bucket)

        # 'file://' output targets
        if target.is_custom_target(k):
            print("processing custom (path-based) target %s" % k)
            target.parse_custom_target(k, output_target[k])
            if target.is_valid:
                print("Target is valid. Uploading..")
                target.upload_to_s3(encrypt_s3_upload=encrypt_s3_upload, endpoint_url=endpoint_url)
            else:
                raise Exception("Invalid target %s -> %s: failed to upload" % k, output_target[k])
        else:
            # legitimate CWL/WDL output targets
            print("processing argument-based target %s" % k)
            target.parse_cwl_target(k, output_target.get(k, ''), prj_out.output_files)
            if target.is_valid:
                print("Target is valid. Uploading..")
                target.upload_to_s3(encrypt_s3_upload=encrypt_s3_upload,
                                    kms_key_id=kms_key_id, endpoint_url=endpoint_url)
                prj_out.output_files[k].add_target(target.dest)

                # upload secondary files
                secondary_output_files = prj_out.output_files[k].secondaryFiles
                if secondary_output_files:
                    stlist = SecondaryTargetList(output_bucket)
                    stlist.parse_target_values(prj_out.secondary_output_target.get(k, []))
                    stlist.reorder_by_source([sf.path for sf in secondary_output_files])
                    for st in stlist.secondary_targets:
                        st.upload_to_s3(encrypt_s3_upload=encrypt_s3_upload,
                                        kms_key_id=kms_key_id, endpoint_url=endpoint_url)
                    for i, sf in enumerate(secondary_output_files):
                        sf.add_target(stlist.secondary_targets[i].dest)
            else:
                raise Exception("Failed to upload to output target %s" % k) def upload_output(prj, endpoint_url=None):
    # parsing output_target and uploading output files to output target
    upload_to_output_target(prj.Job.Output, prj.config.encrypt_s3_upload,
                            kms_key_id=prj.config.kms_key_id, endpoint_url=endpoint_url) def parse_commands(log_content):
    """ 
    parse cwl commands from the line-by-line generator of log file content and 
    returns the commands as a list of command line lists, each corresponding to a step run.
    """ 
    command_list = []
    command = []
    in_command = False
    line = next(log_content)
    while(line):
        line = line.strip('\n')
        if '[job' in line and line.endswith('docker \\'):
            line = 'docker \\'  # remove the other stuff
            in_command = True
        if in_command:
            command.append(line.strip('\\').rstrip(' ')) 
            if not line.endswith('\\'):
                in_command = False
                command_list.append(command)
                command = []
        line = next(log_content)
    return(command_list) def read_logfile_by_line(logfile):
    """generator function that yields the log file content line by line"""
    with open(logfile, 'r') as f:
        for line in f:
            yield line
    yield None class AwsemRunJson(SerializableObject):
    def __init__(self, Job=None, config=None, strict=True):
        if strict:
            if not Job or not config:
                raise MalFormattedPostRunJsonException("Job and config are required fields.")
        if not Job:
            Job = {}
        self.create_Job(Job, strict=strict)
        self.config = Config(**config) if config else None

    def create_Job(self, Job, strict=True):
        self.Job = AwsemRunJsonJob(**Job, strict=strict) class AwsemRunJsonInput(SerializableObject):
    def __init__(self, Input_files_data=None, Input_parameters=None, Secondary_files_data=None,
                 # Input_files_reference is for older postrunjson
                 # Env is missing in older postrunjson
                 Input_files_reference=None, Env=None):
        if not Input_files_data:
            Input_files_data = {}
        if not Input_parameters:
            Input_parameters = {}
        if not Secondary_files_data:
            Secondary_files_data = {}
        self.Input_files_data = {k: AwsemRunJsonInputFile(**v) for k, v in Input_files_data.items()}
        self.Secondary_files_data = {k: AwsemRunJsonInputFile(**v) for k, v in Secondary_files_data.items()}
        self.Input_parameters = Input_parameters
        self.Env = Env
        if Input_files_reference:
            self.Input_files_reference = {k: AwsemRunJsonInputFile(**v) for k, v in Input_files_reference.items()}

    def as_dict_as_cwl_input(self, input_dir='', input_mount_dir_prefix=''):
        d = {k: v.as_dict_as_cwl_input(input_dir, input_mount_dir_prefix) for k, v in self.Input_files_data.items()}
        d.update(self.Input_parameters)
        return d

    def as_dict_as_wdl_input(self, input_dir='', input_mount_dir_prefix=''):
        d = {k: v.as_dict_as_wdl_input(input_dir, input_mount_dir_prefix) for k, v in self.Input_files_data.items()}
        d.update(self.Input_parameters)
        return d

    def check_input_files_key_compatibility(self, language):
        for category in ["Input_files_data", "Secondary_files_data"]:
            for inkey in getattr(self, category):
                if inkey.startswith('file://'):
                    if language not in ['shell', 'snakemake']:
                        raise MalFormattedRunJsonException('input file has to be defined with argument name for CWL and WDL')
                    target = inkey.replace('file://', '') 
                    if not target.startswith('/data1/'):
                        raise Exception('input target directory must be in /data1/')
                    if not target.startswith('/data1/' + language) and \
                        not target.startswith('/data1/input') and \
                        not target.startswith('/data1/out'):
                            raise Exception('input target directory must be in /data1/input, /data1/out or /data1/%s' % language) class AwsemPostRunJsonOutput(AwsemRunJsonOutput):
    def __init__(self, output_bucket_directory=None, output_target=None,
                 secondary_output_target=None, alt_cond_output_argnames=None,
                 **kwargs):  # kwargs includes 'Output files'
        """This class has an additional 'Output files' field which
        stores the output from CWL/WDL runs"""
        super().__init__(output_bucket_directory, output_target,
                         secondary_output_target, alt_cond_output_argnames)
        if 'Output files' in kwargs:
            self.add_output_files(kwargs['Output files'])
        else:
            self.Output_files_ = {}

    @property
    def output_files(self):
        return self.Output_files_

    def add_output_files(self, output_files):
        """add or replace output files. output_files is a dictionary with argnames as keys
           and a dict form of AwsemPostRunJsonOutputFile objects as values"""
        self.Output_files_ = {k: AwsemPostRunJsonOutputFile(**v) for k, v in output_files.items()}

    def as_dict(self):
        d = super().as_dict()
        # handling reserved name key
        if 'Output_files_' in d:
            d['Output files'] = d['Output_files_']
            del(d['Output_files_'])
        return d class AwsemPostRunJson(AwsemRunJson):
    def __init__(self, Job=None, config=None, commands=None,log=None, strict=True):
        if strict:
            if not Job or not config:
                raise MalFormattedPostRunJsonException("Job and config are required fields.")
        super().__init__(Job, config, strict=strict)
        if commands:
            self.commands = commands
        if log:
            self.log = log

    def add_commands(self, command):
        self.command = command

    def create_Job(self, Job, strict=True):
        self.Job = AwsemPostRunJsonJob(**Job, strict=strict) class CaptureOut:
    def __init__(self):
    def get_captured_out(self):
    def __enter__(self):
    def __exit__(self, type, value, tb):
import os
import pytest
import json
import boto3
from datetime import datetime
from awsf3.utils import (
    create_env_def_file,
    create_mount_command_list,
    create_download_command_list,
    create_download_cmd,
    add_download_cmd,
    determine_key_type,
    create_output_files_dict,
    read_md5file,
    update_postrun_json_init,
    update_postrun_json_upload_output,
    postrun_json_final,
    upload_postrun_json,
    upload_to_output_target,
    upload_output
)
from awsf3.log import (
    parse_commands,
    read_logfile_by_line
)
from tibanna.awsem import (
    AwsemRunJson,
    AwsemRunJsonInput,
    AwsemPostRunJsonOutput,
    AwsemPostRunJson
)
from tests.awsf3.conftest import upload_test_bucket
    runjson = AwsemRunJson(**runjson_dict)
    create_env_def_file(envfilename, runjson, 'wdl')

    with open(envfilename, 'r') as f:
        envfile_content = f.read()

    right_content = ('export LANGUAGE=wdl\n'
                     'export WDL_URL=someurl\n'
                     'export MAIN_WDL=somewdl\n'
                     'export WORKFLOW_ENGINE=cromwell\n'
                     'export WDL_FILES="otherwdl1 otherwdl2"\n'
                     'export PRESERVED_ENV_OPTION=""\n'
                     'export DOCKER_ENV_OPTION=""\n')

    assert envfile_content == right_content
    os.remove(envfilename)


def test_create_env_def_file_shell():
    """testing create_env_def_file with shell option and two input Env variables"""
    envfilename = 'someenvfile'

    runjson_dict = {'Job': {'App': {'language': 'shell',
                                    'command': 'com1;com2',
                                    'container_image': 'someimage'},
                            'Input': {'Env': {'ENV1': '1234', 'ENV2': '5678'}},
                            'Output': {'output_bucket_directory': 'somebucket'},
                            'JOBID': 'somejobid'},
                    'config': {'log_bucket': 'somebucket'}}
    runjson = AwsemRunJson(**runjson_dict)
    create_env_def_file(envfilename, runjson, 'shell')

    with open(envfilename, 'r') as f:
        envfile_content = f.read()

    right_content = ('export LANGUAGE=shell\n'
                     'export COMMAND="com1;com2"\n'
                     'export CONTAINER_IMAGE=someimage\n'
                     'export ENV1=1234\n'
                     'export ENV2=5678\n'
                     'export PRESERVED_ENV_OPTION="--preserve-environment ENV1 --preserve-environment ENV2 "\n'
                     'export DOCKER_ENV_OPTION="-e ENV1 -e ENV2 "\n')

    assert envfile_content == right_content
    os.remove(envfilename)


def test_create_env_def_file_shell2():
    """testing create_env_def_file with shell option with complex commands and an env variable"""
    envfilename = 'someenvfile'

    complex_command = 'echo $SOME_ENV | xargs -i echo {} > somedir/somefile'
    runjson_dict = {'Job': {'App': {'language': 'shell',
                                    'command': complex_command,
                                    'container_image': 'someimage'},
                            'Input': {'Env': {'SOME_ENV': '1234'}},
                            'Output': {'output_bucket_directory': 'somebucket'},
                            'JOBID': 'somejobid'},
                    'config': {'log_bucket': 'somebucket'}}
    runjson = AwsemRunJson(**runjson_dict)
    create_env_def_file(envfilename, runjson, 'shell')

    with open(envfilename, 'r') as f:
        envfile_content = f.read()

    right_content = ('export LANGUAGE=shell\n'
                     'export COMMAND="echo $SOME_ENV | xargs -i echo {} > somedir/somefile"\n'
                     'export CONTAINER_IMAGE=someimage\n'
                     'export SOME_ENV=1234\n'
                     'export PRESERVED_ENV_OPTION="--preserve-environment SOME_ENV "\n'
                     'export DOCKER_ENV_OPTION="-e SOME_ENV "\n')

    assert envfile_content == right_content
    os.remove(envfilename)


def test_create_env_def_file_shell3():
    """testing create_env_def_file with shell option with complex commands and an env variable.
    double-quotes are escaped when written to the env file ('"' -> '\"')"""
    envfilename = 'someenvfile'

    complex_command = 'echo "haha" > somefile; ls -1 [st]*'
    runjson_dict = {'Job': {'App': {'language': 'shell',
                                    'command': complex_command,
                                    'container_image': 'someimage'},
                            'Input': {'Env': {}},
                            'Output': {'output_bucket_directory': 'somebucket'},
                            'JOBID': 'somejobid'},
                    'config': {'log_bucket': 'somebucket'}}
    runjson = AwsemRunJson(**runjson_dict)
    create_env_def_file(envfilename, runjson, 'shell')

    with open(envfilename, 'r') as f:
        envfile_content = f.read()

    right_content = ('export LANGUAGE=shell\n'
                     'export COMMAND="echo \\"haha\\" > somefile; ls -1 [st]*"\n'
                     'export CONTAINER_IMAGE=someimage\n'
                     'export PRESERVED_ENV_OPTION=""\n'
                     'export DOCKER_ENV_OPTION=""\n')

    assert envfile_content == right_content
    os.remove(envfilename)


def test_create_env_def_file_snakemake():
    """testing create_env_def_file with shell option and two input Env variables"""
    envfilename = 'someenvfile'

    runjson_dict = {'Job': {'App': {'language': 'snakemake',
                                    'command': 'com1;com2',
                                    'container_image': 'someimage',
                                    'snakemake_url': 'someurl',
                                    'main_snakemake': 'somesnakemake',
                                    'other_snakemake_files': 'othersnakemake1,othersnakemake2'},
                            'JOBID': 'somejobid',
                            'Input': {},
                            'Output': {'output_bucket_directory': 'somebucket'},
                            'JOBID': 'somejobid'},
                    'config': {'log_bucket': 'somebucket'}}

    runjson = AwsemRunJson(**runjson_dict)