<filename>SLpackage/private/pacbio/pythonpkgs/pbsvtools/lib/python2.7/site-packages/pbsvtools/tasks/scatter_align_json_to_svsig.py
#! python

"""
Scatter inputs of pbsvtools.tasks.align_json_to_svsig

align takes two inputs:
    Input: idx 0 - datastore.json containing a list of AlignmentSet files,
                   or a list of ConsensusAlignmentSet files.
           idx 1 - TRF bed file
    Output: idx 0 - FOFN of svsig.gz files
"""
from __future__ import absolute_import

import logging
import sys
import math
import os.path as op

from pbcommand.pb_io import write_pipeline_chunks
from pbcommand.models import FileTypes, PipelineChunk

from pbcoretools.datastore_utils import dataset_to_datastore, datastore_to_datastorefile_objs
from .align_json_to_svsig import Constants as BC
from .scatter_call import put_items_to_boxes
from ..basic import scatter_main, TCP_INPUT, TCP_OUTPUT, BaseScatterConstants


log = logging.getLogger(__name__)


class Constants(BaseScatterConstants):
    """Constants must be identical to align_json_to_svsig
    Input: idx 0 - DataStore json of AlignmentSet (or ConsensusAlignmentSet)
           idx 1 - TRF BED
    Output: idx 0 - FOFN of svsig.gz
    """
    TOOL_NAME = 'scatter_{}'.format(BC.TOOL_NAME)
    DESCRIPTION = "Scatter inputs for pbsvtools.tasks.{}".format(TOOL_NAME)
    CHUNK_KEYS = ('$chunk.datastore_id', '$chunk.bed_id')
    INPUT_FILES = BC.INPUT_FILES
    OUTPUT_FILES = [
        TCP_OUTPUT(FileTypes.CHUNK, "cjson_out", "Chunk sv datastore JSON",
                   "Chunk sv datastore JSON", "align.datastore.chunked")
    ]
    ALLOWED_TYPES = BC.ALLOWED_TYPES


def datastore_to_bam_files(i_datastore_fn):
    """
    Return all external rescources bam files from input datastore json file.
    """
    files, _, readcls, _ = datastore_to_datastorefile_objs(
            i_datastore_fn, allowed_types=Constants.ALLOWED_TYPES)
    dataset_obj = readcls(*[f.path for f in files])
    return dataset_obj.toExternalFiles()


def run_main(i_datastore_fn, i_trf_fn, o_json_fn, max_nchunks):
    """
    Parameters:
      i_datastore_fn --- DataStore json of AlignmentSet or ConsensusAlignmentSet to chunk.
      i_trf_fn --- Tandem Repeats in BED
      o_json_fn -- Output json file
    """
    output_dir = op.dirname(o_json_fn)
    basename = 'chunk'

    # Chunk input datastore json, generate multiple chunked datastore.json, and
    # generate pbcommand.models.PipelineChunk objects
    _, _, readcls, ext = datastore_to_datastorefile_objs(
            i_datastore_fn, allowed_types=Constants.ALLOWED_TYPES)
    bam_fns = datastore_to_bam_files(i_datastore_fn)

    # Put bam files into boxes
    n_chunks = max(1, min(max_nchunks, len(bam_fns)))
    cutoff = math.ceil(len(bam_fns)*1.0/n_chunks)
    boxes = put_items_to_boxes(bam_fns, [1 for _ in range(len(bam_fns))], n_chunks, cutoff)

    chunks = []
    for i, bam_fns_in_box in enumerate(boxes):
        out_xml = op.join(output_dir, '{}.{}.{}'.format(basename, i, ext))
        out_json = op.join(output_dir, '{}.{}.{}'.format(basename, i, 'datastore.json'))
        readcls(*bam_fns_in_box).write(out_xml)
        dataset_to_datastore(out_xml, out_json, Constants.TOOL_NAME)

        # Create a chunk: get $chunk.datastore_id from chunk,
        # use TandemRepeat masker bed as $chunk.bed_id
        d = {Constants.CHUNK_KEYS[0]: out_json,
             Constants.CHUNK_KEYS[1]: i_trf_fn}
        chunk_id = Constants.TOOL_NAME+'_chunk_{}'.format(i)  # chunks MUST have unique IDs
        chunk = PipelineChunk(chunk_id, **d)
        chunks.append(chunk)

    log.info("Writing chunk.json to %s", o_json_fn)
    write_pipeline_chunks(chunks, o_json_fn,
                          "created by %s" % Constants.TOOL_ID())
    return 0


def rtc_runner(rtc):
    """Resolved tool contract runner."""
    max_nchunks = rtc.task.max_nchunks if hasattr(
        rtc.task, 'max_nchunks') else Constants.DEFAULT_NCHUNKS
    return run_main(i_datastore_fn=rtc.task.input_files[0],
                    i_trf_fn=rtc.task.input_files[1],
                    o_json_fn=rtc.task.output_files[0],
                    max_nchunks=int(max_nchunks))


if __name__ == '__main__':
    sys.exit(scatter_main(
        args=sys.argv[1:], const=Constants, rtc_runner=rtc_runner, alog=log))

<filename>SLpackage/private/pacbio/pythonpkgs/pbsvtools/lib/python2.7/site-packages/pbsvtools/tasks/scatter_align_json_to_svsig.py
#! python

"""
Scatter inputs of pbsvtools.tasks.align_json_to_svsig

align takes two inputs:
    Input: idx 0 - datastore.json containing a list of AlignmentSet files,
                   or a list of ConsensusAlignmentSet files.
           idx 1 - TRF bed file
    Output: idx 0 - FOFN of svsig.gz files
"""
from __future__ import absolute_import

import logging
import sys
import math
import os.path as op

from pbcommand.pb_io import write_pipeline_chunks
from pbcommand.models import FileTypes, PipelineChunk

from pbcoretools.datastore_utils import dataset_to_datastore, datastore_to_datastorefile_objs
from .align_json_to_svsig import Constants as BC
from .scatter_call import put_items_to_boxes
from ..basic import scatter_main, TCP_INPUT, TCP_OUTPUT, BaseScatterConstants


log = logging.getLogger(__name__)


class Constants(BaseScatterConstants):
    """Constants must be identical to align_json_to_svsig
    Input: idx 0 - DataStore json of AlignmentSet (or ConsensusAlignmentSet)
           idx 1 - TRF BED
    Output: idx 0 - FOFN of svsig.gz
    """
    TOOL_NAME = 'scatter_{}'.format(BC.TOOL_NAME)
    DESCRIPTION = "Scatter inputs for pbsvtools.tasks.{}".format(TOOL_NAME)
    CHUNK_KEYS = ('$chunk.datastore_id', '$chunk.bed_id')
    INPUT_FILES = BC.INPUT_FILES
    OUTPUT_FILES = [
        TCP_OUTPUT(FileTypes.CHUNK, "cjson_out", "Chunk sv datastore JSON",
                   "Chunk sv datastore JSON", "align.datastore.chunked")
    ]
    ALLOWED_TYPES = BC.ALLOWED_TYPES


def datastore_to_bam_files(i_datastore_fn):
    """
    Return all external rescources bam files from input datastore json file.
    """
    files, _, readcls, _ = datastore_to_datastorefile_objs(
            i_datastore_fn, allowed_types=Constants.ALLOWED_TYPES)
    dataset_obj = readcls(*[f.path for f in files])
    return dataset_obj.toExternalFiles()


def run_main(i_datastore_fn, i_trf_fn, o_json_fn, max_nchunks):
    """
    Parameters:
      i_datastore_fn --- DataStore json of AlignmentSet or ConsensusAlignmentSet to chunk.
      i_trf_fn --- Tandem Repeats in BED
      o_json_fn -- Output json file
    """
    output_dir = op.dirname(o_json_fn)
    basename = 'chunk'

    # Chunk input datastore json, generate multiple chunked datastore.json, and
    # generate pbcommand.models.PipelineChunk objects
    _, _, readcls, ext = datastore_to_datastorefile_objs(
            i_datastore_fn, allowed_types=Constants.ALLOWED_TYPES)
    bam_fns = datastore_to_bam_files(i_datastore_fn)

    # Put bam files into boxes
    n_chunks = max(1, min(max_nchunks, len(bam_fns)))
    cutoff = math.ceil(len(bam_fns)*1.0/n_chunks)
    boxes = put_items_to_boxes(bam_fns, [1 for _ in range(len(bam_fns))], n_chunks, cutoff)

    chunks = []
    for i, bam_fns_in_box in enumerate(boxes):
        out_xml = op.join(output_dir, '{}.{}.{}'.format(basename, i, ext))
        out_json = op.join(output_dir, '{}.{}.{}'.format(basename, i, 'datastore.json'))
        readcls(*bam_fns_in_box).write(out_xml)
        dataset_to_datastore(out_xml, out_json, Constants.TOOL_NAME)

        # Create a chunk: get $chunk.datastore_id from chunk,
        # use TandemRepeat masker bed as $chunk.bed_id
        d = {Constants.CHUNK_KEYS[0]: out_json,
             Constants.CHUNK_KEYS[1]: i_trf_fn}
        chunk_id = Constants.TOOL_NAME+'_chunk_{}'.format(i)  # chunks MUST have unique IDs
        chunk = PipelineChunk(chunk_id, **d)
        chunks.append(chunk)

    log.info("Writing chunk.json to %s", o_json_fn)
    write_pipeline_chunks(chunks, o_json_fn,
                          "created by %s" % Constants.TOOL_ID())
    return 0


def rtc_runner(rtc):
    """Resolved tool contract runner."""
    max_nchunks = rtc.task.max_nchunks if hasattr(
        rtc.task, 'max_nchunks') else Constants.DEFAULT_NCHUNKS
    return run_main(i_datastore_fn=rtc.task.input_files[0],
                    i_trf_fn=rtc.task.input_files[1],
                    o_json_fn=rtc.task.output_files[0],
                    max_nchunks=int(max_nchunks))


if __name__ == '__main__':
    sys.exit(scatter_main(
        args=sys.argv[1:], const=Constants, rtc_runner=rtc_runner, alog=log))

#! python Scatter inputs of pbsvtools.tasks.align_json_to_svsig  align takes two inputs:     Input: idx 0 - datastore.json containing a list of AlignmentSet files,                    or a list of ConsensusAlignmentSet files.            idx 1 - TRF bed file     Output: idx 0 - FOFN of svsig.gz files Constants must be identical to align_json_to_svsig     Input: idx 0 - DataStore json of AlignmentSet (or ConsensusAlignmentSet)            idx 1 - TRF BED     Output: idx 0 - FOFN of svsig.gz Return all external rescources bam files from input datastore json file. Parameters:       i_datastore_fn --- DataStore json of AlignmentSet or ConsensusAlignmentSet to chunk.       i_trf_fn --- Tandem Repeats in BED       o_json_fn -- Output json file # Chunk input datastore json, generate multiple chunked datastore.json, and # generate pbcommand.models.PipelineChunk objects # Put bam files into boxes # Create a chunk: get $chunk.datastore_id from chunk, # use TandemRepeat masker bed as $chunk.bed_id # chunks MUST have unique IDs Resolved tool contract runner.