def fastqtransform(transform, fastq1, fastq2, fastq3, fastq4, keep_fastq_tags,
                   separate_cb, demuxed_cb, cores, fastq1out, fastq2out,
                   min_length):
    ''' Transform input reads to the tagcounts compatible read layout using
    regular expressions as defined in a transform file. Outputs new format to
    stdout.
    '''
    transform = json.load(open(transform))
    options = _infer_transform_options(transform)
    read_template = '{name}'
    logger.info("Transforming %s." % fastq1)
    if options.dual_index:
        logger.info("Detected dual cellular indexes.")
        if separate_cb:
            read_template += ':CELL_{CB1}-{CB2}'
        else:
            read_template += ':CELL_{CB}'
    elif options.triple_index:
        logger.info("Detected triple cellular indexes.")
        if separate_cb:
            read_template += ':CELL_{CB1}-{CB2}-{CB3}'
        else:
            read_template += ':CELL_{CB}'
    elif options.CB or demuxed_cb:
        logger.info("Detected cellular barcodes.")
        read_template += ':CELL_{CB}'
    if options.MB:
        logger.info("Detected UMI.")
        read_template += ':UMI_{MB}'
    if options.SB:
        logger.info("Detected sample.")
        read_template += ':SAMPLE_{SB}'

    read_template += "{readnum}"

    if keep_fastq_tags:
        read_template += ' {fastqtag}'
    read_template += '\n{seq}\n+\n{qual}\n'

    paired = fastq1out and fastq2out

    read1_regex = re.compile(transform['read1'])
    read2_regex = re.compile(transform['read2']) if fastq2 else None
    read3_regex = re.compile(transform['read3']) if fastq3 else None
    read4_regex = re.compile(transform['read4']) if fastq4 else None

    fastq_file1 = read_fastq(fastq1)
    fastq_file2 = read_fastq(fastq2)
    fastq_file3 = read_fastq(fastq3)
    fastq_file4 = read_fastq(fastq4)

    transform = partial(transformer, read1_regex=read1_regex,
                        read2_regex=read2_regex, read3_regex=read3_regex,
                        read4_regex=read4_regex, paired=paired)

    fastq1out_fh = write_fastq(fastq1out)
    fastq2out_fh = write_fastq(fastq2out)

    p = multiprocessing.Pool(cores)

    try :
        zzip = itertools.izip
    except AttributeError:
        zzip = zip

    chunks = tz.partition_all(10000, zzip(fastq_file1, fastq_file2, fastq_file3,
                                          fastq_file4))
    bigchunks = tz.partition_all(cores, chunks)
    for bigchunk in bigchunks:
        for chunk in p.map(transform, list(bigchunk)):
            if paired:
                for read1_dict, read2_dict in tz.partition(2, chunk):
                    if options.dual_index:
                        if not separate_cb:
                            read1_dict['CB'] = read1_dict['CB1'] + read1_dict['CB2']
                            read2_dict['CB'] = read2_dict['CB1'] + read2_dict['CB2']

                    if demuxed_cb:
                        read1_dict['CB'] = demuxed_cb
                        read2_dict['CB'] = demuxed_cb

                    # Deal with spaces in read names
                    if keep_fastq_tags:
                        name, tag = read1_dict['name'].split(' ')
                        read1_dict['name'] = name
                        read1_dict['fastqtag'] = tag
                        name, tag = read2_dict['name'].split(' ')
                        read2_dict['name'] = name
                        read2_dict['fastqtag'] = tag
                    else:
                        read1_dict['name'] = read1_dict['name'].partition(' ')[0]
                        read2_dict['name'] = read2_dict['name'].partition(' ')[0]
                    read1_dict = _extract_readnum(read1_dict)
                    read2_dict = _extract_readnum(read2_dict)

                    tooshort = (len(read1_dict['seq']) < min_length or
                                len(read2_dict['seq']) < min_length)

                    if not tooshort:
                        fastq1out_fh.write(read_template.format(**read1_dict))
                        fastq2out_fh.write(read_template.format(**read2_dict))
            else:
                for read1_dict in chunk:
                    if options.dual_index:
                        if not separate_cb:
                            read1_dict['CB'] = read1_dict['CB1'] + read1_dict['CB2']

                    if demuxed_cb:
                        read1_dict['CB'] = demuxed_cb

                    # Deal with spaces in read names
                    if keep_fastq_tags:
                        name, tag = read1_dict['name'].split(' ')
                        read1_dict['name'] = name
                        read1_dict['fastqtag'] = tag
                    else:
                        read1_dict['name'] = read1_dict['name'].partition(' ')[0]
                    read1_dict = _extract_readnum(read1_dict)
                    if len(read1_dict['seq']) >= min_length:
                        if fastq1out_fh:
                            fastq1out_fh.write(read_template.format(**read1_dict))
                        else:
                            sys.stdout.write(read_template.format(**read1_dict))
def fastqtransform ( transform , fastq1 , fastq2 , fastq3 , fastq4 , keep_fastq_tags , separate_cb , demuxed_cb , cores , fastq1out , fastq2out , min_length ) : transform = json . load ( open ( transform ) ) options = _infer_transform_options ( transform ) read_template = '{name}' logger . info ( "Transforming %s." % fastq1 ) if options . dual_index : logger . info ( "Detected dual cellular indexes." ) if separate_cb : read_template += ':CELL_{CB1}-{CB2}' else : read_template += ':CELL_{CB}' elif options . triple_index : logger . info ( "Detected triple cellular indexes." ) if separate_cb : read_template += ':CELL_{CB1}-{CB2}-{CB3}' else : read_template += ':CELL_{CB}' elif options . CB or demuxed_cb : logger . info ( "Detected cellular barcodes." ) read_template += ':CELL_{CB}' if options . MB : logger . info ( "Detected UMI." ) read_template += ':UMI_{MB}' if options . SB : logger . info ( "Detected sample." ) read_template += ':SAMPLE_{SB}' read_template += "{readnum}" if keep_fastq_tags : read_template += ' {fastqtag}' read_template += '\n{seq}\n+\n{qual}\n' paired = fastq1out and fastq2out read1_regex = re . compile ( transform [ 'read1' ] ) read2_regex = re . compile ( transform [ 'read2' ] ) if fastq2 else None read3_regex = re . compile ( transform [ 'read3' ] ) if fastq3 else None read4_regex = re . compile ( transform [ 'read4' ] ) if fastq4 else None fastq_file1 = read_fastq ( fastq1 ) fastq_file2 = read_fastq ( fastq2 ) fastq_file3 = read_fastq ( fastq3 ) fastq_file4 = read_fastq ( fastq4 ) transform = partial ( transformer , read1_regex = read1_regex , read2_regex = read2_regex , read3_regex = read3_regex , read4_regex = read4_regex , paired = paired ) fastq1out_fh = write_fastq ( fastq1out ) fastq2out_fh = write_fastq ( fastq2out ) p = multiprocessing . Pool ( cores ) try : zzip = itertools . izip except AttributeError : zzip = zip chunks = tz . partition_all ( 10000 , zzip ( fastq_file1 , fastq_file2 , fastq_file3 , fastq_file4 ) ) bigchunks = tz . partition_all ( cores , chunks ) for bigchunk in bigchunks : for chunk in p . map ( transform , list ( bigchunk ) ) : if paired : for read1_dict , read2_dict in tz . partition ( 2 , chunk ) : if options . dual_index : if not separate_cb : read1_dict [ 'CB' ] = read1_dict [ 'CB1' ] + read1_dict [ 'CB2' ] read2_dict [ 'CB' ] = read2_dict [ 'CB1' ] + read2_dict [ 'CB2' ] if demuxed_cb : read1_dict [ 'CB' ] = demuxed_cb read2_dict [ 'CB' ] = demuxed_cb if keep_fastq_tags : name , tag = read1_dict [ 'name' ] . split ( ' ' ) read1_dict [ 'name' ] = name read1_dict [ 'fastqtag' ] = tag name , tag = read2_dict [ 'name' ] . split ( ' ' ) read2_dict [ 'name' ] = name read2_dict [ 'fastqtag' ] = tag else : read1_dict [ 'name' ] = read1_dict [ 'name' ] . partition ( ' ' ) [ 0 ] read2_dict [ 'name' ] = read2_dict [ 'name' ] . partition ( ' ' ) [ 0 ] read1_dict = _extract_readnum ( read1_dict ) read2_dict = _extract_readnum ( read2_dict ) tooshort = ( len ( read1_dict [ 'seq' ] ) < min_length or len ( read2_dict [ 'seq' ] ) < min_length ) if not tooshort : fastq1out_fh . write ( read_template . format ( ** read1_dict ) ) fastq2out_fh . write ( read_template . format ( ** read2_dict ) ) else : for read1_dict in chunk : if options . dual_index : if not separate_cb : read1_dict [ 'CB' ] = read1_dict [ 'CB1' ] + read1_dict [ 'CB2' ] if demuxed_cb : read1_dict [ 'CB' ] = demuxed_cb if keep_fastq_tags : name , tag = read1_dict [ 'name' ] . split ( ' ' ) read1_dict [ 'name' ] = name read1_dict [ 'fastqtag' ] = tag else : read1_dict [ 'name' ] = read1_dict [ 'name' ] . partition ( ' ' ) [ 0 ] read1_dict = _extract_readnum ( read1_dict ) if len ( read1_dict [ 'seq' ] ) >= min_length : if fastq1out_fh : fastq1out_fh . write ( read_template . format ( ** read1_dict ) ) else : sys . stdout . write ( read_template . format ( ** read1_dict ) )
Transform input reads to the tagcounts compatible read layout using
    regular expressions as defined in a transform file. Outputs new format to
    stdout.