def fetch_cluster_se(data, samfile, chrom, rstart, rend):
    """
    Builds a single end cluster from the refmapped data.
    """

    ## If SE then we enforce the minimum overlap distance to avoid the
    ## staircase syndrome of multiple reads overlapping just a little.
    overlap_buffer = data._hackersonly["min_SE_refmap_overlap"]

    ## the *_buff variables here are because we have to play patty
    ## cake here with the rstart/rend vals because we want pysam to
    ## enforce the buffer for SE, but we want the reference sequence
    ## start and end positions to print correctly for downstream.
    rstart_buff = rstart + overlap_buffer
    rend_buff = rend - overlap_buffer

    ## Reads that map to only very short segements of the reference
    ## sequence will return buffer end values that are before the
    ## start values causing pysam to complain. Very short mappings.
    if rstart_buff > rend_buff:
        tmp = rstart_buff
        rstart_buff = rend_buff
        rend_buff = tmp
    ## Buffering can't make start and end equal or pysam returns nothing.
    if rstart_buff == rend_buff:
        rend_buff += 1

    ## store pairs
    rdict = {}
    clust = []
    iterreg = []

    iterreg = samfile.fetch(chrom, rstart_buff, rend_buff)

    ## use dict to match up read pairs
    for read in iterreg:
        if read.qname not in rdict:
            rdict[read.qname] = read

    ## sort dict keys so highest derep is first ('seed')
    sfunc = lambda x: int(x.split(";size=")[1].split(";")[0])
    rkeys = sorted(rdict.keys(), key=sfunc, reverse=True)

    ## get blocks from the seed for filtering, bail out if seed is not paired
    try:
        read1 = rdict[rkeys[0]]
    except ValueError:
        LOGGER.error("Found bad cluster, skipping - key:{} rdict:{}".format(rkeys[0], rdict))
        return ""

    ## the starting blocks for the seed
    poss = read1.get_reference_positions(full_length=True)
    seed_r1start = min(poss)
    seed_r1end = max(poss)

    ## store the seed -------------------------------------------
    if read1.is_reverse:
        seq = revcomp(read1.seq)
    else:
        seq = read1.seq

    ## store, could write orient but just + for now.
    size = sfunc(rkeys[0])
    clust.append(">{}:{}:{};size={};*\n{}"\
        .format(chrom, seed_r1start, seed_r1end, size, seq))

    ## If there's only one hit in this region then rkeys will only have
    ## one element and the call to `rkeys[1:]` will raise. Test for this.
    if len(rkeys) > 1:
        ## store the hits to the seed -------------------------------
        for key in rkeys[1:]:
            skip = False
            try:
                read1 = rdict[key]
            except ValueError:
                ## enter values that will make this read get skipped
                read1 = rdict[key][0]
                skip = True

            ## orient reads only if not skipping
            if not skip:
                poss = read1.get_reference_positions(full_length=True)
                minpos = min(poss)
                maxpos = max(poss)
                ## store the seq
                if read1.is_reverse:
                    seq = revcomp(read1.seq)
                else:
                    seq = read1.seq
                ## store, could write orient but just + for now.
                size = sfunc(key)
                clust.append(">{}:{}:{};size={};+\n{}"\
                    .format(chrom, minpos, maxpos, size, seq))
            else:
                ## seq is excluded, though, we could save it and return
                ## it as a separate cluster that will be aligned separately.
                pass

    return clust

Builds a single end cluster from the refmapped data .


def ref_build_and_muscle_chunk(data, sample):
    """
    1. Run bedtools to get all overlapping regions
    2. Parse out reads from regions using pysam and dump into chunk files.
       We measure it out to create 10 chunk files per sample.
    3. If we really wanted to speed this up, though it is pretty fast already,
       we could parallelize it since we can easily break the regions into
       a list of chunks.
    """

    ## get regions using bedtools
    regions = bedtools_merge(data, sample).strip().split("\n")
    nregions = len(regions)
    chunksize = (nregions / 10) + (nregions % 10)

    LOGGER.debug("nregions {} chunksize {}".format(nregions, chunksize))
    ## create an output file to write clusters to
    idx = 0
    tmpfile = os.path.join(data.tmpdir, sample.name+"_chunk_{}.ali")
    ## remove old files if they exist to avoid append errors
    for i in range(11):
        if os.path.exists(tmpfile.format(i)):
            os.remove(tmpfile.format(i))
    fopen = open

    ## If reference+denovo we drop the reads back into clust.gz
    ## and let the muscle_chunker do it's thing back in cluster_within
    if data.paramsdict["assembly_method"] == "denovo+reference":
        tmpfile = os.path.join(data.dirs.clusts, sample.name+".clust.gz")
        fopen = gzip.open

    ## build clusters for aligning with muscle from the sorted bam file
    samfile = pysam.AlignmentFile(sample.files.mapped_reads, 'rb')
    #"./tortas_refmapping/PZ70-mapped-sorted.bam", "rb")

    ## fill clusts list and dump periodically
    clusts = []
    nclusts = 0
    for region in regions:
        chrom, pos1, pos2 = region.split()
        try:
            ## fetches pairs quickly but then goes slow to merge them.
            if "pair" in data.paramsdict["datatype"]:
                clust = fetch_cluster_pairs(data, samfile, chrom, int(pos1), int(pos2))

            ## fetch but no need to merge
            else:
                clust = fetch_cluster_se(data, samfile, chrom, int(pos1), int(pos2))
        except IndexError as inst:
            LOGGER.error("Bad region chrom:start-end {}:{}-{}".format(chrom, pos1, pos2))
            continue
        if clust:
            clusts.append("\n".join(clust))
            nclusts += 1

            if nclusts == chunksize:
                ## write to file
                tmphandle = tmpfile.format(idx)
                with fopen(tmphandle, 'a') as tmp:
                    #LOGGER.debug("Writing tmpfile - {}".format(tmpfile.format(idx)))
                    #if data.paramsdict["assembly_method"] == "denovo+reference":
                    #    ## This is dumb, but for this method you need to prepend the
                    #    ## separator to maintain proper formatting of clust.gz
                    tmp.write("\n//\n//\n".join(clusts)+"\n//\n//\n")
                idx += 1
                nclusts = 0
                clusts = []
    if clusts:
        ## write remaining to file
        with fopen(tmpfile.format(idx), 'a') as tmp:
            #tmp.write("\n//\n//\n" + ("\n//\n//\n".join(clusts)))
            tmp.write("\n//\n//\n".join(clusts)+"\n//\n//\n")
        clusts = []

    if not data.paramsdict["assembly_method"] == "denovo+reference":
        chunkfiles = glob.glob(os.path.join(data.tmpdir, sample.name+"_chunk_*.ali"))
        LOGGER.info("created chunks %s", chunkfiles)

    ## cleanup
    samfile.close()

1 . Run bedtools to get all overlapping regions 2 . Parse out reads from regions using pysam and dump into chunk files . We measure it out to create 10 chunk files per sample . 3 . If we really wanted to speed this up though it is pretty fast already we could parallelize it since we can easily break the regions into a list of chunks .


def ref_muscle_chunker(data, sample):
    """
    Run bedtools to get all overlapping regions. Pass this list into the func
    'get_overlapping_reads' which will write fastq chunks to the clust.gz file.

    1) Run bedtools merge to get a list of all contiguous blocks of bases
    in the reference seqeunce where one or more of our reads overlap.
    The output will look like this:
            1       45230754        45230783
            1       74956568        74956596
            ...
            1       116202035       116202060
    """
    LOGGER.info('entering ref_muscle_chunker')

    ## Get regions, which will be a giant list of 5-tuples, of which we're
    ## only really interested in the first three: (chrom, start, end) position.
    regions = bedtools_merge(data, sample)

    if len(regions) > 0:
        ## this calls bam_region_to_fasta a billion times
        get_overlapping_reads(data, sample, regions)
    else:
        msg = "No reads mapped to reference sequence - {}".format(sample.name)
        LOGGER.warn(msg)

Run bedtools to get all overlapping regions . Pass this list into the func get_overlapping_reads which will write fastq chunks to the clust . gz file .


def check_insert_size(data, sample):
    """
    check mean insert size for this sample and update
    hackersonly.max_inner_mate_distance if need be. This value controls how
    far apart mate pairs can be to still be considered for bedtools merging
    downstream.
    """

    ## pipe stats output to grep
    cmd1 = [ipyrad.bins.samtools, "stats", sample.files.mapped_reads]
    cmd2 = ["grep", "SN"]
    proc1 = sps.Popen(cmd1, stderr=sps.STDOUT, stdout=sps.PIPE)
    proc2 = sps.Popen(cmd2, stderr=sps.STDOUT, stdout=sps.PIPE, stdin=proc1.stdout)

    ## get piped result
    res = proc2.communicate()[0]

    ## raise exception on failure and do cleanup
    if proc2.returncode:
        raise IPyradWarningExit("error in %s: %s", cmd2, res)

    ## starting vals
    avg_insert = 0
    stdv_insert = 0
    avg_len = 0

    ## iterate over results
    for line in res.split("\n"):
        if "insert size average" in line:
            avg_insert = float(line.split(":")[-1].strip())

        elif "insert size standard deviation" in line:
            ## hack to fix sim data when stdv is 0.0. Shouldn't
            ## impact real data bcz stdv gets rounded up below
            stdv_insert = float(line.split(":")[-1].strip()) + 0.1

        elif "average length" in line:
            avg_len = float(line.split(":")[-1].strip())

    LOGGER.debug("avg {} stdv {} avg_len {}"\
                 .format(avg_insert, stdv_insert, avg_len))

    ## If all values return successfully set the max inner mate distance.
    ## This is tricky. avg_insert is the average length of R1+R2+inner mate
    ## distance. avg_len is the average length of a read. If there are lots
    ## of reads that overlap then avg_insert will be close to but bigger than
    ## avg_len. We are looking for the right value for `bedtools merge -d`
    ## which wants to know the max distance between reads.
    if all([avg_insert, stdv_insert, avg_len]):
        ## If 2 * the average length of a read is less than the average
        ## insert size then most reads DO NOT overlap
        if stdv_insert < 5:
            stdv_insert = 5.
        if (2 * avg_len) < avg_insert:
            hack = avg_insert + (3 * np.math.ceil(stdv_insert)) - (2 * avg_len)

        ## If it is > than the average insert size then most reads DO
        ## overlap, so we have to calculate inner mate distance a little
        ## differently.
        else:
            hack = (avg_insert - avg_len) + (3 * np.math.ceil(stdv_insert))

        ## set the hackerdict value
        LOGGER.info("stdv: hacked insert size is %s", hack)
        data._hackersonly["max_inner_mate_distance"] = int(np.math.ceil(hack))

    else:
        ## If something fsck then set a relatively conservative distance
        data._hackersonly["max_inner_mate_distance"] = 300
        LOGGER.debug("inner mate distance for {} - {}".format(sample.name,\
                    data._hackersonly["max_inner_mate_distance"]))

check mean insert size for this sample and update hackersonly . max_inner_mate_distance if need be . This value controls how far apart mate pairs can be to still be considered for bedtools merging downstream .


def bedtools_merge(data, sample):
    """
    Get all contiguous genomic regions with one or more overlapping
    reads. This is the shell command we'll eventually run

    bedtools bamtobed -i 1A_0.sorted.bam | bedtools merge [-d 100]
        -i <input_bam>  :   specifies the input file to bed'ize
        -d <int>        :   For PE set max distance between reads
    """
    LOGGER.info("Entering bedtools_merge: %s", sample.name)
    mappedreads = os.path.join(data.dirs.refmapping,
                               sample.name+"-mapped-sorted.bam")

    ## command to call `bedtools bamtobed`, and pipe output to stdout
    ## Usage:   bedtools bamtobed [OPTIONS] -i <bam>
    ## Usage:   bedtools merge [OPTIONS] -i <bam>
    cmd1 = [ipyrad.bins.bedtools, "bamtobed", "-i", mappedreads]
    cmd2 = [ipyrad.bins.bedtools, "merge", "-i", "-"]

    ## If PE the -d flag to tell bedtools how far apart to allow mate pairs.
    ## If SE the -d flag is negative, specifying that SE reads need to
    ## overlap by at least a specific number of bp. This prevents the
    ## stairstep syndrome when a + and - read are both extending from
    ## the same cutsite. Passing a negative number to `merge -d` gets this done.
    if 'pair' in data.paramsdict["datatype"]:
        check_insert_size(data, sample)
        #cmd2.insert(2, str(data._hackersonly["max_inner_mate_distance"]))
        cmd2.insert(2, str(data._hackersonly["max_inner_mate_distance"]))
        cmd2.insert(2, "-d")
    else:
        cmd2.insert(2, str(-1 * data._hackersonly["min_SE_refmap_overlap"]))
        cmd2.insert(2, "-d")

    ## pipe output from bamtobed into merge
    LOGGER.info("stdv: bedtools merge cmds: %s %s", cmd1, cmd2)
    proc1 = sps.Popen(cmd1, stderr=sps.STDOUT, stdout=sps.PIPE)
    proc2 = sps.Popen(cmd2, stderr=sps.STDOUT, stdout=sps.PIPE, stdin=proc1.stdout)
    result = proc2.communicate()[0]
    proc1.stdout.close()

    ## check for errors and do cleanup
    if proc2.returncode:
        raise IPyradWarningExit("error in %s: %s", cmd2, result)

    ## Write the bedfile out, because it's useful sometimes.
    if os.path.exists(ipyrad.__debugflag__):
        with open(os.path.join(data.dirs.refmapping, sample.name + ".bed"), 'w') as outfile:
            outfile.write(result)

    ## Report the number of regions we're returning
    nregions = len(result.strip().split("\n"))
    LOGGER.info("bedtools_merge: Got # regions: %s", nregions)
    return result

Get all contiguous genomic regions with one or more overlapping reads . This is the shell command we ll eventually run


def refmap_stats(data, sample):
    """
    Get the number of mapped and unmapped reads for a sample
    and update sample.stats
    """
    ## shorter names
    mapf = os.path.join(data.dirs.refmapping, sample.name+"-mapped-sorted.bam")
    umapf = os.path.join(data.dirs.refmapping, sample.name+"-unmapped.bam")

    ## get from unmapped
    cmd1 = [ipyrad.bins.samtools, "flagstat", umapf]
    proc1 = sps.Popen(cmd1, stderr=sps.STDOUT, stdout=sps.PIPE)
    result1 = proc1.communicate()[0]

    ## get from mapped
    cmd2 = [ipyrad.bins.samtools, "flagstat", mapf]
    proc2 = sps.Popen(cmd2, stderr=sps.STDOUT, stdout=sps.PIPE)
    result2 = proc2.communicate()[0]

    ## store results
    ## If PE, samtools reports the _actual_ number of reads mapped, both
    ## R1 and R2, so here if PE divide the results by 2 to stay consistent
    ## with how we've been reporting R1 and R2 as one "read pair"
    if "pair" in data.paramsdict["datatype"]:
        sample.stats["refseq_unmapped_reads"] = int(result1.split()[0]) / 2
        sample.stats["refseq_mapped_reads"] = int(result2.split()[0]) / 2
    else:
        sample.stats["refseq_unmapped_reads"] = int(result1.split()[0])
        sample.stats["refseq_mapped_reads"] = int(result2.split()[0])

    sample_cleanup(data, sample)

Get the number of mapped and unmapped reads for a sample and update sample . stats


def refmap_init(data, sample, force):
    """ create some file handles for refmapping """
    ## make some persistent file handles for the refmap reads files
    sample.files.unmapped_reads = os.path.join(data.dirs.edits,
                                  "{}-refmap_derep.fastq".format(sample.name))
    sample.files.mapped_reads = os.path.join(data.dirs.refmapping,
                                  "{}-mapped-sorted.bam".format(sample.name))


def _subsample(self):
        """ returns a subsample of unlinked snp sites """
        spans = self.maparr
        samp = np.zeros(spans.shape[0], dtype=np.uint64)
        for i in xrange(spans.shape[0]):
            samp[i] = np.random.randint(spans[i, 0], spans[i, 1], 1)
        return samp

def _subsample ( self ) : spans = self . maparr samp = np . zeros ( spans . shape [ 0 ] , dtype = np . uint64 ) for i in xrange ( spans . shape [ 0 ] ) : samp [ i ] = np . random . randint ( spans [ i , 0 ] , spans [ i , 1 ] , 1 ) return samp


def draw(self, axes):
        """
        Returns a treemix plot on a toyplot.axes object.
        """
        ## create a toytree object from the treemix tree result
        tre = toytree.tree(newick=self.results.tree)
        tre.draw(
            axes=axes,
            use_edge_lengths=True,
            tree_style='c',
            tip_labels_align=True,
            edge_align_style={"stroke-width": 1}
        );

        ## get coords
        for admix in self.results.admixture:
            ## parse admix event
            pidx, pdist, cidx, cdist, weight = admix
            a = _get_admix_point(tre, pidx, pdist)
            b = _get_admix_point(tre, cidx, cdist)

            ## add line for admixture edge
            mark = axes.plot(
                a = (a[0], b[0]),
                b = (a[1], b[1]),
                style={"stroke-width": 10*weight,
                       "stroke-opacity": 0.95,
                       "stroke-linecap": "round"}
            )

            ## add points at admixture sink
            axes.scatterplot(
                a = (b[0]),
                b = (b[1]),
                size=8,
                title="weight: {}".format(weight),
            )

        ## add scale bar for edge lengths
        axes.y.show=False
        axes.x.ticks.show=True
        axes.x.label.text = "Drift parameter"
        return axes

Returns a treemix plot on a toyplot . axes object .


def _resolveambig(subseq):
    """
    Randomly resolves iupac hetero codes. This is a shortcut
    for now, we could instead use the phased alleles in RAD loci.
    """
    N = []
    for col in subseq:
        rand = np.random.binomial(1, 0.5)
        N.append([_AMBIGS[i][rand] for i in col])
    return np.array(N)

def _resolveambig ( subseq ) : N = [ ] for col in subseq : rand = np . random . binomial ( 1 , 0.5 ) N . append ( [ _AMBIGS [ i ] [ rand ] for i in col ] ) return np . array ( N )

Randomly resolves iupac hetero codes . This is a shortcut for now we could instead use the phased alleles in RAD loci .


def _count_PIS(seqsamp, N):
    """ filters for loci with >= N PIS """
    counts = [Counter(col) for col in seqsamp.T if not ("-" in col or "N" in col)]
    pis = [i.most_common(2)[1][1] > 1 for i in counts if len(i.most_common(2))>1]
    if sum(pis) >= N:
        return sum(pis)
    else:
        return 0

def _count_PIS ( seqsamp , N ) : counts = [ Counter ( col ) for col in seqsamp . T if not ( "-" in col or "N" in col ) ] pis = [ i . most_common ( 2 ) [ 1 ] [ 1 ] > 1 for i in counts if len ( i . most_common ( 2 ) ) > 1 ] if sum ( pis ) >= N : return sum ( pis ) else : return 0


def _write_nex(self, mdict, nlocus):
        """
        function that takes a dictionary mapping names to sequences,
        and a locus number, and writes it as a NEXUS file with a mrbayes
        analysis block given a set of mcmc arguments.
        """

        ## create matrix as a string
        max_name_len = max([len(i) for i in mdict])
        namestring = "{:<" + str(max_name_len+1) + "} {}\n"
        matrix = ""
        for i in mdict.items():
            matrix += namestring.format(i[0], i[1])

        ## ensure dir
        minidir = os.path.realpath(os.path.join(self.workdir, self.name))
        if not os.path.exists(minidir):
            os.makedirs(minidir)

        ## write nexus block
        handle = os.path.join(minidir, "{}.nex".format(nlocus))
        with open(handle, 'w') as outnex:
            outnex.write(NEXBLOCK.format(**{
                "ntax": len(mdict),
                "nchar": len(mdict.values()[0]),
                "matrix": matrix,
                "ngen": self.params.mb_mcmc_ngen,
                "sfreq": self.params.mb_mcmc_sample_freq,
                "burnin": self.params.mb_mcmc_burnin,
                }))

def
_write_nex
(
self
,
mdict
,
nlocus
)
:
## create matrix as a string
max_name_len
=
max
(
[
len
(
i
)
for
i
in
mdict
]
)
namestring
=
"{:<"
+
str
(
max_name_len
+
1
)
+
"} {}\n"
matrix
=
""
for
i
in
mdict
.
items
(
)
:
matrix
+=
namestring
.
format
(
i
[
0
]
,
i
[
1
]
)
## ensure dir
minidir
=
os
.
path
.
realpath
(
os
.
path
.
join
(
self
.
workdir
,
self
.
name
)
)
if
not
os
.
path
.
exists
(
minidir
)
:
os
.
makedirs
(
minidir
)
## write nexus block
handle
=
os
.
path
.
join
(
minidir
,
"{}.nex"
.
format
(
nlocus
)
)
with
open
(
handle
,
'w'
)
as
outnex
:
outnex
.
write
(
NEXBLOCK
.
format
(
*
*
{
"ntax"
:
len
(
mdict
)
,
"nchar"
:
len
(
mdict
.
values
(
)
[
0
]
)
,
"matrix"
:
matrix
,
"ngen"
:
self
.
params
.
mb_mcmc_ngen
,
"sfreq"
:
self
.
params
.
mb_mcmc_sample_freq
,
"burnin"
:
self
.
params
.
mb_mcmc_burnin
,
}
)
)

function that takes a dictionary mapping names to sequences and a locus number and writes it as a NEXUS file with a mrbayes analysis block given a set of mcmc arguments .


def _read_sample_names(fname):
    """ Read in sample names from a plain text file. This is a convenience
    function for branching so if you have tons of sample names you can
    pass in a file rather than having to set all the names at the command
    line.
    """
    try:
        with open(fname, 'r') as infile:
            subsamples = [x.split()[0] for x in infile.readlines() if x.strip()]

    except Exception as inst:
        print("Failed to read input file with sample names.\n{}".format(inst))
        raise inst

    return subsamples

Read in sample names from a plain text file . This is a convenience function for branching so if you have tons of sample names you can pass in a file rather than having to set all the names at the command line .


def _bufcountlines(filename, gzipped):
    """
    fast line counter. Used to quickly sum number of input reads when running
    link_fastqs to append files. """
    if gzipped:
        fin = gzip.open(filename)
    else:
        fin = open(filename)
    nlines = 0
    buf_size = 1024 * 1024
    read_f = fin.read # loop optimization
    buf = read_f(buf_size)
    while buf:
        nlines += buf.count('\n')
        buf = read_f(buf_size)
    fin.close()
    return nlines

def
_bufcountlines
(
filename
,
gzipped
)
:
if
gzipped
:
fin
=
gzip
.
open
(
filename
)
else
:
fin
=
open
(
filename
)
nlines
=
0
buf_size
=
1024
*
1024
read_f
=
fin
.
read
# loop optimization
buf
=
read_f
(
buf_size
)
while
buf
:
nlines
+=
buf
.
count
(
'\n'
)
buf
=
read_f
(
buf_size
)
fin
.
close
(
)
return
nlines

fast line counter . Used to quickly sum number of input reads when running link_fastqs to append files .


def _zbufcountlines(filename, gzipped):
    """ faster line counter """
    if gzipped:
        cmd1 = ["gunzip", "-c", filename]
    else:
        cmd1 = ["cat", filename]
    cmd2 = ["wc"]

    proc1 = sps.Popen(cmd1, stdout=sps.PIPE, stderr=sps.PIPE)
    proc2 = sps.Popen(cmd2, stdin=proc1.stdout, stdout=sps.PIPE, stderr=sps.PIPE)
    res = proc2.communicate()[0]
    if proc2.returncode:
        raise IPyradWarningExit("error zbufcountlines {}:".format(res))
    LOGGER.info(res)
    nlines = int(res.split()[0])
    return nlines

def
_zbufcountlines
(
filename
,
gzipped
)
:
if
gzipped
:
cmd1
=
[
"gunzip"
,
"-c"
,
filename
]
else
:
cmd1
=
[
"cat"
,
filename
]
cmd2
=
[
"wc"
]
proc1
=
sps
.
Popen
(
cmd1
,
stdout
=
sps
.
PIPE
,
stderr
=
sps
.
PIPE
)
proc2
=
sps
.
Popen
(
cmd2
,
stdin
=
proc1
.
stdout
,
stdout
=
sps
.
PIPE
,
stderr
=
sps
.
PIPE
)
res
=
proc2
.
communicate
(
)
[
0
]
if
proc2
.
returncode
:
raise
IPyradWarningExit
(
"error zbufcountlines {}:"
.
format
(
res
)
)
LOGGER
.
info
(
res
)
nlines
=
int
(
res
.
split
(
)
[
0
]
)
return
nlines


def _tuplecheck(newvalue, dtype=str):
    """
    Takes a string argument and returns value as a tuple.
    Needed for paramfile conversion from CLI to set_params args
    """

    if isinstance(newvalue, list):
        newvalue = tuple(newvalue)

    if isinstance(newvalue, str):
        newvalue = newvalue.rstrip(")").strip("(")
        try:
            newvalue = tuple([dtype(i.strip()) for i in newvalue.split(",")])

        ## Type error is thrown by tuple if it's applied to a non-iterable.
        except TypeError:
            newvalue = tuple(dtype(newvalue))

        ## If dtype fails to cast any element of newvalue
        except ValueError:
            LOGGER.info("Assembly.tuplecheck() failed to cast to {} - {}"\
                        .format(dtype, newvalue))
            raise

        except Exception as inst:
            LOGGER.info(inst)
            raise SystemExit(\
            "\nError: Param`{}` is not formatted correctly.\n({})\n"\
                 .format(newvalue, inst))

    return newvalue

Takes a string argument and returns value as a tuple . Needed for paramfile conversion from CLI to set_params args


def stats(self):
        """ Returns a data frame with Sample data and state. """
        nameordered = self.samples.keys()
        nameordered.sort()

        ## Set pandas to display all samples instead of truncating
        pd.options.display.max_rows = len(self.samples)
        statdat = pd.DataFrame([self.samples[i].stats for i in nameordered],
                      index=nameordered).dropna(axis=1, how='all')
        # ensure non h,e columns print as ints
        for column in statdat:
            if column not in ["hetero_est", "error_est"]:
                statdat[column] = np.nan_to_num(statdat[column]).astype(int)
        return statdat

Returns a data frame with Sample data and state .


def files(self):
        """ Returns a data frame with Sample files. Not very readable... """
        nameordered = self.samples.keys()
        nameordered.sort()
        ## replace curdir with . for shorter printing
        #fullcurdir = os.path.realpath(os.path.curdir)
        return pd.DataFrame([self.samples[i].files for i in nameordered],
                      index=nameordered).dropna(axis=1, how='all')

Returns a data frame with Sample files . Not very readable ...


def _build_stat(self, idx):
        """ Returns a data frame with Sample stats for each step """
        nameordered = self.samples.keys()
        nameordered.sort()
        newdat = pd.DataFrame([self.samples[i].stats_dfs[idx] \
                               for i in nameordered], index=nameordered)\
                               .dropna(axis=1, how='all')
        return newdat

def _build_stat ( self , idx ) : nameordered = self . samples . keys ( ) nameordered . sort ( ) newdat = pd . DataFrame ( [ self . samples [ i ] . stats_dfs [ idx ] for i in nameordered ] , index = nameordered ) . dropna ( axis = 1 , how = 'all' ) return newdat


def get_params(self, param=""):
        """ pretty prints params if called as a function """
        fullcurdir = os.path.realpath(os.path.curdir)
        if not param:
            for index, (key, value) in enumerate(self.paramsdict.items()):
                if isinstance(value, str):
                    value = value.replace(fullcurdir+"/", "./")
                sys.stdout.write("{}{:<4}{:<28}{:<45}\n"\
                    .format(self._spacer, index, key, value))
        else:
            try:
                if int(param):
                    #sys.stdout.write(self.paramsdict.values()[int(param)-1])
                    return self.paramsdict.values()[int(param)]
            except (ValueError, TypeError, NameError, IndexError):
                try:
                    return self.paramsdict[param]
                except KeyError:
                    return 'key not recognized'


def set_params(self, param, newvalue):
        """
        Set a parameter to a new value. Raises error if newvalue is wrong type.

        Note
        ----
        Use [Assembly].get_params() to see the parameter values currently
        linked to the Assembly object.

        Parameters
        ----------
        param : int or str
            The index (e.g., 1) or string name (e.g., "project_dir")
            for the parameter that will be changed.

        newvalue : int, str, or tuple
            The new value for the parameter selected for `param`. Use
            `ipyrad.get_params_info()` to get further information about
            a given parameter. If the wrong type is entered for newvalue
            (e.g., a str when it should be an int), an error will be raised.
            Further information about each parameter is also available
            in the documentation.

        Examples
        --------
        ## param 'project_dir' takes only a str as input
        [Assembly].set_params('project_dir', 'new_directory')

        ## param 'restriction_overhang' must be a tuple or str, if str it is
        ## converted to a tuple with the second entry empty.
        [Assembly].set_params('restriction_overhang', ('CTGCAG', 'CCGG')

        ## param 'max_shared_Hs_locus' can be an int or a float:
        [Assembly].set_params('max_shared_Hs_locus', 0.25)

        """

        ## this includes current params and some legacy params for conversion
        legacy_params = ["edit_cutsites", "trim_overhang"]
        current_params = self.paramsdict.keys()
        allowed_params = current_params + legacy_params

        ## require parameter recognition
        #if not ((param in range(50)) or \
        #        (param in [str(i) for i in range(50)]) or \
        #        (param in allowed_params)):
        if not param in allowed_params:
            raise IPyradParamsError("Parameter key not recognized: {}"\
                                    .format(param))

        ## make string
        param = str(param)
        ## get index if param is keyword arg (this index is now zero based!)
        if len(param) < 3:
            param = self.paramsdict.keys()[int(param)]

        ## run assertions on new param
        try:
            self = _paramschecker(self, param, newvalue)

        except Exception as inst:
            raise IPyradWarningExit(BAD_PARAMETER\
                                    .format(param, inst, newvalue))

Set a parameter to a new value . Raises error if newvalue is wrong type .


def branch(self, newname, subsamples=None, infile=None):
        """
        Returns a copy of the Assembly object. Does not allow Assembly
        object names to be replicated in namespace or path.
        """
        ## subsample by removal or keeping.
        remove = 0

        ## is there a better way to ask if it already exists?
        if (newname == self.name or os.path.exists(
                                    os.path.join(self.paramsdict["project_dir"],
                                    newname+".assembly"))):
            print("{}Assembly object named {} already exists"\
                  .format(self._spacer, newname))

        else:
            ## Make sure the new name doesn't have any wacky characters
            self._check_name(newname)

            ## Bozo-check. Carve off 'params-' if it's in the new name.
            if newname.startswith("params-"):
                newname = newname.split("params-")[1]

            ## create a copy of the Assembly obj
            newobj = copy.deepcopy(self)
            newobj.name = newname
            newobj.paramsdict["assembly_name"] = newname

            if subsamples and infile:
                print(BRANCH_NAMES_AND_INPUT)

            if infile:
                if infile[0] == "-":
                    remove = 1
                    infile = infile[1:]
                if os.path.exists(infile):
                    subsamples = _read_sample_names(infile)

            ## if remove then swap the samples
            if remove:
                subsamples = list(set(self.samples.keys()) - set(subsamples))

            ## create copies of each subsampled Sample obj
            if subsamples:
                for sname in subsamples:
                    if sname in self.samples:
                        newobj.samples[sname] = copy.deepcopy(self.samples[sname])
                    else:
                        print("Sample name not found: {}".format(sname))

                ## reload sample dict w/o non subsamples
                newobj.samples = {name:sample for name, sample in \
                           newobj.samples.items() if name in subsamples}

            ## create copies of each subsampled Sample obj
            else:
                for sample in self.samples:
                    newobj.samples[sample] = copy.deepcopy(self.samples[sample])

            ## save json of new obj and return object
            newobj.save()
            return newobj

Returns a copy of the Assembly object . Does not allow Assembly object names to be replicated in namespace or path .


def _step1func(self, force, ipyclient):
        """ hidden wrapped function to start step 1 """

        ## check input data files
        sfiles = self.paramsdict["sorted_fastq_path"]
        rfiles = self.paramsdict["raw_fastq_path"]

        ## do not allow both a sorted_fastq_path and a raw_fastq
        if sfiles and rfiles:
            raise IPyradWarningExit(NOT_TWO_PATHS)

        ## but also require that at least one exists
        if not (sfiles or rfiles):
            raise IPyradWarningExit(NO_SEQ_PATH_FOUND)

        ## print headers
        if self._headers:
            if sfiles:
                print("\n{}Step 1: Loading sorted fastq data to Samples"\
                      .format(self._spacer))
            else:
                print("\n{}Step 1: Demultiplexing fastq data to Samples"\
                    .format(self._spacer))

        ## if Samples already exist then no demultiplexing
        if self.samples:
            if not force:
                print(SAMPLES_EXIST.format(len(self.samples), self.name))
            else:
                ## overwrite existing data else do demux
                if glob.glob(sfiles):
                    self._link_fastqs(ipyclient=ipyclient, force=force)
                else:
                    assemble.demultiplex.run2(self, ipyclient, force)

        ## Creating new Samples
        else:
            ## first check if demultiplexed files exist in sorted path
            if glob.glob(sfiles):
                self._link_fastqs(ipyclient=ipyclient)

            ## otherwise do the demultiplexing
            else:
                assemble.demultiplex.run2(self, ipyclient, force)


def _step2func(self, samples, force, ipyclient):
        """ hidden wrapped function to start step 2"""

        ## print header
        if self._headers:
            print("\n  Step 2: Filtering reads ")

        ## If no samples in this assembly then it means you skipped step1,
        if not self.samples.keys():
            raise IPyradWarningExit(FIRST_RUN_1)

        ## Get sample objects from list of strings, if API.
        samples = _get_samples(self, samples)

        if not force:
            ## print warning and skip if all are finished
            if all([i.stats.state >= 2 for i in samples]):
                print(EDITS_EXIST.format(len(samples)))
                return

        ## Run samples through rawedit
        assemble.rawedit.run2(self, samples, force, ipyclient)


def _step4func(self, samples, force, ipyclient):
        """ hidden wrapped function to start step 4 """

        if self._headers:
            print("\n  Step 4: Joint estimation of error rate and heterozygosity")

        ## Get sample objects from list of strings
        samples = _get_samples(self, samples)

        ## Check if all/none in the right state
        if not self._samples_precheck(samples, 4, force):
            raise IPyradError(FIRST_RUN_3)

        elif not force:
            ## skip if all are finished
            if all([i.stats.state >= 4 for i in samples]):
                print(JOINTS_EXIST.format(len(samples)))
                return

        ## send to function
        assemble.jointestimate.run(self, samples, force, ipyclient)


def _step5func(self, samples, force, ipyclient):
        """ hidden wrapped function to start step 5 """
        ## print header
        if self._headers:
            print("\n  Step 5: Consensus base calling ")

        ## Get sample objects from list of strings
        samples = _get_samples(self, samples)

        ## Check if all/none in the right state
        if not self._samples_precheck(samples, 5, force):
            raise IPyradError(FIRST_RUN_4)

        elif not force:
            ## skip if all are finished
            if all([i.stats.state >= 5 for i in samples]):
                print(CONSENS_EXIST.format(len(samples)))
                return
        ## pass samples to rawedit
        assemble.consens_se.run(self, samples, force, ipyclient)


def _step6func(self,
        samples,
        noreverse,
        force,
        randomseed,
        ipyclient,
        **kwargs):
        """
        Hidden function to start Step 6.
        """

        ## Get sample objects from list of strings
        samples = _get_samples(self, samples)

        ## remove samples that aren't ready
        csamples = self._samples_precheck(samples, 6, force)

        ## print CLI header
        if self._headers:
            print("\n  Step 6: Clustering at {} similarity across {} samples".\
                  format(self.paramsdict["clust_threshold"], len(csamples)))

        ## Check if all/none in the right state
        if not csamples:
            raise IPyradError(FIRST_RUN_5)

        elif not force:
            ## skip if all are finished
            if all([i.stats.state >= 6 for i in csamples]):
                print(DATABASE_EXISTS.format(len(samples)))
                return

        ## run if this point is reached. We no longer check for existing
        ## h5 file, since checking Sample states should suffice.
        assemble.cluster_across.run(
            self,
            csamples,
            noreverse,
            force,
            randomseed,
            ipyclient,
            **kwargs)


def _samples_precheck(self, samples, mystep, force):
        """ Return a list of samples that are actually ready for the next step.
            Each step runs this prior to calling run, makes it easier to
            centralize and normalize how each step is checking sample states.
            mystep is the state produced by the current step.
        """
        subsample = []
        ## filter by state
        for sample in samples:
            if sample.stats.state < mystep - 1:
                LOGGER.debug("Sample {} not in proper state."\
                             .format(sample.name))
            else:
                subsample.append(sample)
        return subsample

def
_samples_precheck
(
self
,
samples
,
mystep
,
force
)
:
subsample
=
[
]
## filter by state
for
sample
in
samples
:
if
sample
.
stats
.
state
<
mystep
1
:
LOGGER
.
debug
(
"Sample {} not in proper state."
.
format
(
sample
.
name
)
)
else
:
subsample
.
append
(
sample
)
return
subsample

Return a list of samples that are actually ready for the next step . Each step runs this prior to calling run makes it easier to centralize and normalize how each step is checking sample states . mystep is the state produced by the current step .


def combinefiles(filepath):
    """ Joins first and second read file names """
    ## unpack seq files in filepath
    fastqs = glob.glob(filepath)
    firsts = [i for i in fastqs if "_R1_" in i]

    ## check names
    if not firsts:
        raise IPyradWarningExit("First read files names must contain '_R1_'.")

    ## get paired reads
    seconds = [ff.replace("_R1_", "_R2_") for ff in firsts]
    return zip(firsts, seconds)


def get_barcode_func(data, longbar):
    """ returns the fastest func given data & longbar"""
    ## build func for finding barcode
    if longbar[1] == 'same':
        if data.paramsdict["datatype"] == '2brad':
            def getbarcode(cutters, read1, longbar):
                """ find barcode for 2bRAD data """
                return read1[1][:-(len(cutters[0][0]) + 1)][-longbar[0]:]

        else:
            def getbarcode(_, read1, longbar):
                """ finds barcode for invariable length barcode data """
                return read1[1][:longbar[0]]
    else:
        def getbarcode(cutters, read1, longbar):
            """ finds barcode for variable barcode lengths"""
            return findbcode(cutters, longbar, read1)
    return getbarcode


def get_quart_iter(tups):
    """ returns an iterator to grab four lines at a time """

    if tups[0].endswith(".gz"):
        ofunc = gzip.open
    else:
        ofunc = open

    ## create iterators
    ofile1 = ofunc(tups[0], 'r')
    fr1 = iter(ofile1)
    quart1 = itertools.izip(fr1, fr1, fr1, fr1)
    if tups[1]:
        ofile2 = ofunc(tups[1], 'r')
        fr2 = iter(ofile2)
        quart2 = itertools.izip(fr2, fr2, fr2, fr2)
        quarts = itertools.izip(quart1, quart2)
    else:
        ofile2 = 0
        quarts = itertools.izip(quart1, iter(int, 1))

    ## make a generator
    def feedme(quarts):
        for quart in quarts:
            yield quart
    genquarts = feedme(quarts)

    ## return generator and handles
    return genquarts, ofile1, ofile2

def
get_quart_iter
(
tups
)
:
if
tups
[
0
]
.
endswith
(
".gz"
)
:
ofunc
=
gzip
.
open
else
:
ofunc
=
open
## create iterators
ofile1
=
ofunc
(
tups
[
0
]
,
'r'
)
fr1
=
iter
(
ofile1
)
quart1
=
itertools
.
izip
(
fr1
,
fr1
,
fr1
,
fr1
)
if
tups
[
1
]
:
ofile2
=
ofunc
(
tups
[
1
]
,
'r'
)
fr2
=
iter
(
ofile2
)
quart2
=
itertools
.
izip
(
fr2
,
fr2
,
fr2
,
fr2
)
quarts
=
itertools
.
izip
(
quart1
,
quart2
)
else
:
ofile2
=
0
quarts
=
itertools
.
izip
(
quart1
,
iter
(
int
,
1
)
)
## make a generator
def
feedme
(
quarts
)
:
for
quart
in
quarts
:
yield
quart
genquarts
=
feedme
(
quarts
)
## return generator and handles
return
genquarts
,
ofile1
,
ofile2


def writetofastq(data, dsort, read):
    """
    Writes sorted data 'dsort dict' to a tmp files
    """
    if read == 1:
        rrr = "R1"
    else:
        rrr = "R2"

    for sname in dsort:
        ## skip writing if empty. Write to tmpname
        handle = os.path.join(data.dirs.fastqs,
                "{}_{}_.fastq".format(sname, rrr))
        with open(handle, 'a') as out:
            out.write("".join(dsort[sname]))

Writes sorted data dsort dict to a tmp files


def collate_files(data, sname, tmp1s, tmp2s):
    """
    Collate temp fastq files in tmp-dir into 1 gzipped sample.
    """
    ## out handle
    out1 = os.path.join(data.dirs.fastqs, "{}_R1_.fastq.gz".format(sname))
    out = io.BufferedWriter(gzip.open(out1, 'w'))

    ## build cmd
    cmd1 = ['cat']
    for tmpfile in tmp1s:
        cmd1 += [tmpfile]

    ## compression function
    proc = sps.Popen(['which', 'pigz'], stderr=sps.PIPE, stdout=sps.PIPE).communicate()
    if proc[0].strip():
        compress = ["pigz"]
    else:
        compress = ["gzip"]

    ## call cmd
    proc1 = sps.Popen(cmd1, stderr=sps.PIPE, stdout=sps.PIPE)
    proc2 = sps.Popen(compress, stdin=proc1.stdout, stderr=sps.PIPE, stdout=out)
    err = proc2.communicate()
    if proc2.returncode:
        raise IPyradWarningExit("error in collate_files R1 %s", err)
    proc1.stdout.close()
    out.close()

    ## then cleanup
    for tmpfile in tmp1s:
        os.remove(tmpfile)

    if 'pair' in data.paramsdict["datatype"]:
        ## out handle
        out2 = os.path.join(data.dirs.fastqs, "{}_R2_.fastq.gz".format(sname))
        out = io.BufferedWriter(gzip.open(out2, 'w'))

        ## build cmd
        cmd1 = ['cat']
        for tmpfile in tmp2s:
            cmd1 += [tmpfile]

        ## call cmd
        proc1 = sps.Popen(cmd1, stderr=sps.PIPE, stdout=sps.PIPE)
        proc2 = sps.Popen(compress, stdin=proc1.stdout, stderr=sps.PIPE, stdout=out)
        err = proc2.communicate()
        if proc2.returncode:
            raise IPyradWarningExit("error in collate_files R2 %s", err)
        proc1.stdout.close()
        out.close()

        ## then cleanup
        for tmpfile in tmp2s:
            os.remove(tmpfile)

def
collate_files
(
data
,
sname
,
tmp1s
,
tmp2s
)
:
## out handle
out1
=
os
.
path
.
join
(
data
.
dirs
.
fastqs
,
"{}_R1_.fastq.gz"
.
format
(
sname
)
)
out
=
io
.
BufferedWriter
(
gzip
.
open
(
out1
,
'w'
)
)
## build cmd
cmd1
=
[
'cat'
]
for
tmpfile
in
tmp1s
:
cmd1
+=
[
tmpfile
]
## compression function
proc
=
sps
.
Popen
(
[
'which'
,
'pigz'
]
,
stderr
=
sps
.
PIPE
,
stdout
=
sps
.
PIPE
)
.
communicate
(
)
if
proc
[
0
]
.
strip
(
)
:
compress
=
[
"pigz"
]
else
:
compress
=
[
"gzip"
]
## call cmd
proc1
=
sps
.
Popen
(
cmd1
,
stderr
=
sps
.
PIPE
,
stdout
=
sps
.
PIPE
)
proc2
=
sps
.
Popen
(
compress
,
stdin
=
proc1
.
stdout
,
stderr
=
sps
.
PIPE
,
stdout
=
out
)
err
=
proc2
.
communicate
(
)
if
proc2
.
returncode
:
raise
IPyradWarningExit
(
"error in collate_files R1 %s"
,
err
)
proc1
.
stdout
.
close
(
)
out
.
close
(
)
## then cleanup
for
tmpfile
in
tmp1s
:
os
.
remove
(
tmpfile
)
if
'pair'
in
data
.
paramsdict
[
"datatype"
]
:
## out handle
out2
=
os
.
path
.
join
(
data
.
dirs
.
fastqs
,
"{}_R2_.fastq.gz"
.
format
(
sname
)
)
out
=
io
.
BufferedWriter
(
gzip
.
open
(
out2
,
'w'
)
)
## build cmd
cmd1
=
[
'cat'
]
for
tmpfile
in
tmp2s
:
cmd1
+=
[
tmpfile
]
## call cmd
proc1
=
sps
.
Popen
(
cmd1
,
stderr
=
sps
.
PIPE
,
stdout
=
sps
.
PIPE
)
proc2
=
sps
.
Popen
(
compress
,
stdin
=
proc1
.
stdout
,
stderr
=
sps
.
PIPE
,
stdout
=
out
)
err
=
proc2
.
communicate
(
)
if
proc2
.
returncode
:
raise
IPyradWarningExit
(
"error in collate_files R2 %s"
,
err
)
proc1
.
stdout
.
close
(
)
out
.
close
(
)
## then cleanup
for
tmpfile
in
tmp2s
:
os
.
remove
(
tmpfile
)

Collate temp fastq files in tmp - dir into 1 gzipped sample .


def estimate_optim(data, testfile, ipyclient):
    """
    Estimate a reasonable optim value by grabbing a chunk of sequences,
    decompressing and counting them, to estimate the full file size.
    """
    ## count the len of one file and assume all others are similar len
    insize = os.path.getsize(testfile)
    tmp_file_name = os.path.join(data.paramsdict["project_dir"], "tmp-step1-count.fq")
    if testfile.endswith(".gz"):
        infile = gzip.open(testfile)
        outfile = gzip.open(tmp_file_name, 'wb', compresslevel=5)
    else:
        infile = open(testfile)
        outfile = open(tmp_file_name, 'w')

    ## We'll take the average of the size of a file based on the
    ## first 10000 reads to approximate number of reads in the main file
    outfile.write("".join(itertools.islice(infile, 40000)))
    outfile.close()
    infile.close()

    ## Get the size of the tmp file
    tmp_size = os.path.getsize(tmp_file_name)

    ## divide by the tmp file size and multiply by 10000 to approximate
    ## the size of the input .fq files
    inputreads = int(insize / tmp_size) * 10000
    os.remove(tmp_file_name)

    return inputreads

Estimate a reasonable optim value by grabbing a chunk of sequences decompressing and counting them to estimate the full file size .


def _cleanup_and_die(data):
    """ cleanup func for step 1 """
    tmpfiles = glob.glob(os.path.join(data.dirs.fastqs, "tmp_*_R*.fastq"))
    tmpfiles += glob.glob(os.path.join(data.dirs.fastqs, "tmp_*.p"))
    for tmpf in tmpfiles:
        os.remove(tmpf)

def _cleanup_and_die ( data ) : tmpfiles = glob . glob ( os . path . join ( data . dirs . fastqs , "tmp_*_R*.fastq" ) ) tmpfiles += glob . glob ( os . path . join ( data . dirs . fastqs , "tmp_*.p" ) ) for tmpf in tmpfiles : os . remove ( tmpf )


def splitfiles(data, raws, ipyclient):
    """ sends raws to be chunked"""

    ## create a tmpdir for chunked_files and a chunk optimizer
    tmpdir = os.path.join(data.paramsdict["project_dir"], "tmp-chunks-"+data.name)
    if os.path.exists(tmpdir):
        shutil.rmtree(tmpdir)
    os.makedirs(tmpdir)

    ## chunk into 8M reads
    totalreads = estimate_optim(data, raws[0][0], ipyclient)
    optim = int(8e6)
    njobs = int(totalreads/(optim/4.)) * len(raws)

    ## if more files than cpus: no chunking
    nosplit = 0
    if (len(raws) > len(ipyclient)) or (totalreads < optim):
        nosplit = 1

    ## send slices N at a time. The dict chunkfiles stores a tuple of rawpairs
    ## dictionary to store asyncresults for sorting jobs
    start = time.time()
    chunkfiles = {}
    for fidx, tups in enumerate(raws):
        handle = os.path.splitext(os.path.basename(tups[0]))[0]
        ## if number of lines is > 20M then just submit it
        if nosplit:
            chunkfiles[handle] = [tups]
        else:
            ## chunk the file using zcat_make_temps
            chunklist = zcat_make_temps(data, tups, fidx, tmpdir, optim, njobs, start)
            chunkfiles[handle] = chunklist
    if not nosplit:
        print("")

    return chunkfiles


def putstats(pfile, handle, statdicts):
    """ puts stats from pickles into a dictionary """

    ## load in stats
    with open(pfile, 'r') as infile:
        filestats, samplestats = pickle.load(infile)

    ## get dicts from statdicts tuple
    perfile, fsamplehits, fbarhits, fmisses, fdbars = statdicts

    ## pull new stats
    #handle = os.path.splitext(os.path.basename(handle))[0]
    perfile[handle] += filestats

    ## update sample stats
    samplehits, barhits, misses, dbars = samplestats
    fsamplehits.update(samplehits)
    fbarhits.update(barhits)
    fmisses.update(misses)
    fdbars.update(dbars)

    ## repack the tuple and return
    statdicts = perfile, fsamplehits, fbarhits, fmisses, fdbars
    return statdicts


def _countmatrix(lxs):
    """ fill a matrix with pairwise data sharing """

    ## an empty matrix
    share = np.zeros((lxs.shape[0], lxs.shape[0]))

    ## fill above
    names = range(lxs.shape[0])
    for row in lxs:
        for samp1, samp2 in itertools.combinations(names, 2):
            shared = lxs[samp1, lxs[samp2] > 0].sum()
            share[samp1, samp2] = shared

    ## mirror below
    ##share[]

    ## fill diagonal with total sample coverage
    for row in xrange(len(names)):
        share[row, row] = lxs[row].sum()

    return share


def paramname(param=""):
    """ Get the param name from the dict index value.
    """

    try:
        name = pinfo[str(param)][0].strip().split(" ")[1]
    except (KeyError, ValueError) as err:
        ## TODO: paramsinfo get description by param string not working.
        ## It would be cool to have an assembly object bcz then you could
        ## just do this:
        ##
        ## print(pinfo[data.paramsinfo.keys().index(param)])
        print("\tKey name/number not recognized - ".format(param), err)
        raise

    return name

Get the param name from the dict index value .


def save_json2(data):
    """ save to json."""

    ## convert everything to dicts
    ## skip _ipcluster cuz it's made new.
    datadict = OrderedDict([
        ("outfiles", data.__dict__["outfiles"]),
        ("stats_files", dict(data.__dict__["stats_files"])),
        ("stats_dfs", data.__dict__["stats_dfs"])
        ])

def
save_json2
(
data
)
:
## convert everything to dicts
## skip _ipcluster cuz it's made new.
datadict
=
OrderedDict
(
[
(
"outfiles"
,
data
.
__dict__
[
"outfiles"
]
)
,
(
"stats_files"
,
dict
(
data
.
__dict__
[
"stats_files"
]
)
)
,
(
"stats_dfs"
,
data
.
__dict__
[
"stats_dfs"
]
)
]
)


def save_json(data):
    """ Save assembly and samples as json """

    ## data as dict
    #### skip _ipcluster because it's made new
    #### skip _headers because it's loaded new
    #### statsfiles save only keys
    #### samples save only keys
    datadict = OrderedDict([
        ("_version", data.__dict__["_version"]),
        ("_checkpoint", data.__dict__["_checkpoint"]),
        ("name", data.__dict__["name"]),
        ("dirs", data.__dict__["dirs"]),
        ("paramsdict", data.__dict__["paramsdict"]),
        ("samples", data.__dict__["samples"].keys()),
        ("populations", data.__dict__["populations"]),
        ("database", data.__dict__["database"]),
        ("clust_database", data.__dict__["clust_database"]),
        ("outfiles", data.__dict__["outfiles"]),
        ("barcodes", data.__dict__["barcodes"]),
        ("stats_files", data.__dict__["stats_files"]),
        ("_hackersonly", data.__dict__["_hackersonly"]),
        ])

    ## sample dict
    sampledict = OrderedDict([])
    for key, sample in data.samples.iteritems():
        sampledict[key] = sample._to_fulldict()

    ## json format it using cumstom Encoder class
    fulldumps = json.dumps({
        "assembly": datadict,
        "samples": sampledict
        },
        cls=Encoder,
        sort_keys=False, indent=4, separators=(",", ":"),
        )

    ## save to file
    assemblypath = os.path.join(data.dirs.project, data.name+".json")
    if not os.path.exists(data.dirs.project):
        os.mkdir(data.dirs.project)

    ## protect save from interruption
    done = 0
    while not done:
        try:
            with open(assemblypath, 'w') as jout:
                jout.write(fulldumps)
            done = 1
        except (KeyboardInterrupt, SystemExit):
            print('.')
            continue


def encode(self, obj):
        """ function to encode json string"""
        def hint_tuples(item):
            """ embeds __tuple__ hinter in json strings """
            if isinstance(item, tuple):
                return {'__tuple__': True, 'items': item}
            if isinstance(item, list):
                return [hint_tuples(e) for e in item]
            if isinstance(item, dict):
                return {
                    key: hint_tuples(val) for key, val in item.iteritems()
                    }
            else:
                return item

        return super(Encoder, self).encode(hint_tuples(obj))


def depthplot(data, samples=None, dims=(None,None), canvas=(None,None),
              xmax=50, log=False, outprefix=None, use_maxdepth=False):
    """ plots histogram of coverages across clusters"""

    ## select samples to be plotted, requires depths info
    if not samples:
        samples = data.samples.keys()
        samples.sort()
    subsamples = OrderedDict([(i, data.samples[i]) for i in samples])

    ## get canvas dimensions based on n-samples
    if any(dims):
        ## user-supplied dimensions (...)
        print("userdims")
    else:
        if len(subsamples) <= 4:
            ## set dimension to N samples
            dims = (1, len(subsamples))
        else:
            dims = (len(subsamples)/4, 4)

    ## create canvas
    if any(canvas):
        print("usercanvas")
        canvas = toyplot.Canvas(width=canvas[0], height=canvas[1])
    else:
        canvas = toyplot.Canvas(width=200*dims[1], height=150*dims[0])

    ## get all of the data arrays
    for panel, sample in enumerate(subsamples):
        ## statistical called bins
        statdat = subsamples[sample].depths
        statdat = statdat[statdat >= data.paramsdict["mindepth_statistical"]]
        if use_maxdepth:
            statdat = {i:j for (i, j) in statdat if \
                                        i < data.paramsdict["maxdepth"]}
        sdat = np.histogram(statdat, range(50))

        ## majrule called bins
        statdat = subsamples[sample].depths
        statdat = statdat[statdat < data.paramsdict["mindepth_statistical"]]
        statdat = statdat[statdat >= data.paramsdict["mindepth_majrule"]]
        if use_maxdepth:
            statdat = statdat[statdat < data.paramsdict["maxdepth"]]
        mdat = np.histogram(statdat, range(50))

        ## excluded bins
        tots = data.samples[sample].depths
        tots = tots[tots < data.paramsdict["mindepth_majrule"]]
        if use_maxdepth:
            tots = tots[tots < data.paramsdict["maxdepth"]]
        edat = np.histogram(tots, range(50))

        ## fill in each panel of canvas with a sample
        axes = canvas.cartesian(grid=(dims[0], dims[1], panel), gutter=25)
        axes.x.domain.xmax = xmax
        axes.label.text = sample
        if log:
            axes.y.scale = "log"

        # heights = np.column_stack((sdat,mdat,edat))
        axes.bars(sdat)
        axes.bars(edat)
        axes.bars(mdat)

    ## return objects to be saved...
    if outprefix:
        toyplot.html.render(canvas, fobj=outprefix+".html")
        toyplot.svg.render(canvas, fobj=outprefix+".svg")


def _parse_00(ofile):
    """
    return 00 outfile as a pandas DataFrame
    """
    with open(ofile) as infile:
        ## read in the results summary from the end of the outfile
        arr = np.array(
            [" "] + infile.read().split("Summary of MCMC results\n\n\n")[1:][0]\
            .strip().split())

        ## reshape array
        rows = 12
        cols = (arr.shape[0] + 1) / rows
        arr = arr.reshape(rows, cols)

        ## make into labeled data frame
        df = pd.DataFrame(
            data=arr[1:, 1:],
            columns=arr[0, 1:],
            index=arr[1:, 0],
            ).T
        return df


def _parse_01(ofiles, individual=False):
    """
    a subfunction for summarizing results
    """

    ## parse results from outfiles
    cols = []
    dats = []
    for ofile in ofiles:

        ## parse file
        with open(ofile) as infile:
            dat  = infile.read()
        lastbits = dat.split(".mcmc.txt\n\n")[1:]
        results = lastbits[0].split("\n\n")[0].split()

        ## get shape from ...
        shape = (((len(results) - 3) / 4), 4)
        dat = np.array(results[3:]).reshape(shape)
        cols.append(dat[:, 3].astype(float))

    if not individual:
        ## get mean results across reps
        cols = np.array(cols)
        cols = cols.sum(axis=0) / len(ofiles) #10.
        dat[:, 3] = cols.astype(str)

        ## format as a DF
        df = pd.DataFrame(dat[:, 1:])
        df.columns = ["delim", "prior", "posterior"]
        nspecies = 1 + np.array([list(i) for i in dat[:, 1]], dtype=int).sum(axis=1)
        df["nspecies"] = nspecies
        return df

    else:
        ## get mean results across reps
        #return cols
        res = []
        for i in xrange(len(cols)):
            x = dat
            x[:, 3] = cols[i].astype(str)
            x = pd.DataFrame(x[:, 1:])
            x.columns = ['delim', 'prior', 'posterior']
            nspecies = 1 + np.array([list(i) for i in dat[:, 1]], dtype=int).sum(axis=1)
            x["nspecies"] = nspecies
            res.append(x)
        return res

def
_parse_01
(
ofiles
,
individual
=
False
)
:
## parse results from outfiles
cols
=
[
]
dats
=
[
]
for
ofile
in
ofiles
:
## parse file
with
open
(
ofile
)
as
infile
:
dat
=
infile
.
read
(
)
lastbits
=
dat
.
split
(
".mcmc.txt\n\n"
)
[
1
:
]
results
=
lastbits
[
0
]
.
split
(
"\n\n"
)
[
0
]
.
split
(
)
## get shape from ...
shape
=
(
(
(
len
(
results
)
3
)
/
4
)
,
4
)
dat
=
np
.
array
(
results
[
3
:
]
)
.
reshape
(
shape
)
cols
.
append
(
dat
[
:
,
3
]
.
astype
(
float
)
)
if
not
individual
:
## get mean results across reps
cols
=
np
.
array
(
cols
)
cols
=
cols
.
sum
(
axis
=
0
)
/
len
(
ofiles
)
#10.
dat
[
:
,
3
]
=
cols
.
astype
(
str
)
## format as a DF
df
=
pd
.
DataFrame
(
dat
[
:
,
1
:
]
)
df
.
columns
=
[
"delim"
,
"prior"
,
"posterior"
]
nspecies
=
1
+
np
.
array
(
[
list
(
i
)
for
i
in
dat
[
:
,
1
]
]
,
dtype
=
int
)
.
sum
(
axis
=
1
)
df
[
"nspecies"
]
=
nspecies
return
df
else
:
## get mean results across reps
#return cols
res
=
[
]
for
i
in
xrange
(
len
(
cols
)
)
:
x
=
dat
x
[
:
,
3
]
=
cols
[
i
]
.
astype
(
str
)
x
=
pd
.
DataFrame
(
x
[
:
,
1
:
]
)
x
.
columns
=
[
'delim'
,
'prior'
,
'posterior'
]
nspecies
=
1
+
np
.
array
(
[
list
(
i
)
for
i
in
dat
[
:
,
1
]
]
,
dtype
=
int
)
.
sum
(
axis
=
1
)
x
[
"nspecies"
]
=
nspecies
res
.
append
(
x
)
return
res


def _load_existing_results(self, name, workdir):
        """
        Load existing results files for an object with this workdir and name.
        This does NOT reload the parameter settings for the object...
        """
        ## get mcmcs
        path = os.path.realpath(os.path.join(self.workdir, self.name))
        mcmcs = glob.glob(path+"_r*.mcmc.txt")
        outs = glob.glob(path+"_r*.out.txt")
        trees = glob.glob(path+"_r*.tre")

        for mcmcfile in mcmcs:
            if mcmcfile not in self.files.mcmcfiles:
                self.files.mcmcfiles.append(mcmcfile)
        for outfile in outs:
            if outfile not in self.files.outfiles:
                self.files.outfiles.append(outfile)
        for tree in trees:
            if tree not in self.files.treefiles:
                self.files.treefiles.append(tree)

def
_load_existing_results
(
self
,
name
,
workdir
)
:
## get mcmcs
path
=
os
.
path
.
realpath
(
os
.
path
.
join
(
self
.
workdir
,
self
.
name
)
)
mcmcs
=
glob
.
glob
(
path
+
"_r*.mcmc.txt"
)
outs
=
glob
.
glob
(
path
+
"_r*.out.txt"
)
trees
=
glob
.
glob
(
path
+
"_r*.tre"
)
for
mcmcfile
in
mcmcs
:
if
mcmcfile
not
in
self
.
files
.
mcmcfiles
:
self
.
files
.
mcmcfiles
.
append
(
mcmcfile
)
for
outfile
in
outs
:
if
outfile
not
in
self
.
files
.
outfiles
:
self
.
files
.
outfiles
.
append
(
outfile
)
for
tree
in
trees
:
if
tree
not
in
self
.
files
.
treefiles
:
self
.
files
.
treefiles
.
append
(
tree
)

Load existing results files for an object with this workdir and name . This does NOT reload the parameter settings for the object ...


def summarize_results(self, individual_results=False):
        """
        Prints a summarized table of results from replicate runs, or,
        if individual_result=True, then returns a list of separate
        dataframes for each replicate run.
        """

        ## return results depending on algorithm

        ## algorithm 00
        if (not self.params.infer_delimit) & (not self.params.infer_sptree):
            if individual_results:
                ## return a list of parsed CSV results
                return [_parse_00(i) for i in self.files.outfiles]
            else:
                ## concatenate each CSV and then get stats w/ describe
                return pd.concat(
                    [pd.read_csv(i, sep='\t', index_col=0) \
                    for i in self.files.mcmcfiles]).describe().T

        ## algorithm 01
        if self.params.infer_delimit & (not self.params.infer_sptree):
            return _parse_01(self.files.outfiles, individual=individual_results)

        ## others
        else:
            return "summary function not yet ready for this type of result"

Prints a summarized table of results from replicate runs or if individual_result = True then returns a list of separate dataframes for each replicate run .


def multi_muscle_align(data, samples, ipyclient):
    """
    Sends the cluster bits to nprocessors for muscle alignment. They return
    with indel.h5 handles to be concatenated into a joint h5.
    """
    LOGGER.info("starting alignments")

    ## get client
    lbview = ipyclient.load_balanced_view()
    start = time.time()
    printstr = " aligning clusters     | {} | s6 |"
    elapsed = datetime.timedelta(seconds=int(time.time()-start))
    progressbar(20, 0, printstr.format(elapsed), spacer=data._spacer)

    ## submit clustbits as jobs to engines. The chunkfiles are removed when they
    ## are finished so this job can even be restarted if it was half finished,
    ## though that is probably rare.
    path = os.path.join(data.tmpdir, data.name + ".chunk_*")
    clustbits = glob.glob(path)
    jobs = {}
    for idx in xrange(len(clustbits)):
        args = [data, samples, clustbits[idx]]
        jobs[idx] = lbview.apply(persistent_popen_align3, *args)
    allwait = len(jobs)
    elapsed = datetime.timedelta(seconds=int(time.time()-start))
    progressbar(20, 0, printstr.format(elapsed), spacer=data._spacer)

    ## print progress while bits are aligning
    while 1:
        finished = [i.ready() for i in jobs.values()]
        fwait = sum(finished)
        elapsed = datetime.timedelta(seconds=int(time.time()-start))
        progressbar(allwait, fwait, printstr.format(elapsed), spacer=data._spacer)
        time.sleep(0.1)
        if all(finished):
            break

    ## check for errors in muscle_align_across
    keys = jobs.keys()
    for idx in keys:
        if not jobs[idx].successful():
            LOGGER.error("error in persistent_popen_align %s", jobs[idx].exception())
            raise IPyradWarningExit("error in step 6 {}".format(jobs[idx].exception()))
        del jobs[idx]
    print("")

Sends the cluster bits to nprocessors for muscle alignment . They return with indel . h5 handles to be concatenated into a joint h5 .


def concatclusts(outhandle, alignbits):
    """ concatenates sorted aligned cluster tmpfiles and removes them."""
    with gzip.open(outhandle, 'wb') as out:
        for fname in alignbits:
            with open(fname) as infile:
                out.write(infile.read()+"//\n//\n")

def concatclusts ( outhandle , alignbits ) : with gzip . open ( outhandle , 'wb' ) as out : for fname in alignbits : with open ( fname ) as infile : out . write ( infile . read ( ) + "//\n//\n" )

concatenates sorted aligned cluster tmpfiles and removes them .


def fill_dups_arr(data):
    """
    fills the duplicates array from the multi_muscle_align tmp files
    """
    ## build the duplicates array
    duplefiles = glob.glob(os.path.join(data.tmpdir, "duples_*.tmp.npy"))
    duplefiles.sort(key=lambda x: int(x.rsplit("_", 1)[-1][:-8]))

    ## enter the duplicates filter into super h5 array
    io5 = h5py.File(data.clust_database, 'r+')
    dfilter = io5["duplicates"]

    ## enter all duple arrays into full duplicates array
    init = 0
    for dupf in duplefiles:
        end = int(dupf.rsplit("_", 1)[-1][:-8])
        inarr = np.load(dupf)
        dfilter[init:end] = inarr
        init += end-init
        #os.remove(dupf)
    #del inarr

    ## continued progress bar
    LOGGER.info("all duplicates: %s", dfilter[:].sum())
    io5.close()


def build_tmp_h5(data, samples):
    """ build tmp h5 arrays that can return quick access for nloci"""
    ## get samples and names, sorted
    snames = [i.name for i in samples]
    snames.sort()

    ## Build an array for quickly indexing consens reads from catg files.
    ## save as a npy int binary file.
    uhandle = os.path.join(data.dirs.across, data.name+".utemp.sort")
    bseeds = os.path.join(data.dirs.across, data.name+".tmparrs.h5")

    ## send as first async1 job
    get_seeds_and_hits(uhandle, bseeds, snames)


def get_nloci(data):
    """ return nloci from the tmp h5 arr"""
    bseeds = os.path.join(data.dirs.across, data.name+".tmparrs.h5")
    with h5py.File(bseeds) as io5:
        return io5["seedsarr"].shape[0]

def get_nloci ( data ) : bseeds = os . path . join ( data . dirs . across , data . name + ".tmparrs.h5" ) with h5py . File ( bseeds ) as io5 : return io5 [ "seedsarr" ] . shape [ 0 ]


def singlecat(data, sample, bseeds, sidx, nloci):
    """
    Orders catg data for each sample into the final locus order. This allows
    all of the individual catgs to simply be combined later. They are also in
    the same order as the indels array, so indels are inserted from the indel
    array that is passed in.
    """

    LOGGER.info("in single cat here")
    ## enter ref data?
    isref = 'reference' in data.paramsdict["assembly_method"]

    ## grab seeds and hits info for this sample
    with h5py.File(bseeds, 'r') as io5:
        ## get hits just for this sample and sort them by sample order index
        hits = io5["uarr"][:]
        hits = hits[hits[:, 1] == sidx, :]
        #hits = hits[hits[:, 2].argsort()]
        ## get seeds just for this sample and sort them by sample order index
        seeds = io5["seedsarr"][:]
        seeds = seeds[seeds[:, 1] == sidx, :]
        #seeds = seeds[seeds[:, 2].argsort()]
        full = np.concatenate((seeds, hits))
        full = full[full[:, 0].argsort()]

    ## still using max+20 len limit, rare longer merged reads get trimmed
    ## we need to allow room for indels to be added too
    maxlen = data._hackersonly["max_fragment_length"] + 20

    ## we'll fill a new catg and alleles arr for this sample in locus order,
    ## which is known from seeds and hits
    ocatg = np.zeros((nloci, maxlen, 4), dtype=np.uint32)
    onall = np.zeros(nloci, dtype=np.uint8)
    ochrom = np.zeros((nloci, 3), dtype=np.int64)

    ## grab the sample's data and write to ocatg and onall
    if not sample.files.database:
        raise IPyradWarningExit("missing catg file - {}".format(sample.name))

    with h5py.File(sample.files.database, 'r') as io5:
        ## get it and delete it
        catarr = io5["catg"][:]
        tmp = catarr[full[:, 2], :maxlen, :]
        del catarr
        ocatg[full[:, 0], :tmp.shape[1], :] = tmp
        del tmp

        ## get it and delete it
        nall = io5["nalleles"][:]
        onall[full[:, 0]] = nall[full[:, 2]]
        del nall

        ## fill the reference data
        if isref:
            chrom = io5["chroms"][:]
            ochrom[full[:, 0]] = chrom[full[:, 2]]
            del chrom

    ## get indel locations for this sample
    ipath = os.path.join(data.dirs.across, data.name+".tmp.indels.hdf5")
    with h5py.File(ipath, 'r') as ih5:
        indels = ih5["indels"][sidx, :, :maxlen]

    ## insert indels into ocatg
    newcatg = inserted_indels(indels, ocatg)
    del ocatg, indels

    ## save individual tmp h5 data
    smpio = os.path.join(data.dirs.across, sample.name+'.tmp.h5')
    with h5py.File(smpio, 'w') as oh5:
        oh5.create_dataset("icatg", data=newcatg, dtype=np.uint32)
        oh5.create_dataset("inall", data=onall, dtype=np.uint8)
        if isref:
            oh5.create_dataset("ichrom", data=ochrom, dtype=np.int64)

Orders catg data for each sample into the final locus order . This allows all of the individual catgs to simply be combined later . They are also in the same order as the indels array so indels are inserted from the indel array that is passed in .


def write_to_fullarr(data, sample, sidx):
    """ writes arrays to h5 disk """

    ## enter ref data?
    #isref = 'reference' in data.paramsdict["assembly_method"]
    LOGGER.info("writing fullarr %s %s", sample.name, sidx)

    ## save big arrays to disk temporarily
    with h5py.File(data.clust_database, 'r+') as io5:
        ## open views into the arrays we plan to fill
        chunk = io5["catgs"].attrs["chunksize"][0]
        catg = io5["catgs"]
        nall = io5["nalleles"]

        ## adding an axis to newcatg makes it write about 1000X faster.
        smpio = os.path.join(data.dirs.across, sample.name+'.tmp.h5')
        with h5py.File(smpio) as indat:

            ## grab all of the data from this sample's arrays
            newcatg = indat["icatg"] #[:]
            onall = indat["inall"]   #[:]

            ## enter it into the full array one chunk at a time
            for cidx in xrange(0, catg.shape[0], chunk):
                end = cidx + chunk
                catg[cidx:end, sidx:sidx+1, :] = np.expand_dims(newcatg[cidx:end, :], axis=1)
                nall[:, sidx:sidx+1] = np.expand_dims(onall, axis=1)


def dask_chroms(data, samples):
    """
    A dask relay function to fill chroms for all samples
    """

    ## example concatenating with dask
    h5s = [os.path.join(data.dirs.across, s.name+".tmp.h5") for s in samples]
    handles = [h5py.File(i) for i in h5s]
    dsets = [i['/ichrom'] for i in handles]
    arrays = [da.from_array(dset, chunks=(10000, 3)) for dset in dsets]
    stack = da.stack(arrays, axis=2)

    ## max chrom (should we check for variable hits? if so, things can get wonk)
    maxchrom = da.max(stack, axis=2)[:, 0]

    ## max pos
    maxpos = da.max(stack, axis=2)[:, 2]

    ## min pos
    mask = stack == 0
    stack[mask] = 9223372036854775807  ## max int64 value
    minpos = da.min(stack, axis=2)[:, 1]
    final = da.stack([maxchrom, minpos, maxpos], axis=1)
    final.to_hdf5(data.clust_database, "/chroms")

    ## close the h5 handles
    _ = [i.close() for i in handles]


def inserted_indels(indels, ocatg):
    """
    inserts indels into the catg array
    """
    ## return copy with indels inserted
    newcatg = np.zeros(ocatg.shape, dtype=np.uint32)

    ## iterate over loci and make extensions for indels
    for iloc in xrange(ocatg.shape[0]):
        ## get indels indices
        indidx = np.where(indels[iloc, :])[0]
        if np.any(indidx):
            ## which new (empty) rows will be added
            allrows = np.arange(ocatg.shape[1])
            mask = np.ones(allrows.shape[0], dtype=np.bool_)
            for idx in indidx:
                mask[idx] = False
            not_idx = allrows[mask == 1]

            ## fill in new data into all other spots
            newcatg[iloc][not_idx] = ocatg[iloc, :not_idx.shape[0]]
        else:
            newcatg[iloc] = ocatg[iloc]
    return newcatg


def count_seeds(usort):
    """
    uses bash commands to quickly count N seeds from utemp file
    """
    with open(usort, 'r') as insort:
        cmd1 = ["cut", "-f", "2"]
        cmd2 = ["uniq"]
        cmd3 = ["wc"]
        proc1 = sps.Popen(cmd1, stdin=insort, stdout=sps.PIPE, close_fds=True)
        proc2 = sps.Popen(cmd2, stdin=proc1.stdout, stdout=sps.PIPE, close_fds=True)
        proc3 = sps.Popen(cmd3, stdin=proc2.stdout, stdout=sps.PIPE, close_fds=True)
        res = proc3.communicate()
        nseeds = int(res[0].split()[0])
        proc1.stdout.close()
        proc2.stdout.close()
        proc3.stdout.close()
    return nseeds

def count_seeds ( usort ) : with open ( usort , 'r' ) as insort : cmd1 = [ "cut" , "-f" , "2" ] cmd2 = [ "uniq" ] cmd3 = [ "wc" ] proc1 = sps . Popen ( cmd1 , stdin = insort , stdout = sps . PIPE , close_fds = True ) proc2 = sps . Popen ( cmd2 , stdin = proc1 . stdout , stdout = sps . PIPE , close_fds = True ) proc3 = sps . Popen ( cmd3 , stdin = proc2 . stdout , stdout = sps . PIPE , close_fds = True ) res = proc3 . communicate ( ) nseeds = int ( res [ 0 ] . split ( ) [ 0 ] ) proc1 . stdout . close ( ) proc2 . stdout . close ( ) proc3 . stdout . close ( ) return nseeds


def sort_seeds(uhandle, usort):
    """ sort seeds from cluster results"""
    cmd = ["sort", "-k", "2", uhandle, "-o", usort]
    proc = sps.Popen(cmd, close_fds=True)
    proc.communicate()

def sort_seeds ( uhandle , usort ) : cmd = [ "sort" , "-k" , "2" , uhandle , "-o" , usort ] proc = sps . Popen ( cmd , close_fds = True ) proc . communicate ( )


def build_clustbits(data, ipyclient, force):
    """
    Reconstitutes clusters from .utemp and htemp files and writes them
    to chunked files for aligning in muscle.
    """

    ## If you run this step then we clear all tmp .fa and .indel.h5 files
    if os.path.exists(data.tmpdir):
        shutil.rmtree(data.tmpdir)
        os.mkdir(data.tmpdir)

    ## parallel client
    lbview = ipyclient.load_balanced_view()
    start = time.time()
    printstr = " building clusters     | {} | s6 |"
    elapsed = datetime.timedelta(seconds=int(time.time()-start))
    progressbar(3, 0, printstr.format(elapsed), spacer=data._spacer)

    uhandle = os.path.join(data.dirs.across, data.name+".utemp")
    usort = os.path.join(data.dirs.across, data.name+".utemp.sort")

    async1 = ""
    ## skip usorting if not force and already exists
    if not os.path.exists(usort) or force:

        ## send sort job to engines. Sorted seeds allows us to work through
        ## the utemp file one locus at a time instead of reading all into mem.
        LOGGER.info("building reads file -- loading utemp file into mem")
        async1 = lbview.apply(sort_seeds, *(uhandle, usort))
        while 1:
            elapsed = datetime.timedelta(seconds=int(time.time()-start))
            progressbar(3, 0, printstr.format(elapsed), spacer=data._spacer)
            if async1.ready():
                break
            else:
                time.sleep(0.1)

    ## send count seeds job to engines.
    async2 = lbview.apply(count_seeds, usort)
    while 1:
        elapsed = datetime.timedelta(seconds=int(time.time()-start))
        progressbar(3, 1, printstr.format(elapsed), spacer=data._spacer)
        if async2.ready():
            break
        else:
            time.sleep(0.1)

    ## wait for both to finish while printing progress timer
    nseeds = async2.result()

    ## send the clust bit building job to work and track progress
    async3 = lbview.apply(sub_build_clustbits, *(data, usort, nseeds))
    while 1:
        elapsed = datetime.timedelta(seconds=int(time.time()-start))
        progressbar(3, 2, printstr.format(elapsed), spacer=data._spacer)
        if async3.ready():
            break
        else:
            time.sleep(0.1)
    elapsed = datetime.timedelta(seconds=int(time.time()-start))
    progressbar(3, 3, printstr.format(elapsed), spacer=data._spacer)
    print("")

    ## check for errors
    for job in [async1, async2, async3]:
        try:
            if not job.successful():
                raise IPyradWarningExit(job.result())
        except AttributeError:
            ## If we skip usorting then async1 == "" so the call to
            ## successful() raises, but we can ignore it.
            pass

Reconstitutes clusters from . utemp and htemp files and writes them to chunked files for aligning in muscle .


def sub_build_clustbits(data, usort, nseeds):
    """
    A subfunction of build_clustbits to allow progress tracking. This func
    splits the unaligned clusters into bits for aligning on separate cores.
    """

    ## load FULL concat fasta file into a dict. This could cause RAM issues.
    ## this file has iupac codes in it, not ambigs resolved, and is gzipped.
    LOGGER.info("loading full _catcons file into memory")
    allcons = {}
    conshandle = os.path.join(data.dirs.across, data.name+"_catcons.tmp")
    with gzip.open(conshandle, 'rb') as iocons:
        cons = itertools.izip(*[iter(iocons)]*2)
        for namestr, seq in cons:
            nnn, sss = [i.strip() for i in namestr, seq]
            allcons[nnn[1:]] = sss

    ## set optim to approximately 4 chunks per core. Smaller allows for a bit
    ## cleaner looking progress bar. 40 cores will make 160 files.
    optim = ((nseeds // (data.cpus*4)) + (nseeds % (data.cpus*4)))
    LOGGER.info("building clustbits, optim=%s, nseeds=%s, cpus=%s",
                optim, nseeds, data.cpus)

    ## iterate through usort grabbing seeds and matches
    with open(usort, 'rb') as insort:
        ## iterator, seed null, and seqlist null
        isort = iter(insort)
        loci = 0
        lastseed = 0
        fseqs = []
        seqlist = []
        seqsize = 0

        while 1:
            ## grab the next line
            try:
                hit, seed, ori = isort.next().strip().split()
            except StopIteration:
                break

            try:
                ## if same seed, append match
                if seed != lastseed:
                    ## store the last fseq, count it, and clear it
                    if fseqs:
                        seqlist.append("\n".join(fseqs))
                        seqsize += 1
                        fseqs = []
                    ## occasionally write to file
                    if seqsize >= optim:
                        if seqlist:
                            loci += seqsize
                            with open(os.path.join(data.tmpdir,
                                data.name+".chunk_{}".format(loci)), 'w') as clustsout:
                                LOGGER.debug("writing chunk - seqsize {} loci {} {}".format(seqsize, loci, clustsout.name))
                                clustsout.write("\n//\n//\n".join(seqlist)+"\n//\n//\n")
                            ## reset list and counter
                            seqlist = []
                            seqsize = 0

                    ## store the new seed on top of fseq
                    fseqs.append(">{}\n{}".format(seed, allcons[seed]))
                    lastseed = seed

                ## add match to the seed
                seq = allcons[hit]
                ## revcomp if orientation is reversed
                if ori == "-":
                    seq = fullcomp(seq)[::-1]
                fseqs.append(">{}\n{}".format(hit, seq))
            except KeyError as inst:
                ## Caught bad seed or hit? Log and continue.
                LOGGER.error("Bad Seed/Hit: seqsize {}\tloci {}\tseed {}\thit {}".format(seqsize, loci, seed, hit))

    ## write whatever is left over to the clusts file
    if fseqs:
        seqlist.append("\n".join(fseqs))
        seqsize += 1
        loci += seqsize
    if seqlist:
        with open(os.path.join(data.tmpdir,
            data.name+".chunk_{}".format(loci)), 'w') as clustsout:
            clustsout.write("\n//\n//\n".join(seqlist)+"\n//\n//\n")

    ## final progress and cleanup
    del allcons
    clustbits = glob.glob(os.path.join(data.tmpdir, data.name+".chunk_*"))

    ## return stuff
    return clustbits, loci

A subfunction of build_clustbits to allow progress tracking . This func splits the unaligned clusters into bits for aligning on separate cores .


def cleanup_tempfiles(data):
    """
    Function to remove older files. This is called either in substep 1 or after
    the final substep so that tempfiles are retained for restarting interrupted
    jobs until we're sure they're no longer needed.
    """

    ## remove align-related tmp files
    tmps1 = glob.glob(os.path.join(data.tmpdir, "*.fa"))
    tmps2 = glob.glob(os.path.join(data.tmpdir, "*.npy"))
    for tmp in tmps1 + tmps2:
        if os.path.exists(tmp):
            os.remove(tmp)

    ## remove cluster related files
    removal = [
        os.path.join(data.dirs.across, data.name+".utemp"),
        os.path.join(data.dirs.across, data.name+".htemp"),
        os.path.join(data.dirs.across, data.name+"_catcons.tmp"),
        os.path.join(data.dirs.across, data.name+"_cathaps.tmp"),
        os.path.join(data.dirs.across, data.name+"_catshuf.tmp"),
        os.path.join(data.dirs.across, data.name+"_catsort.tmp"),
        os.path.join(data.dirs.across, data.name+".tmparrs.h5"),
        os.path.join(data.dirs.across, data.name+".tmp.indels.hdf5"),
        ]
    for rfile in removal:
        if os.path.exists(rfile):
            os.remove(rfile)

    ## remove singlecat related h5 files
    smpios = glob.glob(os.path.join(data.dirs.across, '*.tmp.h5'))
    for smpio in smpios:
        if os.path.exists(smpio):
            os.remove(smpio)

def
cleanup_tempfiles
(
data
)
:
## remove align-related tmp files
tmps1
=
glob
.
glob
(
os
.
path
.
join
(
data
.
tmpdir
,
"*.fa"
)
)
tmps2
=
glob
.
glob
(
os
.
path
.
join
(
data
.
tmpdir
,
"*.npy"
)
)
for
tmp
in
tmps1
+
tmps2
:
if
os
.
path
.
exists
(
tmp
)
:
os
.
remove
(
tmp
)
## remove cluster related files
removal
=
[
os
.
path
.
join
(
data
.
dirs
.
across
,
data
.
name
+
".utemp"
)
,
os
.
path
.
join
(
data
.
dirs
.
across
,
data
.
name
+
".htemp"
)
,
os
.
path
.
join
(
data
.
dirs
.
across
,
data
.
name
+
"_catcons.tmp"
)
,
os
.
path
.
join
(
data
.
dirs
.
across
,
data
.
name
+
"_cathaps.tmp"
)
,
os
.
path
.
join
(
data
.
dirs
.
across
,
data
.
name
+
"_catshuf.tmp"
)
,
os
.
path
.
join
(
data
.
dirs
.
across
,
data
.
name
+
"_catsort.tmp"
)
,
os
.
path
.
join
(
data
.
dirs
.
across
,
data
.
name
+
".tmparrs.h5"
)
,
os
.
path
.
join
(
data
.
dirs
.
across
,
data
.
name
+
".tmp.indels.hdf5"
)
,
]
for
rfile
in
removal
:
if
os
.
path
.
exists
(
rfile
)
:
os
.
remove
(
rfile
)
## remove singlecat related h5 files
smpios
=
glob
.
glob
(
os
.
path
.
join
(
data
.
dirs
.
across
,
'*.tmp.h5'
)
)
for
smpio
in
smpios
:
if
os
.
path
.
exists
(
smpio
)
:
os
.
remove
(
smpio
)

Function to remove older files . This is called either in substep 1 or after the final substep so that tempfiles are retained for restarting interrupted jobs until we re sure they re no longer needed .


def assembly_cleanup(data):
    """ cleanup for assembly object """

    ## build s2 results data frame
    data.stats_dfs.s2 = data._build_stat("s2")
    data.stats_files.s2 = os.path.join(data.dirs.edits, 's2_rawedit_stats.txt')

    ## write stats for all samples
    with io.open(data.stats_files.s2, 'w', encoding='utf-8') as outfile:
        data.stats_dfs.s2.fillna(value=0).astype(np.int).to_string(outfile)

def
assembly_cleanup
(
data
)
:
## build s2 results data frame
data
.
stats_dfs
.
s2
=
data
.
_build_stat
(
"s2"
)
data
.
stats_files
.
s2
=
os
.
path
.
join
(
data
.
dirs
.
edits
,
's2_rawedit_stats.txt'
)
## write stats for all samples
with
io
.
open
(
data
.
stats_files
.
s2
,
'w'
,
encoding
=
'utf-8'
)
as
outfile
:
data
.
stats_dfs
.
s2
.
fillna
(
value
=
0
)
.
astype
(
np
.
int
)
.
to_string
(
outfile
)


def parse_single_results(data, sample, res1):
    """ parse results from cutadapt into sample data"""

    ## set default values
    #sample.stats_dfs.s2["reads_raw"] = 0
    sample.stats_dfs.s2["trim_adapter_bp_read1"] = 0
    sample.stats_dfs.s2["trim_quality_bp_read1"] = 0
    sample.stats_dfs.s2["reads_filtered_by_Ns"] = 0
    sample.stats_dfs.s2["reads_filtered_by_minlen"] = 0
    sample.stats_dfs.s2["reads_passed_filter"] = 0

    ## parse new values from cutadapt results output
    lines = res1.strip().split("\n")
    for line in lines:

        if "Total reads processed:" in line:
            value = int(line.split()[3].replace(",", ""))
            sample.stats_dfs.s2["reads_raw"] = value

        if "Reads with adapters:" in line:
            value = int(line.split()[3].replace(",", ""))
            sample.stats_dfs.s2["trim_adapter_bp_read1"] = value

        if "Quality-trimmed" in line:
            value = int(line.split()[1].replace(",", ""))
            sample.stats_dfs.s2["trim_quality_bp_read1"] = value

        if "Reads that were too short" in line:
            value = int(line.split()[5].replace(",", ""))
            sample.stats_dfs.s2["reads_filtered_by_minlen"] = value

        if "Reads with too many N" in line:
            value = int(line.split()[5].replace(",", ""))
            sample.stats_dfs.s2["reads_filtered_by_Ns"] = value

        if "Reads written (passing filters):" in line:
            value = int(line.split()[4].replace(",", ""))
            sample.stats_dfs.s2["reads_passed_filter"] = value

    ## save to stats summary
    if sample.stats_dfs.s2.reads_passed_filter:
        sample.stats.state = 2
        sample.stats.reads_passed_filter = sample.stats_dfs.s2.reads_passed_filter
        sample.files.edits = [
            (OPJ(data.dirs.edits, sample.name+".trimmed_R1_.fastq.gz"), 0)]
        ## write the long form output to the log file.
        LOGGER.info(res1)

    else:
        print("{}No reads passed filtering in Sample: {}".format(data._spacer, sample.name))


def run2(data, samples, force, ipyclient):
    """
    Filter for samples that are already finished with this step, allow others
    to run, pass them to parallel client function to filter with cutadapt.
    """

    ## create output directories
    data.dirs.edits = os.path.join(os.path.realpath(
                                   data.paramsdict["project_dir"]),
                                   data.name+"_edits")
    if not os.path.exists(data.dirs.edits):
        os.makedirs(data.dirs.edits)

    ## get samples
    subsamples = choose_samples(samples, force)

    ## only allow extra adapters in filters==3,
    ## and add poly repeats if not in list of adapters
    if int(data.paramsdict["filter_adapters"]) == 3:
        if not data._hackersonly["p3_adapters_extra"]:
            for poly in ["A"*8, "T"*8, "C"*8, "G"*8]:
                data._hackersonly["p3_adapters_extra"].append(poly)
        if not data._hackersonly["p5_adapters_extra"]:
            for poly in ["A"*8, "T"*8, "C"*8, "G"*8]:
                data._hackersonly["p5_adapters_extra"].append(poly)
    else:
        data._hackersonly["p5_adapters_extra"] = []
        data._hackersonly["p3_adapters_extra"] = []

    ## concat is not parallelized (since it's disk limited, generally)
    subsamples = concat_reads(data, subsamples, ipyclient)

    ## cutadapt is parallelized by ncores/2 because cutadapt spawns threads
    lbview = ipyclient.load_balanced_view(targets=ipyclient.ids[::2])
    run_cutadapt(data, subsamples, lbview)

    ## cleanup is ...
    assembly_cleanup(data)

Filter for samples that are already finished with this step allow others to run pass them to parallel client function to filter with cutadapt .


def concat_reads(data, subsamples, ipyclient):
    """ concatenate if multiple input files for a single samples """

    ## concatenate reads if they come from merged assemblies.
    if any([len(i.files.fastqs) > 1 for i in subsamples]):
        ## run on single engine for now
        start = time.time()
        printstr = " concatenating inputs  | {} | s2 |"
        finished = 0
        catjobs = {}
        for sample in subsamples:
            if len(sample.files.fastqs) > 1:
                catjobs[sample.name] = ipyclient[0].apply(\
                                       concat_multiple_inputs, *(data, sample))
            else:
                sample.files.concat = sample.files.fastqs

        ## wait for all to finish
        while 1:
            finished = sum([i.ready() for i in catjobs.values()])
            elapsed = datetime.timedelta(seconds=int(time.time()-start))
            progressbar(len(catjobs), finished, printstr.format(elapsed), spacer=data._spacer)
            time.sleep(0.1)
            if finished == len(catjobs):
                print("")
                break

        ## collect results, which are concat file handles.
        for async in catjobs:
            if catjobs[async].successful():
                data.samples[async].files.concat = catjobs[async].result()
            else:
                error = catjobs[async].result()#exception()
                LOGGER.error("error in step2 concat %s", error)
                raise IPyradWarningExit("error in step2 concat: {}".format(error))
    else:
        for sample in subsamples:
            ## just copy fastqs handles to concat attribute
            sample.files.concat = sample.files.fastqs

    return subsamples


def run_cutadapt(data, subsamples, lbview):
    """
    sends fastq files to cutadapt
    """
    ## choose cutadapt function based on datatype
    start = time.time()
    printstr = " processing reads      | {} | s2 |"
    finished = 0
    rawedits = {}

    ## sort subsamples so that the biggest files get submitted first
    subsamples.sort(key=lambda x: x.stats.reads_raw, reverse=True)
    LOGGER.info([i.stats.reads_raw for i in subsamples])

    ## send samples to cutadapt filtering
    if "pair" in data.paramsdict["datatype"]:
        for sample in subsamples:
            rawedits[sample.name] = lbview.apply(cutadaptit_pairs, *(data, sample))
    else:
        for sample in subsamples:
            rawedits[sample.name] = lbview.apply(cutadaptit_single, *(data, sample))

    ## wait for all to finish
    while 1:
        finished = sum([i.ready() for i in rawedits.values()])
        elapsed = datetime.timedelta(seconds=int(time.time()-start))
        progressbar(len(rawedits), finished, printstr.format(elapsed), spacer=data._spacer)
        time.sleep(0.1)
        if finished == len(rawedits):
            print("")
            break

    ## collect results, report failures, and store stats. async = sample.name
    for async in rawedits:
        if rawedits[async].successful():
            res = rawedits[async].result()

            ## if single cleanup is easy
            if "pair" not in data.paramsdict["datatype"]:
                parse_single_results(data, data.samples[async], res)
            else:
                parse_pair_results(data, data.samples[async], res)
        else:
            print("  found an error in step2; see ipyrad_log.txt")
            LOGGER.error("error in run_cutadapt(): %s", rawedits[async].exception())


def concat_multiple_inputs(data, sample):
    """
    If multiple fastq files were appended into the list of fastqs for samples
    then we merge them here before proceeding.
    """

    ## if more than one tuple in fastq list
    if len(sample.files.fastqs) > 1:
        ## create a cat command to append them all (doesn't matter if they
        ## are gzipped, cat still works). Grab index 0 of tuples for R1s.
        cmd1 = ["cat"] + [i[0] for i in sample.files.fastqs]

        isgzip = ".gz"
        if not sample.files.fastqs[0][0].endswith(".gz"):
            isgzip = ""

        ## write to new concat handle
        conc1 = os.path.join(data.dirs.edits, sample.name+"_R1_concat.fq{}".format(isgzip))
        with open(conc1, 'w') as cout1:
            proc1 = sps.Popen(cmd1, stderr=sps.STDOUT, stdout=cout1, close_fds=True)
            res1 = proc1.communicate()[0]
        if proc1.returncode:
            raise IPyradWarningExit("error in: {}, {}".format(cmd1, res1))

        ## Only set conc2 if R2 actually exists
        conc2 = 0
        if "pair" in data.paramsdict["datatype"]:
            cmd2 = ["cat"] + [i[1] for i in sample.files.fastqs]
            conc2 = os.path.join(data.dirs.edits, sample.name+"_R2_concat.fq{}".format(isgzip))
            with open(conc2, 'w') as cout2:
                proc2 = sps.Popen(cmd2, stderr=sps.STDOUT, stdout=cout2, close_fds=True)
                res2 = proc2.communicate()[0]
            if proc2.returncode:
                raise IPyradWarningExit("Error concatenating fastq files. Make sure all "\
                    + "these files exist: {}\nError message: {}".format(cmd2, proc2.returncode))

        ## store new file handles
        sample.files.concat = [(conc1, conc2)]
    return sample.files.concat

If multiple fastq files were appended into the list of fastqs for samples then we merge them here before proceeding .


def make( data, samples ):
    """ Convert vcf from step6 to .loci format to facilitate downstream format conversion """
    invcffile   =  os.path.join( data.dirs.consens, data.name+".vcf" )
    outlocifile  =  os.path.join( data.dirs.outfiles, data.name+".loci" )

    importvcf( invcffile, outlocifile )

def make ( data , samples ) : invcffile = os . path . join ( data . dirs . consens , data . name + ".vcf" ) outlocifile = os . path . join ( data . dirs . outfiles , data . name + ".loci" ) importvcf ( invcffile , outlocifile )

Convert vcf from step6 to . loci format to facilitate downstream format conversion


def importvcf( vcffile, locifile ):
    """ Function for importing a vcf file into loci format. Arguments
        are the input vcffile and the loci file to write out. """

    try:
        ## Get names of all individuals in the vcf
        with open( invcffile, 'r' ) as invcf:
            for line in invcf:
                if line.split()[0] == "#CHROM":
                    ## This is maybe a little clever. The names in the vcf are everything after
                    ## the "FORMAT" column, so find that index, then slice everything after it.
                    names_col = line.split().index( "FORMAT" ) + 1
                    names = line.split()[ names_col:]
                    LOGGER.debug( "Got names - %s", names )
                    break

            print( "wat" )
        ## Get the column to start reading at
    except Exception:
        print( "wat" )

Function for importing a vcf file into loci format . Arguments are the input vcffile and the loci file to write out .


def get_targets(ipyclient):
    """
    A function to find 2 engines per hostname on the ipyclient.
    We'll assume that the CPUs are hyperthreaded, which is why
    we grab two. If they are not then no foul. Two multi-threaded
    jobs will be run on each of the 2 engines per host.
    """
    ## fill hosts with async[gethostname]
    hosts = []
    for eid in ipyclient.ids:
        engine = ipyclient[eid]
        if not engine.outstanding:
            hosts.append(engine.apply(socket.gethostname))

    ## capture results of asyncs
    hosts = [i.get() for i in hosts]
    hostset = set(hosts)
    hostzip = zip(hosts, ipyclient.ids)
    hostdict = {host: [i[1] for i in hostzip if i[0] == host] for host in hostset}
    targets = list(itertools.chain(*[hostdict[i][:2] for i in hostdict]))

    ## return first two engines from each host
    return targets

A function to find 2 engines per hostname on the ipyclient . We ll assume that the CPUs are hyperthreaded which is why we grab two . If they are not then no foul . Two multi - threaded jobs will be run on each of the 2 engines per host .


def compute_tree_stats(self, ipyclient):
    """
    compute stats for stats file and NHX tree features
    """

    ## get name indices
    names = self.samples

    ## get majority rule consensus tree of weighted Q bootstrap trees
    if self.params.nboots:

        ## Tree object
        fulltre = ete3.Tree(self.trees.tree, format=0)
        fulltre.unroot()

        ## only grab as many boots as the last option said was max
        with open(self.trees.boots, 'r') as inboots:
            bb = [ete3.Tree(i.strip(), format=0) for i in inboots.readlines()]
            wboots = [fulltre] + bb[-self.params.nboots:]

        ## infer consensus tree and write to file
        wctre, wcounts = consensus_tree(wboots, names=names)
        self.trees.cons = os.path.join(self.dirs, self.name + ".cons")
        with open(self.trees.cons, 'w') as ocons:
            ocons.write(wctre.write(format=0))
    else:
        wctre = ete3.Tree(self.trees.tree, format=0)
        wctre.unroot()

    ## build stats file and write trees
    self.trees.nhx = os.path.join(self.dirs, self.name + ".nhx")
    with open(self.files.stats, 'w') as ostats:

        ## print Tetrad info
        #ostats.write(STATS_STRING.format(**self.stats))

        ## print bootstrap splits
        if self.params.nboots:
            ostats.write("## splits observed in {} trees\n".format(len(wboots)))
            for i, j in enumerate(self.samples):
                ostats.write("{:<3} {}\n".format(i, j))
            ostats.write("\n")
            for split, freq in wcounts:
                if split.count('1') > 1:
                    ostats.write("{}   {:.2f}\n".format(split, round(freq, 2)))
            ostats.write("\n")

    ## parallelized this function because it can be slogging
    lbview = ipyclient.load_balanced_view()

    ## store results in dicts
    qtots = {}
    qsamp = {}
    tots = sum(1 for i in wctre.iter_leaves())
    totn = set(wctre.get_leaf_names())

    ## iterate over node traversal.
    for node in wctre.traverse():
        ## this is slow, needs to look at every sampled quartet
        ## so we send it be processed on an engine
        qtots[node] = lbview.apply(_get_total, *(tots, node))
        qsamp[node] = lbview.apply(_get_sampled, *(self, totn, node))

    ## wait for jobs to finish
    ipyclient.wait()

    ## put results into tree
    for node in wctre.traverse():
        ## this is fast, just calcs n_choose_k
        total = qtots[node].result()
        sampled = qsamp[node].result()
        ## store the results to the tree
        node.add_feature("quartets_total", total)
        node.add_feature("quartets_sampled", sampled)
    features = ["quartets_total", "quartets_sampled"]

    ## return as NHX format with extra info
    with open(self.trees.nhx, 'w') as outtre:
        outtre.write(wctre.write(format=0, features=features))