def _run_qmc(self, boot):
        """
        Runs quartet max-cut QMC on the quartets qdump file.
        """

        ## build command
        self._tmp = os.path.join(self.dirs, ".tmptre")
        cmd = [ip.bins.qmc, "qrtt="+self.files.qdump, "otre="+self._tmp]

        ## run it
        proc = subprocess.Popen(cmd, stderr=subprocess.STDOUT, stdout=subprocess.PIPE)
        res = proc.communicate()
        if proc.returncode:
            raise IPyradWarningExit(res[1])

        ## parse tmp file written by qmc into a tree and rename it
        with open(self._tmp, 'r') as intree:
            tre = ete3.Tree(intree.read().strip())
            names = tre.get_leaves()
            for name in names:
                name.name = self.samples[int(name.name)]
            tmptre = tre.write(format=9)

        ## save the tree to file
        if boot:
            self.trees.boots = os.path.join(self.dirs, self.name+".boots")
            with open(self.trees.boots, 'a') as outboot:
                outboot.write(tmptre+"\n")
        else:
            self.trees.tree = os.path.join(self.dirs, self.name+".tree")
            with open(self.trees.tree, 'w') as outtree:
                outtree.write(tmptre)

        ## save the file
        self._save()

Runs quartet max - cut QMC on the quartets qdump file .


def _insert_to_array(self, chunk, results):
        """
        Enters results arrays into the HDF5 database.
        """

        ## two result arrs
        chunksize = self._chunksize
        qrts, invs = results

        ## enter into db
        with h5py.File(self.database.output, 'r+') as io5:
            io5['quartets'][chunk:chunk+chunksize] = qrts

            ## entered as 0-indexed !
            if self.params.save_invariants:
                if self.checkpoint.boots:
                    key = "invariants/boot{}".format(self.checkpoint.boots)
                    io5[key][chunk:chunk+chunksize] = invs
                else:
                    io5["invariants/boot0"][chunk:chunk+chunksize] = invs

def
_insert_to_array
(
self
,
chunk
,
results
)
:
## two result arrs
chunksize
=
self
.
_chunksize
qrts
,
invs
=
results
## enter into db
with
h5py
.
File
(
self
.
database
.
output
,
'r+'
)
as
io5
:
io5
[
'quartets'
]
[
chunk
:
chunk
+
chunksize
]
=
qrts
## entered as 0-indexed !
if
self
.
params
.
save_invariants
:
if
self
.
checkpoint
.
boots
:
key
=
"invariants/boot{}"
.
format
(
self
.
checkpoint
.
boots
)
io5
[
key
]
[
chunk
:
chunk
+
chunksize
]
=
invs
else
:
io5
[
"invariants/boot0"
]
[
chunk
:
chunk
+
chunksize
]
=
invs

Enters results arrays into the HDF5 database .


def get_client(cluster_id, profile, engines, timeout, cores, quiet, spacer, **kwargs):
    """
    Creates a client to view ipcluster engines for a given profile and
    returns it with at least one engine spun up and ready to go. If no
    engines are found after nwait amount of time then an error is raised.
    If engines==MPI it waits a bit longer to find engines. If the number
    of engines is set then it waits even longer to try to find that number
    of engines.
    """

    ## save stds for later, we're gonna hide them to prevent external printing
    save_stdout = sys.stdout
    save_stderr = sys.stderr
    sys.stdout = cStringIO.StringIO()
    sys.stderr = cStringIO.StringIO()

    ## get cluster_info print string
    connection_string = "{}establishing parallel connection:".format(spacer)

    ## wrapped search for ipcluster
    try:
        ## are we looking for a running ipcluster instance?
        if profile not in [None, "default"]:
            args = {'profile': profile, "timeout": timeout}
        else:
            clusterargs = [cluster_id, profile, timeout]
            argnames = ["cluster_id", "profile", "timeout"]
            args = {key:value for key, value in zip(argnames, clusterargs)}

        ## get connection within timeout window of wait time and hide messages
        ipyclient = ipp.Client(**args)
        sys.stdout = save_stdout
        sys.stderr = save_stderr

        ## check that all engines have connected
        if (engines == "MPI") or ("ipyrad-cli-" in cluster_id):
            if not quiet:
                print(connection_string)

        for _ in range(6000):
            initid = len(ipyclient)
            time.sleep(0.01)
            ## If MPI then wait for all engines to start so we can report
            ## how many cores are on each host. If Local then only wait for
            ## one engine to be ready and then just go.
            if (engines == "MPI") or ("ipyrad-cli-" in cluster_id):
                ## wait for cores to be connected
                if cores:
                    time.sleep(0.1)
                    if initid == cores:
                        break
                if initid:
                    time.sleep(3)
                    if len(ipyclient) == initid:
                        break
            else:
                if cores:
                    if initid == cores:
                        break
                else:
                    if initid:
                        break

    except KeyboardInterrupt as inst:
        ## ensure stdout is reset even if Exception was raised
        sys.stdout = save_stdout
        sys.stderr = save_stderr
        raise inst

    ## This is raised if ipcluster is not running ------------
    except IOError as inst:
        ## ensure stdout is reset even if Exception was raised
        sys.stdout = save_stdout
        sys.stderr = save_stderr
        if "ipyrad-cli-" in cluster_id:
            raise IPyradWarningExit(NO_IPCLUSTER_CLI)
        else:
            raise IPyradWarningExit(NO_IPCLUSTER_API)

    except (ipp.TimeoutError, ipp.NoEnginesRegistered) as inst:
        ## raised by ipp if no connection file is found for 'nwait' seconds
        sys.stdout = save_stdout
        sys.stderr = save_stderr
        raise inst

    except Exception as inst:
        ## if any other exceptions were missed...
        sys.stdout = save_stdout
        sys.stderr = save_stderr
        raise inst

    finally:
        ## ensure that no matter what we reset the stds
        sys.stdout = save_stdout
        sys.stderr = save_stderr

    return ipyclient

Creates a client to view ipcluster engines for a given profile and returns it with at least one engine spun up and ready to go . If no engines are found after nwait amount of time then an error is raised . If engines == MPI it waits a bit longer to find engines . If the number of engines is set then it waits even longer to try to find that number of engines .


def memoize(func):
    """ Memoization decorator for a function taking one or more arguments. """
    class Memodict(dict):
        """ just a dict"""
        def __getitem__(self, *key):
            return dict.__getitem__(self, key)

        def __missing__(self, key):
            """ this makes it faster """
            ret = self[key] = func(*key)
            return ret

    return Memodict().__getitem__

def
memoize
(
func
)
:
class
Memodict
(
dict
)
:
""" just a dict"""
def
__getitem__
(
self
,
*
key
)
:
return
dict
.
__getitem__
(
self
,
key
)
def
__missing__
(
self
,
key
)
:
""" this makes it faster """
ret
=
self
[
key
]
=
func
(
*
key
)
return
ret
return
Memodict
(
)
.
__getitem__

Memoization decorator for a function taking one or more arguments .


def ambigcutters(seq):
    """
    Returns both resolutions of a cut site that has an ambiguous base in
    it, else the single cut site
    """
    resos = []
    if any([i in list("RKSYWM") for i in seq]):
        for base in list("RKSYWM"):
            if base in seq:
                resos.append(seq.replace(base, AMBIGS[base][0]))
                resos.append(seq.replace(base, AMBIGS[base][1]))
        return resos
    else:
        return [seq, ""]

def ambigcutters ( seq ) : resos = [ ] if any ( [ i in list ( "RKSYWM" ) for i in seq ] ) : for base in list ( "RKSYWM" ) : if base in seq : resos . append ( seq . replace ( base , AMBIGS [ base ] [ 0 ] ) ) resos . append ( seq . replace ( base , AMBIGS [ base ] [ 1 ] ) ) return resos else : return [ seq , "" ]

Returns both resolutions of a cut site that has an ambiguous base in it else the single cut site


def splitalleles(consensus):
    """ takes diploid consensus alleles with phase data stored as a mixture
    of upper and lower case characters and splits it into 2 alleles """

    ## store two alleles, allele1 will start with bigbase
    allele1 = list(consensus)
    allele2 = list(consensus)
    hidx = [i for (i, j) in enumerate(consensus) if j in "RKSWYMrkswym"]

    ## do remaining h sites
    for idx in hidx:
        hsite = consensus[idx]
        if hsite.isupper():
            allele1[idx] = PRIORITY[hsite]
            allele2[idx] = MINOR[hsite]
        else:
            allele1[idx] = MINOR[hsite.upper()]
            allele2[idx] = PRIORITY[hsite.upper()]

    ## convert back to strings
    allele1 = "".join(allele1)
    allele2 = "".join(allele2)

    return allele1, allele2


def comp(seq):
    """ returns a seq with complement. Preserves little n's for splitters."""
    ## makes base to its small complement then makes upper
    return seq.replace("A", 't')\
              .replace('T', 'a')\
              .replace('C', 'g')\
              .replace('G', 'c')\
              .replace('n', 'Z')\
              .upper()\
              .replace("Z", "n")

returns a seq with complement . Preserves little n s for splitters .


def fullcomp(seq):
    """ returns complement of sequence including ambiguity characters,
    and saves lower case info for multiple hetero sequences"""
    ## this is surely not the most efficient...
    seq = seq.replace("A", 'u')\
             .replace('T', 'v')\
             .replace('C', 'p')\
             .replace('G', 'z')\
             .replace('u', 'T')\
             .replace('v', 'A')\
             .replace('p', 'G')\
             .replace('z', 'C')

    ## No complement for S & W b/c complements are S & W, respectively
    seq = seq.replace('R', 'u')\
             .replace('K', 'v')\
             .replace('Y', 'b')\
             .replace('M', 'o')\
             .replace('u', 'Y')\
             .replace('v', 'M')\
             .replace('b', 'R')\
             .replace('o', 'K')

    seq = seq.replace('r', 'u')\
             .replace('k', 'v')\
             .replace('y', 'b')\
             .replace('m', 'o')\
             .replace('u', 'y')\
             .replace('v', 'm')\
             .replace('b', 'r')\
             .replace('o', 'k')
    return seq

returns complement of sequence including ambiguity characters and saves lower case info for multiple hetero sequences


def fastq_touchup_for_vsearch_merge(read, outfile, reverse=False):
    """ option to change orientation of reads and sets Qscore to B """

    counts = 0
    with open(outfile, 'w') as out:
        ## read in paired end read files 4 lines at a time
        if read.endswith(".gz"):
            fr1 = gzip.open(read, 'rb')
        else:
            fr1 = open(read, 'rb')
        quarts = itertools.izip(*[iter(fr1)]*4)

        ## a list to store until writing
        writing = []

        while 1:
            try:
                lines = quarts.next()
            except StopIteration:
                break
            if reverse:
                seq = lines[1].strip()[::-1]
            else:
                seq = lines[1].strip()
            writing.append("".join([
                lines[0],
                seq+"\n",
                lines[2],
                "B"*len(seq)
            ]))

            ## write to disk
            counts += 1
            if not counts % 1000:
                out.write("\n".join(writing)+"\n")
                writing = []
        if writing:
            out.write("\n".join(writing))

    out.close()
    fr1.close()


def revcomp(sequence):
    "returns reverse complement of a string"
    sequence = sequence[::-1].strip()\
                             .replace("A", "t")\
                             .replace("T", "a")\
                             .replace("C", "g")\
                             .replace("G", "c").upper()
    return sequence

def revcomp ( sequence ) : sequence = sequence [ : : - 1 ] . strip ( ) . replace ( "A" , "t" ) . replace ( "T" , "a" ) . replace ( "C" , "g" ) . replace ( "G" , "c" ) . upper ( ) return sequence


def clustdealer(pairdealer, optim):
    """ return optim clusters given iterators, and whether it got all or not"""
    ccnt = 0
    chunk = []
    while ccnt < optim:
        ## try refreshing taker, else quit
        try:
            taker = itertools.takewhile(lambda x: x[0] != "//\n", pairdealer)
            oneclust = ["".join(taker.next())]
        except StopIteration:
            #LOGGER.debug('last chunk %s', chunk)
            return 1, chunk

        ## load one cluster
        while 1:
            try:
                oneclust.append("".join(taker.next()))
            except StopIteration:
                break
        chunk.append("".join(oneclust))
        ccnt += 1
    return 0, chunk

def
clustdealer
(
pairdealer
,
optim
)
:
ccnt
=
0
chunk
=
[
]
while
ccnt
<
optim
:
## try refreshing taker, else quit
try
:
taker
=
itertools
.
takewhile
(
lambda
x
:
x
[
0
]
!=
"//\n"
,
pairdealer
)
oneclust
=
[
""
.
join
(
taker
.
next
(
)
)
]
except
StopIteration
:
#LOGGER.debug('last chunk %s', chunk)
return
1
,
chunk
## load one cluster
while
1
:
try
:
oneclust
.
append
(
""
.
join
(
taker
.
next
(
)
)
)
except
StopIteration
:
break
chunk
.
append
(
""
.
join
(
oneclust
)
)
ccnt
+=
1
return
0
,
chunk

return optim clusters given iterators and whether it got all or not


def progressbar(njobs, finished, msg="", spacer="  "):
    """ prints a progress bar """
    if njobs:
        progress = 100*(finished / float(njobs))
    else:
        progress = 100

    hashes = '#'*int(progress/5.)
    nohash = ' '*int(20-len(hashes))
    if not ipyrad.__interactive__:
        msg = msg.rsplit("|", 2)[0]

    args = [spacer, hashes+nohash, int(progress), msg]
    print("\r{}[{}] {:>3}% {} ".format(*args), end="")
    sys.stdout.flush()

def
progressbar
(
njobs
,
finished
,
msg
=
""
,
spacer
=
"  "
)
:
if
njobs
:
progress
=
100
*
(
finished
/
float
(
njobs
)
)
else
:
progress
=
100
hashes
=
'#'
*
int
(
progress
/
5.
)
nohash
=
' '
*
int
(
20
len
(
hashes
)
)
if
not
ipyrad
.
__interactive__
:
msg
=
msg
.
rsplit
(
"|"
,
2
)
[
0
]
args
=
[
spacer
,
hashes
+
nohash
,
int
(
progress
)
,
msg
]
print
(
"\r{}[{}] {:>3}% {} "
.
format
(
*
args
)
,
end
=
""
)
sys
.
stdout
.
flush
(
)


def get_threaded_view(ipyclient, split=True):
    """ gets optimum threaded view of ids given the host setup """
    ## engine ids
    ## e.g., [0, 1, 2, 3, 4, 5, 6, 7, 8]
    eids = ipyclient.ids

    ## get host names
    ## e.g., ['a', 'a', 'b', 'b', 'a', 'c', 'c', 'c', 'c']
    dview = ipyclient.direct_view()
    hosts = dview.apply_sync(socket.gethostname)

    ## group ids into a dict by their hostnames
    ## e.g., {a: [0, 1, 4], b: [2, 3], c: [5, 6, 7, 8]}
    hostdict = defaultdict(list)
    for host, eid in zip(hosts, eids):
        hostdict[host].append(eid)

    ## Now split threads on the same host into separate proc if there are many
    hostdictkeys = hostdict.keys()
    for key in hostdictkeys:
        gids = hostdict[key]
        maxt = len(gids)
        if len(gids) >= 4:
            maxt = 2
        ## if 4 nodes and 4 ppn, put one sample per host
        if (len(gids) == 4) and (len(hosts) >= 4):
            maxt = 4
        if len(gids) >= 6:
            maxt = 3
        if len(gids) >= 8:
            maxt = 4
        if len(gids) >= 16:
            maxt = 4
        ## split ids into groups of maxt
        threaded = [gids[i:i+maxt] for i in xrange(0, len(gids), maxt)]
        lth = len(threaded)
        ## if anything was split (lth>1) update hostdict with new proc
        if lth > 1:
            hostdict.pop(key)
            for hostid in range(lth):
                hostdict[str(key)+"_"+str(hostid)] = threaded[hostid]

    ## make sure split numbering is correct
    #threaded = hostdict.values()
    #assert len(ipyclient.ids) <= len(list(itertools.chain(*threaded)))
    LOGGER.info("threaded_view: %s", dict(hostdict))
    return hostdict


def detect_cpus():
    """
    Detects the number of CPUs on a system. This is better than asking
    ipyparallel since ipp has to wait for Engines to spin up.
    """
    # Linux, Unix and MacOS:
    if hasattr(os, "sysconf"):
        if os.sysconf_names.has_key("SC_NPROCESSORS_ONLN"):
            # Linux & Unix:
            ncpus = os.sysconf("SC_NPROCESSORS_ONLN")
            if isinstance(ncpus, int) and ncpus > 0:
                return ncpus
        else: # OSX:
            return int(os.popen2("sysctl -n hw.ncpu")[1].read())
    # Windows:
    if os.environ.has_key("NUMBER_OF_PROCESSORS"):
        ncpus = int(os.environ["NUMBER_OF_PROCESSORS"])
        if ncpus > 0:
            return ncpus
    return 1

def
detect_cpus
(
)
:
# Linux, Unix and MacOS:
if
hasattr
(
os
,
"sysconf"
)
:
if
os
.
sysconf_names
.
has_key
(
"SC_NPROCESSORS_ONLN"
)
:
# Linux & Unix:
ncpus
=
os
.
sysconf
(
"SC_NPROCESSORS_ONLN"
)
if
isinstance
(
ncpus
,
int
)
and
ncpus
>
0
:
return
ncpus
else
:
# OSX:
return
int
(
os
.
popen2
(
"sysctl -n hw.ncpu"
)
[
1
]
.
read
(
)
)
# Windows:
if
os
.
environ
.
has_key
(
"NUMBER_OF_PROCESSORS"
)
:
ncpus
=
int
(
os
.
environ
[
"NUMBER_OF_PROCESSORS"
]
)
if
ncpus
>
0
:
return
ncpus
return
1

Detects the number of CPUs on a system . This is better than asking ipyparallel since ipp has to wait for Engines to spin up .


def _call_structure(mname, ename, sname, name, workdir, seed, ntaxa, nsites, kpop, rep):
    """ make the subprocess call to structure """
    ## create call string
    outname = os.path.join(workdir, "{}-K-{}-rep-{}".format(name, kpop, rep))

    cmd = ["structure",
           "-m", mname,
           "-e", ename,
           "-K", str(kpop),
           "-D", str(seed),
           "-N", str(ntaxa),
           "-L", str(nsites),
           "-i", sname,
           "-o", outname]

    ## call the shell function
    proc = subprocess.Popen(cmd,
                            stdout=subprocess.PIPE,
                            stderr=subprocess.STDOUT)
    comm = proc.communicate()

    ## cleanup
    oldfiles = [mname, ename, sname]
    for oldfile in oldfiles:
        if os.path.exists(oldfile):
            os.remove(oldfile)
    return comm

def
_call_structure
(
mname
,
ename
,
sname
,
name
,
workdir
,
seed
,
ntaxa
,
nsites
,
kpop
,
rep
)
:
## create call string
outname
=
os
.
path
.
join
(
workdir
,
"{}-K-{}-rep-{}"
.
format
(
name
,
kpop
,
rep
)
)
cmd
=
[
"structure"
,
"-m"
,
mname
,
"-e"
,
ename
,
"-K"
,
str
(
kpop
)
,
"-D"
,
str
(
seed
)
,
"-N"
,
str
(
ntaxa
)
,
"-L"
,
str
(
nsites
)
,
"-i"
,
sname
,
"-o"
,
outname
]
## call the shell function
proc
=
subprocess
.
Popen
(
cmd
,
stdout
=
subprocess
.
PIPE
,
stderr
=
subprocess
.
STDOUT
)
comm
=
proc
.
communicate
(
)
## cleanup
oldfiles
=
[
mname
,
ename
,
sname
]
for
oldfile
in
oldfiles
:
if
os
.
path
.
exists
(
oldfile
)
:
os
.
remove
(
oldfile
)
return
comm


def _get_clumpp_table(self, kpop, max_var_multiple, quiet):
    """ private function to clumpp results"""

    ## concat results for k=x
    reps, excluded = _concat_reps(self, kpop, max_var_multiple, quiet)
    if reps:
        ninds = reps[0].inds
        nreps = len(reps)
    else:
        ninds = nreps = 0
    if not reps:
        return "no result files found"

    clumphandle = os.path.join(self.workdir, "tmp.clumppparams.txt")
    self.clumppparams.kpop = kpop
    self.clumppparams.c = ninds
    self.clumppparams.r = nreps
    with open(clumphandle, 'w') as tmp_c:
        tmp_c.write(self.clumppparams._asfile())

    ## create CLUMPP args string
    outfile = os.path.join(self.workdir,
                "{}-K-{}.outfile".format(self.name, kpop))
    indfile = os.path.join(self.workdir,
                "{}-K-{}.indfile".format(self.name, kpop))
    miscfile = os.path.join(self.workdir,
                "{}-K-{}.miscfile".format(self.name, kpop))
    cmd = ["CLUMPP", clumphandle,
           "-i", indfile,
           "-o", outfile,
           "-j", miscfile,
           "-r", str(nreps),
           "-c", str(ninds),
           "-k", str(kpop)]

    ## call clumpp
    proc = subprocess.Popen(cmd,
                            stderr=subprocess.STDOUT,
                            stdout=subprocess.PIPE)
    _ = proc.communicate()

    ## cleanup
    for rfile in [indfile, miscfile]:
        if os.path.exists(rfile):
            os.remove(rfile)

    ## parse clumpp results file
    ofile = os.path.join(self.workdir, "{}-K-{}.outfile".format(self.name, kpop))
    if os.path.exists(ofile):
        csvtable = pd.read_csv(ofile, delim_whitespace=True, header=None)
        table = csvtable.loc[:, 5:]

        ## apply names to cols and rows
        table.columns = range(table.shape[1])
        table.index = self.labels
        if not quiet:
            sys.stderr.write(
                "[K{}] {}/{} results permuted across replicates (max_var={}).\n"\
                .format(kpop, nreps, nreps+excluded, max_var_multiple))
        return table

    else:
        sys.stderr.write("No files ready for {}-K-{} in {}\n"\
                         .format(self.name, kpop, self.workdir))
        return


def _get_evanno_table(self, kpops, max_var_multiple, quiet):
    """
    Calculates Evanno method K value scores for a series
    of permuted clumpp results.
    """

    ## iterate across k-vals
    kpops = sorted(kpops)
    replnliks = []

    for kpop in kpops:
        ## concat results for k=x
        reps, excluded = _concat_reps(self, kpop, max_var_multiple, quiet)

        ## report if some results were excluded
        if excluded:
            if not quiet:
                sys.stderr.write(
                "[K{}] {} reps excluded (not converged) see 'max_var_multiple'.\n"\
                .format(kpop, excluded))

        if reps:
            ninds = reps[0].inds
            nreps = len(reps)
        else:
            ninds = nreps = 0
        if not reps:
            print "no result files found"

        ## all we really need is the lnlik
        replnliks.append([i.est_lnlik for i in reps])

    ## compare lnlik and var of results
    if len(replnliks) > 1:
        lnmean = [np.mean(i) for i in replnliks]
        lnstds = [np.std(i, ddof=1) for i in replnliks]
    else:
        lnmean = replnliks
        lnstds = np.nan

    tab = pd.DataFrame(
        index=kpops,
        data={
            "Nreps": [len(i) for i in replnliks],
            "lnPK": [0] * len(kpops),
            "lnPPK": [0] * len(kpops),
            "deltaK": [0] * len(kpops),
            "estLnProbMean": lnmean,
            "estLnProbStdev": lnstds,
        }
        )

    ## calculate Evanno's
    for kpop in kpops[1:]:
        tab.loc[kpop, "lnPK"] = tab.loc[kpop, "estLnProbMean"] \
                              - tab.loc[kpop-1, "estLnProbMean"]

    for kpop in kpops[1:-1]:
        tab.loc[kpop, "lnPPK"] = abs(tab.loc[kpop+1, "lnPK"]
                                     - tab.loc[kpop, "lnPK"])
        tab.loc[kpop, "deltaK"] = (abs(
                                    tab.loc[kpop+1, "estLnProbMean"] - \
                                    2.0 * tab.loc[kpop, "estLnProbMean"] + \
                                    tab.loc[kpop-1, "estLnProbMean"]) / \
                                   tab.loc[kpop, "estLnProbStdev"])

    ## return table
    return tab

Calculates Evanno method K value scores for a series of permuted clumpp results .


def result_files(self):
        """ returns a list of files that have finished structure """
        reps = OPJ(self.workdir, self.name+"-K-*-rep-*_f")
        repfiles = glob.glob(reps)
        return repfiles

def result_files ( self ) : reps = OPJ ( self . workdir , self . name + "-K-*-rep-*_f" ) repfiles = glob . glob ( reps ) return repfiles


def get_evanno_table(self, kvalues, max_var_multiple=0, quiet=False):
        """
        Calculates the Evanno table from results files for tests with
        K-values in the input list kvalues. The values lnPK, lnPPK,
        and deltaK are calculated. The max_var_multiplier arg can be used
        to exclude results files based on variance of the likelihood as a
        proxy for convergence.

        Parameters:
        -----------
        kvalues : list
            The list of K-values for which structure was run for this object.
            e.g., kvalues = [3, 4, 5]

        max_var_multiple: int
            A multiplier value to use as a filter for convergence of runs.
            Default=0=no filtering. As an example, if 10 replicates
            were run then the variance of the run with the minimum variance is
            used as a benchmark. If other runs have a variance that is N times
            greater then that run will be excluded. Remember, if replicate runs
            sampled different distributions of SNPs then it is not unexpected that
            they will have very different variances. However, you may still want
            to exclude runs with very high variance since they likely have
            not converged.

        quiet: bool
            Suppresses printed messages about convergence.

        Returns:
        --------
        table : pandas.DataFrame
            A data frame with LPK, LNPPK, and delta K. The latter is typically
            used to find the best fitting value of K. But be wary of over
            interpreting a single best K value.
        """
        ## do not allow bad vals
        if max_var_multiple:
            if max_var_multiple < 1:
                raise ValueError('max_variance_multiplier must be >1')

        table = _get_evanno_table(self, kvalues, max_var_multiple, quiet)
        return table

def
get_evanno_table
(
self
,
kvalues
,
max_var_multiple
=
0
,
quiet
=
False
)
:
## do not allow bad vals
if
max_var_multiple
:
if
max_var_multiple
<
1
:
raise
ValueError
(
'max_variance_multiplier must be >1'
)
table
=
_get_evanno_table
(
self
,
kvalues
,
max_var_multiple
,
quiet
)
return
table

Calculates the Evanno table from results files for tests with K - values in the input list kvalues . The values lnPK lnPPK and deltaK are calculated . The max_var_multiplier arg can be used to exclude results files based on variance of the likelihood as a proxy for convergence .


def parse(self, psearch, dsearch):
        """ parse an _f structure output file """
        stable = ""
        with open(self.repfile) as orep:
            dat = orep.readlines()
            for line in dat:
                ## stat lines
                if "Estimated Ln Prob of Data" in line:
                    self.est_lnlik = float(line.split()[-1])
                if "Mean value of ln likelihood" in line:
                    self.mean_lnlik = float(line.split()[-1])
                if "Variance of ln likelihood" in line:
                    self.var_lnlik = float(line.split()[-1])
                if "Mean value of alpha" in line:
                    self.alpha = float(line.split()[-1])

                ## matrix lines
                nonline = psearch.search(line)
                popline = dsearch.search(line)

                #if ")   :  " in line:
                if nonline:
                    ## check if sample is supervised...
                    abc = line.strip().split()
                    outstr = "{}{}{}".format(
                        " ".join([abc[0], abc[0], abc[2],
                                  abc[0].split('.')[0]]),
                        " :  ",
                        " ".join(abc[4:])
                    )
                    self.inds += 1
                    stable += outstr+"\n"

                elif popline:
                    ## check if sample is supervised...
                    abc = line.strip().split()
                    prop = ["0.000"] * self.kpop
                    pidx = int(abc[3]) - 1
                    prop[pidx] = "1.000"
                    outstr = "{}{}{}".format(
                        " ".join([abc[0], abc[0], abc[2],
                                  abc[0].split('.')[0]]),
                        " :  ",
                        " ".join(prop)
                    )
                    self.inds += 1
                    stable += outstr+"\n"

            stable += "\n"
        return stable

def
parse
(
self
,
psearch
,
dsearch
)
:
stable
=
""
with
open
(
self
.
repfile
)
as
orep
:
dat
=
orep
.
readlines
(
)
for
line
in
dat
:
## stat lines
if
"Estimated Ln Prob of Data"
in
line
:
self
.
est_lnlik
=
float
(
line
.
split
(
)
[
1
]
)
if
"Mean value of ln likelihood"
in
line
:
self
.
mean_lnlik
=
float
(
line
.
split
(
)
[
1
]
)
if
"Variance of ln likelihood"
in
line
:
self
.
var_lnlik
=
float
(
line
.
split
(
)
[
1
]
)
if
"Mean value of alpha"
in
line
:
self
.
alpha
=
float
(
line
.
split
(
)
[
1
]
)
## matrix lines
nonline
=
psearch
.
search
(
line
)
popline
=
dsearch
.
search
(
line
)
#if ")   :  " in line:
if
nonline
:
## check if sample is supervised...
abc
=
line
.
strip
(
)
.
split
(
)
outstr
=
"{}{}{}"
.
format
(
" "
.
join
(
[
abc
[
0
]
,
abc
[
0
]
,
abc
[
2
]
,
abc
[
0
]
.
split
(
'.'
)
[
0
]
]
)
,
" :  "
,
" "
.
join
(
abc
[
4
:
]
)
)
self
.
inds
+=
1
stable
+=
outstr
+
"\n"
elif
popline
:
## check if sample is supervised...
abc
=
line
.
strip
(
)
.
split
(
)
prop
=
[
"0.000"
]
*
self
.
kpop
pidx
=
int
(
abc
[
3
]
)
1
prop
[
pidx
]
=
"1.000"
outstr
=
"{}{}{}"
.
format
(
" "
.
join
(
[
abc
[
0
]
,
abc
[
0
]
,
abc
[
2
]
,
abc
[
0
]
.
split
(
'.'
)
[
0
]
]
)
,
" :  "
,
" "
.
join
(
prop
)
)
self
.
inds
+=
1
stable
+=
outstr
+
"\n"
stable
+=
"\n"
return
stable


def _call_raxml(command_list):
    """ call the command as sps """
    proc = subprocess.Popen(
        command_list,
        stderr=subprocess.STDOUT,
        stdout=subprocess.PIPE
        )
    comm = proc.communicate()
    return comm

def _call_raxml ( command_list ) : proc = subprocess . Popen ( command_list , stderr = subprocess . STDOUT , stdout = subprocess . PIPE ) comm = proc . communicate ( ) return comm


def run(self,
        ipyclient=None,
        quiet=False,
        force=False,
        block=False,
        ):
        """
        Submits raxml job to run. If no ipyclient object is provided then
        the function will block until the raxml run is finished. If an ipyclient
        is provided then the job is sent to a remote engine and an asynchronous
        result object is returned which can be queried or awaited until it finishes.

        Parameters
        -----------
        ipyclient:
            Not yet supported...
        quiet:
            suppress print statements
        force:
            overwrite existing results files with this job name.
        block:
            will block progress in notebook until job finishes, even if job
            is running on a remote ipyclient.
        """

        ## stop before trying in raxml
        if force:
            for key, oldfile in self.trees:
                if os.path.exists(oldfile):
                    os.remove(oldfile)
        if os.path.exists(self.trees.info):
            print("Error: set a new name for this job or use Force flag.\nFile exists: {}"\
                  .format(self.trees.info))
            return

        ## TODO: add a progress bar tracker here. It could even read it from
        ## the info file that is being written.
        ## submit it
        if not ipyclient:
            proc = _call_raxml(self._command_list)
            self.stdout = proc[0]
            self.stderr = proc[1]
        else:
            ## find all hosts and submit job to the host with most available engines
            lbview = ipyclient.load_balanced_view()
            self.async = lbview.apply(_call_raxml, self._command_list)

        ## initiate random seed
        if not quiet:
            if not ipyclient:
                ## look for errors
                if "Overall execution time" not in self.stdout:
                    print("Error in raxml run\n" + self.stdout)
                else:
                    print("job {} finished successfully".format(self.params.n))
            else:
                print("job {} submitted to cluster".format(self.params.n))

Submits raxml job to run . If no ipyclient object is provided then the function will block until the raxml run is finished . If an ipyclient is provided then the job is sent to a remote engine and an asynchronous result object is returned which can be queried or awaited until it finishes .


def _get_binary(self):
        """ find binaries available"""

        ## check for binary
        backup_binaries = ["raxmlHPC-PTHREADS", "raxmlHPC-PTHREADS-SSE3"]

        ## check user binary first, then backups
        for binary in [self.params.binary] + backup_binaries:
            proc = subprocess.Popen(["which", self.params.binary],
                    stdout=subprocess.PIPE,
                    stderr=subprocess.STDOUT).communicate()
            ## update the binary
            if proc:
                self.params.binary = binary

        ## if none then raise error
        if not proc[0]:
            raise Exception(BINARY_ERROR.format(self.params.binary))


def dstat(inarr, taxdict, mindict=1, nboots=1000, name=0):
    """ private function to perform a single D-stat test"""

    #if isinstance(inarr, str):
    #    with open(inarr, 'r') as infile:
    #        inarr = infile.read().strip().split("|\n")

    # ## get data as an array from loci file
    # ## if loci-list then parse arr from loci
    if isinstance(inarr, list):
        arr, _ = _loci_to_arr(inarr, taxdict, mindict)

    # ## if it's an array already then go ahead
    # elif isinstance(inarr, np.ndarray):
    #     arr = inarr
    # ## if it's a simulation object get freqs from array
    # elif isinstance(inarr, Sim):
    #     arr = _msp_to_arr(inarr, taxdict)

    #elif isinstance(inarr, types.GeneratorType):
    #    arr = _msp_to_arr(inarr, taxdict)
    #elif isinstance(inarr, list):
    #    arr = _msp_to_arr(inarr, taxdict)
    ## get data from Sim object, do not digest the ms generator
    #else:
    #    raise Exception("Must enter either a 'locifile' or 'arr'")

    ## run tests
    #if len(taxdict) == 4:
    if arr.shape[1] == 4:

        ## get results
        res, boots = _get_signif_4(arr, nboots)

        ## make res into a nice DataFrame
        res = pd.DataFrame(res,
            columns=[name],
            index=["Dstat", "bootmean", "bootstd", "Z", "ABBA", "BABA", "nloci"])

    else:
        ## get results
        res, boots = _get_signif_5(arr, nboots)
         ## make int a DataFrame
        res = pd.DataFrame(res,
            index=["p3", "p4", "shared"],
            columns=["Dstat", "bootmean", "bootstd", "Z", "ABxxA", "BAxxA", "nloci"]
            )

    return res.T, boots

private function to perform a single D - stat test


def _get_boots(arr, nboots):
    """
    return array of bootstrap D-stats
    """
    ## hold results (nboots, [dstat, ])
    boots = np.zeros((nboots,))

    ## iterate to fill boots
    for bidx in xrange(nboots):
        ## sample with replacement
        lidx = np.random.randint(0, arr.shape[0], arr.shape[0])
        tarr = arr[lidx]
        _, _, dst = _prop_dstat(tarr)
        boots[bidx] = dst

    ## return bootarr
    return boots

def
_get_boots
(
arr
,
nboots
)
:
## hold results (nboots, [dstat, ])
boots
=
np
.
zeros
(
(
nboots
,
)
)
## iterate to fill boots
for
bidx
in
xrange
(
nboots
)
:
## sample with replacement
lidx
=
np
.
random
.
randint
(
0
,
arr
.
shape
[
0
]
,
arr
.
shape
[
0
]
)
tarr
=
arr
[
lidx
]
_
,
_
,
dst
=
_prop_dstat
(
tarr
)
boots
[
bidx
]
=
dst
## return bootarr
return
boots


def taxon_table(self):
        """
        Returns the .tests list of taxa as a pandas dataframe.
        By auto-generating this table from tests it means that
        the table itself cannot be modified unless it is returned
        and saved.
        """
        if self.tests:
            keys = sorted(self.tests[0].keys())
            if isinstance(self.tests, list):
                ld = [[(key, i[key]) for key in keys] for i in self.tests]
                dd = [dict(i) for i in ld]
                df = pd.DataFrame(dd)
                return df
            else:
                return pd.DataFrame(pd.Series(self.tests)).T
        else:
            return None

def taxon_table ( self ) : if self . tests : keys = sorted ( self . tests [ 0 ] . keys ( ) ) if isinstance ( self . tests , list ) : ld = [ [ ( key , i [ key ] ) for key in keys ] for i in self . tests ] dd = [ dict ( i ) for i in ld ] df = pd . DataFrame ( dd ) return df else : return pd . DataFrame ( pd . Series ( self . tests ) ) . T else : return None

Returns the . tests list of taxa as a pandas dataframe . By auto - generating this table from tests it means that the table itself cannot be modified unless it is returned and saved .


def nexmake(mdict, nlocus, dirs, mcmc_burnin, mcmc_ngen, mcmc_sample_freq):
    """
    function that takes a dictionary mapping names to
    sequences, and a locus number, and writes it as a NEXUS
    file with a mrbayes analysis block.
    """
    ## create matrix as a string
    max_name_len = max([len(i) for i in mdict])
    namestring = "{:<" + str(max_name_len+1) + "} {}\n"
    matrix = ""
    for i in mdict.items():
        matrix += namestring.format(i[0], i[1])

    ## write nexus block
    handle = os.path.join(dirs, "{}.nex".format(nlocus))
    with open(handle, 'w') as outnex:
        outnex.write(NEXBLOCK.format(**{
            "ntax": len(mdict),
            "nchar": len(mdict.values()[0]),
            "matrix": matrix,
            "ngen": mcmc_ngen,
            "sfreq": mcmc_sample_freq,
            "burnin": mcmc_burnin,
            }))

def
nexmake
(
mdict
,
nlocus
,
dirs
,
mcmc_burnin
,
mcmc_ngen
,
mcmc_sample_freq
)
:
## create matrix as a string
max_name_len
=
max
(
[
len
(
i
)
for
i
in
mdict
]
)
namestring
=
"{:<"
+
str
(
max_name_len
+
1
)
+
"} {}\n"
matrix
=
""
for
i
in
mdict
.
items
(
)
:
matrix
+=
namestring
.
format
(
i
[
0
]
,
i
[
1
]
)
## write nexus block
handle
=
os
.
path
.
join
(
dirs
,
"{}.nex"
.
format
(
nlocus
)
)
with
open
(
handle
,
'w'
)
as
outnex
:
outnex
.
write
(
NEXBLOCK
.
format
(
*
*
{
"ntax"
:
len
(
mdict
)
,
"nchar"
:
len
(
mdict
.
values
(
)
[
0
]
)
,
"matrix"
:
matrix
,
"ngen"
:
mcmc_ngen
,
"sfreq"
:
mcmc_sample_freq
,
"burnin"
:
mcmc_burnin
,
}
)
)

function that takes a dictionary mapping names to sequences and a locus number and writes it as a NEXUS file with a mrbayes analysis block .


def call_fastq_dump_on_SRRs(self, srr, outname, paired):
    """
    calls fastq-dump on SRRs, relabels fastqs by their accession
    names, and writes them to the workdir. Saves temp sra files
    in the designated tmp folder and immediately removes them.
    """

    ## build command for fastq-dumping
    fd_cmd = [
        "fastq-dump", srr,
        "--accession", outname,
        "--outdir", self.workdir,
        "--gzip",
        ]
    if paired:
        fd_cmd += ["--split-files"]

    ## call fq dump command
    proc = sps.Popen(fd_cmd, stderr=sps.STDOUT, stdout=sps.PIPE)
    o, e = proc.communicate()

    ## delete the stupid temp sra file from the place
    ## that it is very hard-coded to be written to, and
    ## LEFT IN, for some crazy reason.
    srafile = os.path.join(self.workdir, "sra", srr+".sra")
    if os.path.exists(srafile):
        os.remove(srafile)

calls fastq - dump on SRRs relabels fastqs by their accession names and writes them to the workdir . Saves temp sra files in the designated tmp folder and immediately removes them .


def fields_checker(fields):
    """
    returns a fields argument formatted as a list of strings.
    and doesn't allow zero.
    """
    ## make sure fields will work
    if isinstance(fields, int):
        fields = str(fields)
    if isinstance(fields, str):
        if "," in fields:
            fields = [str(i) for i in fields.split(",")]
        else:
            fields = [str(fields)]
    elif isinstance(fields, (tuple, list)):
        fields = [str(i) for i in fields]
    else:
        raise IPyradWarningExit("fields not properly formatted")

    ## do not allow zero in fields
    fields = [i for i in fields if i != '0']

    return fields

def
fields_checker
(
fields
)
:
## make sure fields will work
if
isinstance
(
fields
,
int
)
:
fields
=
str
(
fields
)
if
isinstance
(
fields
,
str
)
:
if
","
in
fields
:
fields
=
[
str
(
i
)
for
i
in
fields
.
split
(
","
)
]
else
:
fields
=
[
str
(
fields
)
]
elif
isinstance
(
fields
,
(
tuple
,
list
)
)
:
fields
=
[
str
(
i
)
for
i
in
fields
]
else
:
raise
IPyradWarningExit
(
"fields not properly formatted"
)
## do not allow zero in fields
fields
=
[
i
for
i
in
fields
if
i
!=
'0'
]
return
fields

returns a fields argument formatted as a list of strings . and doesn t allow zero .


def run(self,
        force=False,
        ipyclient=None,
        name_fields=30,
        name_separator="_",
        dry_run=False):
        """
        Download the accessions into a the designated workdir.

        Parameters
        ----------
        force: (bool)
            If force=True then existing files with the same name
            will be overwritten.

        ipyclient: (ipyparallel.Client)
            If provided, work will be distributed across a parallel
            client, otherwise download will be run on a single core.

        name_fields: (int, str):
            Provide the index of the name fields to be used as a prefix
            for fastq output files. The default is 30, which is the
            SampleName field. Use sra.fetch_fields to see all available
            fields and their indices. A likely alternative is 1 (Run).
            If multiple are listed then they will be joined by a "_"
            character. For example (29,30) would yield something like:
            latin-name_sample-name (e.g., mus_musculus-NR10123).

        dry_run: (bool)
            If True then a table of file names that _would_ be downloaded
            will be shown, but the actual files will note be downloaded.
        """

        ## temporarily set directory for tmpfiles used by fastq-dump
        ## if this fails then just skip it.
        try:
            ## ensure output directory, also used as tmpdir
            if not os.path.exists(self.workdir):
                os.makedirs(self.workdir)

            ## get original directory for sra files
            ## probably /home/ncbi/public/sra by default.
            self._set_vdbconfig_path()

            ## register ipyclient for cleanup
            if ipyclient:
                self._ipcluster["pids"] = {}
                for eid in ipyclient.ids:
                    engine = ipyclient[eid]
                    if not engine.outstanding:
                        pid = engine.apply(os.getpid).get()
                        self._ipcluster["pids"][eid] = pid

            ## submit jobs to engines or local
            self._submit_jobs(
                force=force,
                ipyclient=ipyclient,
                name_fields=name_fields,
                name_separator=name_separator,
                dry_run=dry_run,
                )

        except IPyradWarningExit as inst:
            print(inst)
        ## exceptions to catch, cleanup and handle ipyclient interrupts
        except KeyboardInterrupt:
            print("keyboard interrupt...")
        except Exception as inst:
            print("Exception in run() - {}".format(inst))
        finally:
            ## reset working sra path
            self._restore_vdbconfig_path()

            ## if it made a new sra directory then it should be empty when
            ## we are finished if all .sra files were removed. If so, then
            ## let's also remove the dir. if not empty, leave it.
            sradir = os.path.join(self.workdir, "sra")
            if os.path.exists(sradir) and (not os.listdir(sradir)):
                shutil.rmtree(sradir)
            else:
                ## print warning
                try:
                    print(FAILED_DOWNLOAD.format(os.listdir(sradir)))
                except OSError as inst:
                    ## If sra dir doesn't even exist something very bad is broken.
                    raise IPyradWarningExit("Download failed. Exiting.")
                ## remove fastq file matching to cached sra file
                for srr in os.listdir(sradir):
                    isrr = srr.split(".")[0]
                    ipath = os.path.join(self.workdir, "*_{}*.gz".format(isrr))
                    ifile = glob.glob(ipath)[0]
                    if os.path.exists(ifile):
                        os.remove(ifile)
                ## remove cache of sra files
                shutil.rmtree(sradir)

            ## cleanup ipcluster shutdown
            if ipyclient:
                ## send SIGINT (2) to all engines still running tasks
                try:
                    ipyclient.abort()
                    time.sleep(0.5)
                    for engine_id, pid in self._ipcluster["pids"].items():
                        if ipyclient.queue_status()[engine_id]["tasks"]:
                            os.kill(pid, 2)
                        time.sleep(0.1)
                except ipp.NoEnginesRegistered:
                    pass
                ## clean memory space
                if not ipyclient.outstanding:
                    ipyclient.purge_everything()
                ## uh oh, kill everything, something bad happened
                else:
                    ipyclient.shutdown(hub=True, block=False)
                    ipyclient.close()
                    print("\nwarning: ipcluster shutdown and must be restarted")

Download the accessions into a the designated workdir .


def Async(cls, token, session=None, **options):
        """Returns the client in async mode."""
        return cls(token, session=session, is_async=True, **options)

def Async ( cls , token , session = None , * * options ) : return cls ( token , session = session , is_async = True , * * options )


def get_constants(self, **params: keys):
        """Get the CR Constants

        Parameters
        ----------
        \*\*keys: Optional[list] = None
            Filter which keys should be included in the
            response
        \*\*exclude: Optional[list] = None
            Filter which keys should be excluded from the
            response
        \*\*timeout: Optional[int] = None
            Custom timeout that overwrites Client.timeout
        """
        url = self.api.CONSTANTS
        return self._get_model(url, **params)

def get_constants ( self , * * params : keys ) : url = self . api . CONSTANTS return self . _get_model ( url , * * params )


def get_player(self, *tags: crtag, **params: keys):
        """Get a player information

        Parameters
        ----------
        \*tags: str
            Valid player tags. Minimum length: 3
            Valid characters: 0289PYLQGRJCUV
        \*\*keys: Optional[list] = None
            Filter which keys should be included in the
            response
        \*\*exclude: Optional[list] = None
            Filter which keys should be excluded from the
            response
        \*\*timeout: Optional[int] = None
            Custom timeout that overwrites Client.timeout
        """
        url = self.api.PLAYER + '/' + ','.join(tags)
        return self._get_model(url, FullPlayer, **params)

def get_player ( self , * tags : crtag , * * params : keys ) : url = self . api . PLAYER + '/' + ',' . join ( tags ) return self . _get_model ( url , FullPlayer , * * params )


def get_clan(self, *tags: crtag, **params: keys):
        """Get a clan information

        Parameters
        ----------
        \*tags: str
            Valid clan tags. Minimum length: 3
            Valid characters: 0289PYLQGRJCUV
        \*\*keys: Optional[list] = None
            Filter which keys should be included in the
            response
        \*\*exclude: Optional[list] = None
            Filter which keys should be excluded from the
            response
        \*\*timeout: Optional[int] = None
            Custom timeout that overwrites Client.timeout
        """
        url = self.api.CLAN + '/' + ','.join(tags)
        return self._get_model(url, FullClan, **params)

def get_clan ( self , * tags : crtag , * * params : keys ) : url = self . api . CLAN + '/' + ',' . join ( tags ) return self . _get_model ( url , FullClan , * * params )


def search_tournaments(self, **params: keys):
        """Search for a tournament

        Parameters
        ----------
        name: str
            The name of the tournament
        \*\*keys: Optional[list] = None
            Filter which keys should be included in the
            response
        \*\*exclude: Optional[list] = None
            Filter which keys should be excluded from the
            response
        \*\*max: Optional[int] = None
            Limit the number of items returned in the response
        \*\*page: Optional[int] = None
            Works with max, the zero-based page of the
            items
        \*\*timeout: Optional[int] = None
            Custom timeout that overwrites Client.timeout
        """
        url = self.api.TOURNAMENT + '/search'
        return self._get_model(url, PartialClan, **params)

def search_tournaments ( self , * * params : keys ) : url = self . api . TOURNAMENT + '/search' return self . _get_model ( url , PartialClan , * * params )


def get_top_war_clans(self, country_key='', **params: keys):
        """Get a list of top clans by war

        location_id: Optional[str] = ''
            A location ID or '' (global)
            See https://github.com/RoyaleAPI/cr-api-data/blob/master/json/regions.json
            for a list of acceptable location IDs
        \*\*keys: Optional[list] = None
            Filter which keys should be included in the
            response
        \*\*exclude: Optional[list] = None
            Filter which keys should be excluded from the
            response
        \*\*max: Optional[int] = None
            Limit the number of items returned in the response
        \*\*page: Optional[int] = None
            Works with max, the zero-based page of the
            items
        \*\*timeout: Optional[int] = None
            Custom timeout that overwrites Client.timeout
        """
        url = self.api.TOP + '/war/' + str(country_key)
        return self._get_model(url, PartialClan, **params)

def get_top_war_clans ( self , country_key = '' , * * params : keys ) : url = self . api . TOP + '/war/' + str ( country_key ) return self . _get_model ( url , PartialClan , * * params )


def get_popular_clans(self, **params: keys):
        """Get a list of most queried clans

        \*\*keys: Optional[list] = None
            Filter which keys should be included in the
            response
        \*\*exclude: Optional[list] = None
            Filter which keys should be excluded from the
            response
        \*\*max: Optional[int] = None
            Limit the number of items returned in the response
        \*\*page: Optional[int] = None
            Works with max, the zero-based page of the
            items
        \*\*timeout: Optional[int] = None
            Custom timeout that overwrites Client.timeout
        """
        url = self.api.POPULAR + '/clans'
        return self._get_model(url, PartialClan, **params)

def get_popular_clans ( self , * * params : keys ) : url = self . api . POPULAR + '/clans' return self . _get_model ( url , PartialClan , * * params )


def get_popular_players(self, **params: keys):
        """Get a list of most queried players

        \*\*keys: Optional[list] = None
            Filter which keys should be included in the
            response
        \*\*exclude: Optional[list] = None
            Filter which keys should be excluded from the
            response
        \*\*max: Optional[int] = None
            Limit the number of items returned in the response
        \*\*page: Optional[int] = None
            Works with max, the zero-based page of the
            items
        \*\*timeout: Optional[int] = None
            Custom timeout that overwrites Client.timeout
        """
        url = self.api.POPULAR + '/players'
        return self._get_model(url, PartialPlayerClan, **params)

def get_popular_players ( self , * * params : keys ) : url = self . api . POPULAR + '/players' return self . _get_model ( url , PartialPlayerClan , * * params )


def get_popular_tournaments(self, **params: keys):
        """Get a list of most queried tournaments

        \*\*keys: Optional[list] = None
            Filter which keys should be included in the
            response
        \*\*exclude: Optional[list] = None
            Filter which keys should be excluded from the
            response
        \*\*max: Optional[int] = None
            Limit the number of items returned in the response
        \*\*page: Optional[int] = None
            Works with max, the zero-based page of the
            items
        \*\*timeout: Optional[int] = None
            Custom timeout that overwrites Client.timeout
        """
        url = self.api.POPULAR + '/tournament'
        return self._get_model(url, PartialTournament, **params)

def get_popular_tournaments ( self , * * params : keys ) : url = self . api . POPULAR + '/tournament' return self . _get_model ( url , PartialTournament , * * params )


def get_popular_decks(self, **params: keys):
        """Get a list of most queried decks

        \*\*keys: Optional[list] = None
            Filter which keys should be included in the
            response
        \*\*exclude: Optional[list] = None
            Filter which keys should be excluded from the
            response
        \*\*max: Optional[int] = None
            Limit the number of items returned in the response
        \*\*page: Optional[int] = None
            Works with max, the zero-based page of the
            items
        \*\*timeout: Optional[int] = None
            Custom timeout that overwrites Client.timeout
        """
        url = self.api.POPULAR + '/decks'
        return self._get_model(url, **params)

def get_popular_decks ( self , * * params : keys ) : url = self . api . POPULAR + '/decks' return self . _get_model ( url , * * params )


def get_known_tournaments(self, **params: tournamentfilter):
        """Get a list of queried tournaments

        \*\*keys: Optional[list] = None
            Filter which keys should be included in the
            response
        \*\*exclude: Optional[list] = None
            Filter which keys should be excluded from the
            response
        \*\*max: Optional[int] = None
            Limit the number of items returned in the response
        \*\*page: Optional[int] = None
            Works with max, the zero-based page of the
            items
        \*\*timeout: Optional[int] = None
            Custom timeout that overwrites Client.timeout
        """
        url = self.api.TOURNAMENT + '/known'
        return self._get_model(url, PartialTournament, **params)

def get_known_tournaments ( self , * * params : tournamentfilter ) : url = self . api . TOURNAMENT + '/known' return self . _get_model ( url , PartialTournament , * * params )


def get_player(self, tag: crtag, timeout=None):
        """Get information about a player

        Parameters
        ----------
        tag: str
            A valid tournament tag. Minimum length: 3
            Valid characters: 0289PYLQGRJCUV
        timeout: Optional[int] = None
            Custom timeout that overwrites Client.timeout
        """
        url = self.api.PLAYER + '/' + tag
        return self._get_model(url, FullPlayer, timeout=timeout)

def get_player ( self , tag : crtag , timeout = None ) : url = self . api . PLAYER + '/' + tag return self . _get_model ( url , FullPlayer , timeout = timeout )


def get_player_chests(self, tag: crtag, timeout: int=None):
        """Get information about a player's chest cycle

        Parameters
        ----------
        tag: str
            A valid tournament tag. Minimum length: 3
            Valid characters: 0289PYLQGRJCUV
        timeout: Optional[int] = None
            Custom timeout that overwrites Client.timeout
        """
        url = self.api.PLAYER + '/' + tag + '/upcomingchests'
        return self._get_model(url, timeout=timeout)

def get_player_chests ( self , tag : crtag , timeout : int = None ) : url = self . api . PLAYER + '/' + tag + '/upcomingchests' return self . _get_model ( url , timeout = timeout )

Get information about a player s chest cycle


def get_clan(self, tag: crtag, timeout: int=None):
        """Get inforamtion about a clan

        Parameters
        ----------
        tag: str
            A valid tournament tag. Minimum length: 3
            Valid characters: 0289PYLQGRJCUV
        timeout: Optional[int] = None
            Custom timeout that overwrites Client.timeout
        """
        url = self.api.CLAN + '/' + tag
        return self._get_model(url, FullClan, timeout=timeout)

def get_clan ( self , tag : crtag , timeout : int = None ) : url = self . api . CLAN + '/' + tag return self . _get_model ( url , FullClan , timeout = timeout )


def search_clans(self, **params: clansearch):
        """Search for a clan. At least one
        of the filters must be present

        Parameters
        ----------
        name: Optional[str]
            The name of a clan
            (has to be at least 3 characters long)
        locationId: Optional[int]
            A location ID
        minMembers: Optional[int]
            The minimum member count
            of a clan
        maxMembers: Optional[int]
            The maximum member count
            of a clan
        minScore: Optional[int]
            The minimum trophy score of
            a clan
        \*\*limit: Optional[int] = None
            Limit the number of items returned in the response
        \*\*timeout: Optional[int] = None
            Custom timeout that overwrites Client.timeout
        """
        url = self.api.CLAN
        return self._get_model(url, PartialClan, **params)

def search_clans ( self , * * params : clansearch ) : url = self . api . CLAN return self . _get_model ( url , PartialClan , * * params )

Search for a clan . At least one of the filters must be present


def search_tournaments(self, name: str, **params: keys):
        """Search for a tournament by its name

        Parameters
        ----------
        name: str
            The name of a tournament
        \*\*limit: Optional[int] = None
            Limit the number of items returned in the response
        \*\*timeout: Optional[int] = None
            Custom timeout that overwrites Client.timeout
        """
        url = self.api.TOURNAMENT
        params['name'] = name
        return self._get_model(url, PartialTournament, **params)

def search_tournaments ( self , name : str , * * params : keys ) : url = self . api . TOURNAMENT params [ 'name' ] = name return self . _get_model ( url , PartialTournament , * * params )


def get_all_cards(self, timeout: int=None):
        """Get a list of all the cards in the game

        Parameters
        ----------
        timeout: Optional[int] = None
            Custom timeout that overwrites Client.timeout
        """
        url = self.api.CARDS
        return self._get_model(url, timeout=timeout)

def get_all_cards ( self , timeout : int = None ) : url = self . api . CARDS return self . _get_model ( url , timeout = timeout )


def get_all_locations(self, timeout: int=None):
        """Get a list of all locations

        Parameters
        ----------
        timeout: Optional[int] = None
            Custom timeout that overwrites Client.timeout
        """
        url = self.api.LOCATIONS
        return self._get_model(url, timeout=timeout)

def get_all_locations ( self , timeout : int = None ) : url = self . api . LOCATIONS return self . _get_model ( url , timeout = timeout )


def get_location(self, location_id: int, timeout: int=None):
        """Get a location information

        Parameters
        ----------
        location_id: int
            A location ID
            See https://github.com/RoyaleAPI/cr-api-data/blob/master/json/regions.json
            for a list of acceptable location IDs
        timeout: Optional[int] = None
            Custom timeout that overwrites Client.timeout
        """
        url = self.api.LOCATIONS + '/' + str(location_id)
        return self._get_model(url, timeout=timeout)

def get_location ( self , location_id : int , timeout : int = None ) : url = self . api . LOCATIONS + '/' + str ( location_id ) return self . _get_model ( url , timeout = timeout )


def get_top_clans(self, location_id='global', **params: keys):
        """Get a list of top clans by trophy

        Parameters
        ----------
        location_id: Optional[str] = 'global'
            A location ID or global
            See https://github.com/RoyaleAPI/cr-api-data/blob/master/json/regions.json
            for a list of acceptable location IDs
        \*\*limit: Optional[int] = None
            Limit the number of items returned in the response
        \*\*timeout: Optional[int] = None
            Custom timeout that overwrites Client.timeout
        """
        url = self.api.LOCATIONS + '/' + str(location_id) + '/rankings/clans'
        return self._get_model(url, PartialClan, **params)

def get_top_clans ( self , location_id = 'global' , * * params : keys ) : url = self . api . LOCATIONS + '/' + str ( location_id ) + '/rankings/clans' return self . _get_model ( url , PartialClan , * * params )


def get_clan_image(self, obj: BaseAttrDict):
        """Get the clan badge image URL

        Parameters
        ---------
        obj: official_api.models.BaseAttrDict
            An object that has the clan badge ID either in ``.clan.badge_id`` or ``.badge_id``
            Can be a clan or a profile for example.

        Returns str
        """

        try:
            badge_id = obj.clan.badge_id
        except AttributeError:
            try:
                badge_id = obj.badge_id
            except AttributeError:
                return 'https://i.imgur.com/Y3uXsgj.png'

        if badge_id is None:
            return 'https://i.imgur.com/Y3uXsgj.png'

        for i in self.constants.alliance_badges:
            if i.id == badge_id:
                return 'https://royaleapi.github.io/cr-api-assets/badges/' + i.name + '.png'


def get_arena_image(self, obj: BaseAttrDict):
        """Get the arena image URL

        Parameters
        ---------
        obj: official_api.models.BaseAttrDict
            An object that has the arena ID in ``.arena.id``
            Can be ``Profile`` for example.

        Returns None or str
        """
        badge_id = obj.arena.id
        for i in self.constants.arenas:
            if i.id == badge_id:
                return 'https://royaleapi.github.io/cr-api-assets/arenas/arena{}.png'.format(i.arena_id)


def get_deck_link(self, deck: BaseAttrDict):
        """Form a deck link

        Parameters
        ---------
        deck: official_api.models.BaseAttrDict
            An object is a deck. Can be retrieved from ``Player.current_deck``

        Returns str
        """
        deck_link = 'https://link.clashroyale.com/deck/en?deck='

        for i in deck:
            card = self.get_card_info(i.name)
            deck_link += '{0.id};'.format(card)

        return deck_link


def get_datetime(self, timestamp: str, unix=True):
        """Converts a %Y%m%dT%H%M%S.%fZ to a UNIX timestamp
        or a datetime.datetime object

        Parameters
        ---------
        timestamp: str
            A timstamp in the %Y%m%dT%H%M%S.%fZ format, usually returned by the API
            in the ``created_time`` field for example (eg. 20180718T145906.000Z)
        unix: Optional[bool] = True
            Whether to return a POSIX timestamp (seconds since epoch) or not

        Returns int or datetime.datetime
        """
        time = datetime.strptime(timestamp, '%Y%m%dT%H%M%S.%fZ')
        if unix:
            return int(time.timestamp())
        else:
            return time

def get_datetime ( self , timestamp : str , unix = True ) : time = datetime . strptime ( timestamp , '%Y%m%dT%H%M%S.%fZ' ) if unix : return int ( time . timestamp ( ) ) else : return time

Converts a %Y%m%dT%H%M%S . %fZ to a UNIX timestamp or a datetime . datetime object


def typecasted(func):
    """Decorator that converts arguments via annotations."""
    signature = inspect.signature(func).parameters.items()

    @wraps(func)
    def wrapper(*args, **kwargs):
        args = list(args)
        new_args = []
        new_kwargs = {}
        for _, param in signature:
            converter = param.annotation
            if converter is inspect._empty:
                converter = lambda a: a  # do nothing
            if param.kind is param.POSITIONAL_OR_KEYWORD:
                if args:
                    to_conv = args.pop(0)
                    new_args.append(converter(to_conv))
            elif param.kind is param.VAR_POSITIONAL:
                for a in args:
                    new_args.append(converter(a))
            else:
                for k, v in kwargs.items():
                    nk, nv = converter(k, v)
                    new_kwargs[nk] = nv
        return func(*new_args, **new_kwargs)
    return wrapper

def
typecasted
(
func
)
:
signature
=
inspect
.
signature
(
func
)
.
parameters
.
items
(
)
@
wraps
(
func
)
def
wrapper
(
*
args
,
*
*
kwargs
)
:
args
=
list
(
args
)
new_args
=
[
]
new_kwargs
=
{
}
for
_
,
param
in
signature
:
converter
=
param
.
annotation
if
converter
is
inspect
.
_empty
:
converter
=
lambda
a
:
a
# do nothing
if
param
.
kind
is
param
.
POSITIONAL_OR_KEYWORD
:
if
args
:
to_conv
=
args
.
pop
(
0
)
new_args
.
append
(
converter
(
to_conv
)
)
elif
param
.
kind
is
param
.
VAR_POSITIONAL
:
for
a
in
args
:
new_args
.
append
(
converter
(
a
)
)
else
:
for
k
,
v
in
kwargs
.
items
(
)
:
nk
,
nv
=
converter
(
k
,
v
)
new_kwargs
[
nk
]
=
nv
return
func
(
*
new_args
,
*
*
new_kwargs
)
return
wrapper

Decorator that converts arguments via annotations .


def coerce_annotation(ann, namespace):
    '''Validate that the annotation has the correct namespace,
    and is well-formed.

    If the annotation is not of the correct namespace, automatic conversion
    is attempted.

    Parameters
    ----------
    ann : jams.Annotation
        The annotation object in question

    namespace : str
        The namespace pattern to match `ann` against

    Returns
    -------
    ann_coerced: jams.Annotation
        The annotation coerced to the target namespace

    Raises
    ------
    NamespaceError
        If `ann` does not match the proper namespace

    SchemaError
        If `ann` fails schema validation

    See Also
    --------
    jams.nsconvert.convert
    '''

    ann = convert(ann, namespace)
    ann.validate(strict=True)

    return ann

def coerce_annotation ( ann , namespace ) : ann = convert ( ann , namespace ) ann . validate ( strict = True ) return ann

Validate that the annotation has the correct namespace and is well - formed .


def beat(ref, est, **kwargs):
    r'''Beat tracking evaluation

    Parameters
    ----------
    ref : jams.Annotation
        Reference annotation object
    est : jams.Annotation
        Estimated annotation object
    kwargs
        Additional keyword arguments

    Returns
    -------
    scores : dict
        Dictionary of scores, where the key is the metric name (str) and
        the value is the (float) score achieved.

    See Also
    --------
    mir_eval.beat.evaluate

    Examples
    --------
    >>> # Load in the JAMS objects
    >>> ref_jam = jams.load('reference.jams')
    >>> est_jam = jams.load('estimated.jams')
    >>> # Select the first relevant annotations
    >>> ref_ann = ref_jam.search(namespace='beat')[0]
    >>> est_ann = est_jam.search(namespace='beat')[0]
    >>> scores = jams.eval.beat(ref_ann, est_ann)
    '''

    namespace = 'beat'
    ref = coerce_annotation(ref, namespace)
    est = coerce_annotation(est, namespace)

    ref_times, _ = ref.to_event_values()
    est_times, _ = est.to_event_values()

    return mir_eval.beat.evaluate(ref_times, est_times, **kwargs)

def beat ( ref , est , * * kwargs ) : namespace = 'beat' ref = coerce_annotation ( ref , namespace ) est = coerce_annotation ( est , namespace ) ref_times , _ = ref . to_event_values ( ) est_times , _ = est . to_event_values ( ) return mir_eval . beat . evaluate ( ref_times , est_times , * * kwargs )


def chord(ref, est, **kwargs):
    r'''Chord evaluation

    Parameters
    ----------
    ref : jams.Annotation
        Reference annotation object
    est : jams.Annotation
        Estimated annotation object
    kwargs
        Additional keyword arguments

    Returns
    -------
    scores : dict
        Dictionary of scores, where the key is the metric name (str) and
        the value is the (float) score achieved.

    See Also
    --------
    mir_eval.chord.evaluate

    Examples
    --------
    >>> # Load in the JAMS objects
    >>> ref_jam = jams.load('reference.jams')
    >>> est_jam = jams.load('estimated.jams')
    >>> # Select the first relevant annotations
    >>> ref_ann = ref_jam.search(namespace='chord')[0]
    >>> est_ann = est_jam.search(namespace='chord')[0]
    >>> scores = jams.eval.chord(ref_ann, est_ann)
    '''

    namespace = 'chord'
    ref = coerce_annotation(ref, namespace)
    est = coerce_annotation(est, namespace)
    ref_interval, ref_value = ref.to_interval_values()
    est_interval, est_value = est.to_interval_values()

    return mir_eval.chord.evaluate(ref_interval, ref_value,
                                   est_interval, est_value, **kwargs)

def chord ( ref , est , * * kwargs ) : namespace = 'chord' ref = coerce_annotation ( ref , namespace ) est = coerce_annotation ( est , namespace ) ref_interval , ref_value = ref . to_interval_values ( ) est_interval , est_value = est . to_interval_values ( ) return mir_eval . chord . evaluate ( ref_interval , ref_value , est_interval , est_value , * * kwargs )


def hierarchy_flatten(annotation):
    '''Flatten a multi_segment annotation into mir_eval style.

    Parameters
    ----------
    annotation : jams.Annotation
        An annotation in the `multi_segment` namespace

    Returns
    -------
    hier_intervalss : list
        A list of lists of intervals, ordered by increasing specificity.

    hier_labels : list
        A list of lists of labels, ordered by increasing specificity.
    '''

    intervals, values = annotation.to_interval_values()

    ordering = dict()

    for interval, value in zip(intervals, values):
        level = value['level']
        if level not in ordering:
            ordering[level] = dict(intervals=list(), labels=list())

        ordering[level]['intervals'].append(interval)
        ordering[level]['labels'].append(value['label'])

    levels = sorted(list(ordering.keys()))
    hier_intervals = [ordering[level]['intervals'] for level in levels]
    hier_labels = [ordering[level]['labels'] for level in levels]

    return hier_intervals, hier_labels

def hierarchy_flatten ( annotation ) : intervals , values = annotation . to_interval_values ( ) ordering = dict ( ) for interval , value in zip ( intervals , values ) : level = value [ 'level' ] if level not in ordering : ordering [ level ] = dict ( intervals = list ( ) , labels = list ( ) ) ordering [ level ] [ 'intervals' ] . append ( interval ) ordering [ level ] [ 'labels' ] . append ( value [ 'label' ] ) levels = sorted ( list ( ordering . keys ( ) ) ) hier_intervals = [ ordering [ level ] [ 'intervals' ] for level in levels ] hier_labels = [ ordering [ level ] [ 'labels' ] for level in levels ] return hier_intervals , hier_labels

Flatten a multi_segment annotation into mir_eval style .


def hierarchy(ref, est, **kwargs):
    r'''Multi-level segmentation evaluation

    Parameters
    ----------
    ref : jams.Annotation
        Reference annotation object
    est : jams.Annotation
        Estimated annotation object
    kwargs
        Additional keyword arguments

    Returns
    -------
    scores : dict
        Dictionary of scores, where the key is the metric name (str) and
        the value is the (float) score achieved.

    See Also
    --------
    mir_eval.hierarchy.evaluate

    Examples
    --------
    >>> # Load in the JAMS objects
    >>> ref_jam = jams.load('reference.jams')
    >>> est_jam = jams.load('estimated.jams')
    >>> # Select the first relevant annotations
    >>> ref_ann = ref_jam.search(namespace='multi_segment')[0]
    >>> est_ann = est_jam.search(namespace='multi_segment')[0]
    >>> scores = jams.eval.hierarchy(ref_ann, est_ann)
    '''
    namespace = 'multi_segment'
    ref = coerce_annotation(ref, namespace)
    est = coerce_annotation(est, namespace)
    ref_hier, ref_hier_lab = hierarchy_flatten(ref)
    est_hier, est_hier_lab = hierarchy_flatten(est)

    return mir_eval.hierarchy.evaluate(ref_hier, ref_hier_lab,
                                       est_hier, est_hier_lab,
                                       **kwargs)

def hierarchy ( ref , est , * * kwargs ) : namespace = 'multi_segment' ref = coerce_annotation ( ref , namespace ) est = coerce_annotation ( est , namespace ) ref_hier , ref_hier_lab = hierarchy_flatten ( ref ) est_hier , est_hier_lab = hierarchy_flatten ( est ) return mir_eval . hierarchy . evaluate ( ref_hier , ref_hier_lab , est_hier , est_hier_lab , * * kwargs )


def tempo(ref, est, **kwargs):
    r'''Tempo evaluation

    Parameters
    ----------
    ref : jams.Annotation
        Reference annotation object
    est : jams.Annotation
        Estimated annotation object
    kwargs
        Additional keyword arguments

    Returns
    -------
    scores : dict
        Dictionary of scores, where the key is the metric name (str) and
        the value is the (float) score achieved.

    See Also
    --------
    mir_eval.tempo.evaluate

    Examples
    --------
    >>> # Load in the JAMS objects
    >>> ref_jam = jams.load('reference.jams')
    >>> est_jam = jams.load('estimated.jams')
    >>> # Select the first relevant annotations
    >>> ref_ann = ref_jam.search(namespace='tempo')[0]
    >>> est_ann = est_jam.search(namespace='tempo')[0]
    >>> scores = jams.eval.tempo(ref_ann, est_ann)
    '''

    ref = coerce_annotation(ref, 'tempo')
    est = coerce_annotation(est, 'tempo')

    ref_tempi = np.asarray([o.value for o in ref])
    ref_weight = ref.data[0].confidence
    est_tempi = np.asarray([o.value for o in est])

    return mir_eval.tempo.evaluate(ref_tempi, ref_weight, est_tempi, **kwargs)

def tempo ( ref , est , * * kwargs ) : ref = coerce_annotation ( ref , 'tempo' ) est = coerce_annotation ( est , 'tempo' ) ref_tempi = np . asarray ( [ o . value for o in ref ] ) ref_weight = ref . data [ 0 ] . confidence est_tempi = np . asarray ( [ o . value for o in est ] ) return mir_eval . tempo . evaluate ( ref_tempi , ref_weight , est_tempi , * * kwargs )


def melody(ref, est, **kwargs):
    r'''Melody extraction evaluation

    Parameters
    ----------
    ref : jams.Annotation
        Reference annotation object
    est : jams.Annotation
        Estimated annotation object
    kwargs
        Additional keyword arguments

    Returns
    -------
    scores : dict
        Dictionary of scores, where the key is the metric name (str) and
        the value is the (float) score achieved.

    See Also
    --------
    mir_eval.melody.evaluate

    Examples
    --------
    >>> # Load in the JAMS objects
    >>> ref_jam = jams.load('reference.jams')
    >>> est_jam = jams.load('estimated.jams')
    >>> # Select the first relevant annotations
    >>> ref_ann = ref_jam.search(namespace='pitch_contour')[0]
    >>> est_ann = est_jam.search(namespace='pitch_contour')[0]
    >>> scores = jams.eval.melody(ref_ann, est_ann)
    '''

    namespace = 'pitch_contour'
    ref = coerce_annotation(ref, namespace)
    est = coerce_annotation(est, namespace)

    ref_times, ref_p = ref.to_event_values()
    est_times, est_p = est.to_event_values()

    ref_freq = np.asarray([p['frequency'] * (-1)**(~p['voiced']) for p in ref_p])
    est_freq = np.asarray([p['frequency'] * (-1)**(~p['voiced']) for p in est_p])

    return mir_eval.melody.evaluate(ref_times, ref_freq,
                                    est_times, est_freq,
                                    **kwargs)

def melody ( ref , est , * * kwargs ) : namespace = 'pitch_contour' ref = coerce_annotation ( ref , namespace ) est = coerce_annotation ( est , namespace ) ref_times , ref_p = ref . to_event_values ( ) est_times , est_p = est . to_event_values ( ) ref_freq = np . asarray ( [ p [ 'frequency' ] * ( - 1 ) ** ( ~ p [ 'voiced' ] ) for p in ref_p ] ) est_freq = np . asarray ( [ p [ 'frequency' ] * ( - 1 ) ** ( ~ p [ 'voiced' ] ) for p in est_p ] ) return mir_eval . melody . evaluate ( ref_times , ref_freq , est_times , est_freq , * * kwargs )


def pattern_to_mireval(ann):
    '''Convert a pattern_jku annotation object to mir_eval format.

    Parameters
    ----------
    ann : jams.Annotation
        Must have `namespace='pattern_jku'`

    Returns
    -------
    patterns : list of list of tuples
        - `patterns[x]` is a list containing all occurrences of pattern x

        - `patterns[x][y]` is a list containing all notes for
           occurrence y of pattern x

        - `patterns[x][y][z]` contains a time-note tuple
          `(time, midi note)`
    '''

    # It's easier to work with dictionaries, since we can't assume
    # sequential pattern or occurrence identifiers

    patterns = defaultdict(lambda: defaultdict(list))

    # Iterate over the data in interval-value format

    for time, observation in zip(*ann.to_event_values()):

        pattern_id = observation['pattern_id']
        occurrence_id = observation['occurrence_id']
        obs = (time, observation['midi_pitch'])

        # Push this note observation into the correct pattern/occurrence
        patterns[pattern_id][occurrence_id].append(obs)

    # Convert to list-list-tuple format for mir_eval
    return [list(_.values()) for _ in six.itervalues(patterns)]

Convert a pattern_jku annotation object to mir_eval format .


def pattern(ref, est, **kwargs):
    r'''Pattern detection evaluation

    Parameters
    ----------
    ref : jams.Annotation
        Reference annotation object
    est : jams.Annotation
        Estimated annotation object
    kwargs
        Additional keyword arguments

    Returns
    -------
    scores : dict
        Dictionary of scores, where the key is the metric name (str) and
        the value is the (float) score achieved.

    See Also
    --------
    mir_eval.pattern.evaluate

    Examples
    --------
    >>> # Load in the JAMS objects
    >>> ref_jam = jams.load('reference.jams')
    >>> est_jam = jams.load('estimated.jams')
    >>> # Select the first relevant annotations
    >>> ref_ann = ref_jam.search(namespace='pattern_jku')[0]
    >>> est_ann = est_jam.search(namespace='pattern_jku')[0]
    >>> scores = jams.eval.pattern(ref_ann, est_ann)
    '''

    namespace = 'pattern_jku'
    ref = coerce_annotation(ref, namespace)
    est = coerce_annotation(est, namespace)

    ref_patterns = pattern_to_mireval(ref)
    est_patterns = pattern_to_mireval(est)

    return mir_eval.pattern.evaluate(ref_patterns, est_patterns, **kwargs)

def pattern ( ref , est , * * kwargs ) : namespace = 'pattern_jku' ref = coerce_annotation ( ref , namespace ) est = coerce_annotation ( est , namespace ) ref_patterns = pattern_to_mireval ( ref ) est_patterns = pattern_to_mireval ( est ) return mir_eval . pattern . evaluate ( ref_patterns , est_patterns , * * kwargs )


def transcription(ref, est, **kwargs):
    r'''Note transcription evaluation

    Parameters
    ----------
    ref : jams.Annotation
        Reference annotation object
    est : jams.Annotation
        Estimated annotation object
    kwargs
        Additional keyword arguments

    Returns
    -------
    scores : dict
        Dictionary of scores, where the key is the metric name (str) and
        the value is the (float) score achieved.

    See Also
    --------
    mir_eval.transcription.evaluate

    Examples
    --------
    >>> # Load in the JAMS objects
    >>> ref_jam = jams.load('reference.jams')
    >>> est_jam = jams.load('estimated.jams')
    >>> # Select the first relevant annotations. You can use any annotation
    >>> # type that can be converted to pitch_contour (such as pitch_midi)
    >>> ref_ann = ref_jam.search(namespace='pitch_contour')[0]
    >>> est_ann = est_jam.search(namespace='note_hz')[0]
    >>> scores = jams.eval.transcription(ref_ann, est_ann)
    '''

    namespace = 'pitch_contour'
    ref = coerce_annotation(ref, namespace)
    est = coerce_annotation(est, namespace)
    ref_intervals, ref_p = ref.to_interval_values()
    est_intervals, est_p = est.to_interval_values()

    ref_pitches = np.asarray([p['frequency'] * (-1)**(~p['voiced']) for p in ref_p])
    est_pitches = np.asarray([p['frequency'] * (-1)**(~p['voiced']) for p in est_p])

    return mir_eval.transcription.evaluate(
        ref_intervals, ref_pitches, est_intervals, est_pitches, **kwargs)

def transcription ( ref , est , * * kwargs ) : namespace = 'pitch_contour' ref = coerce_annotation ( ref , namespace ) est = coerce_annotation ( est , namespace ) ref_intervals , ref_p = ref . to_interval_values ( ) est_intervals , est_p = est . to_interval_values ( ) ref_pitches = np . asarray ( [ p [ 'frequency' ] * ( - 1 ) ** ( ~ p [ 'voiced' ] ) for p in ref_p ] ) est_pitches = np . asarray ( [ p [ 'frequency' ] * ( - 1 ) ** ( ~ p [ 'voiced' ] ) for p in est_p ] ) return mir_eval . transcription . evaluate ( ref_intervals , ref_pitches , est_intervals , est_pitches , * * kwargs )


def add_namespace(filename):
    '''Add a namespace definition to our working set.

    Namespace files consist of partial JSON schemas defining the behavior
    of the `value` and `confidence` fields of an Annotation.

    Parameters
    ----------
    filename : str
        Path to json file defining the namespace object
    '''
    with open(filename, mode='r') as fileobj:
        __NAMESPACE__.update(json.load(fileobj))

def add_namespace ( filename ) : with open ( filename , mode = 'r' ) as fileobj : __NAMESPACE__ . update ( json . load ( fileobj ) )

Add a namespace definition to our working set .


def namespace(ns_key):
    '''Construct a validation schema for a given namespace.

    Parameters
    ----------
    ns_key : str
        Namespace key identifier (eg, 'beat' or 'segment_tut')

    Returns
    -------
    schema : dict
        JSON schema of `namespace`
    '''

    if ns_key not in __NAMESPACE__:
        raise NamespaceError('Unknown namespace: {:s}'.format(ns_key))

    sch = copy.deepcopy(JAMS_SCHEMA['definitions']['SparseObservation'])

    for key in ['value', 'confidence']:
        try:
            sch['properties'][key] = __NAMESPACE__[ns_key][key]
        except KeyError:
            pass

    return sch

def
namespace
(
ns_key
)
:
if
ns_key
not
in
__NAMESPACE__
:
raise
NamespaceError
(
'Unknown namespace: {:s}'
.
format
(
ns_key
)
)
sch
=
copy
.
deepcopy
(
JAMS_SCHEMA
[
'definitions'
]
[
'SparseObservation'
]
)
for
key
in
[
'value'
,
'confidence'
]
:
try
:
sch
[
'properties'
]
[
key
]
=
__NAMESPACE__
[
ns_key
]
[
key
]
except
KeyError
:
pass
return
sch

Construct a validation schema for a given namespace .


def namespace_array(ns_key):
    '''Construct a validation schema for arrays of a given namespace.

    Parameters
    ----------
    ns_key : str
        Namespace key identifier

    Returns
    -------
    schema : dict
        JSON schema of `namespace` observation arrays
    '''

    obs_sch = namespace(ns_key)
    obs_sch['title'] = 'Observation'

    sch = copy.deepcopy(JAMS_SCHEMA['definitions']['SparseObservationList'])
    sch['items'] = obs_sch
    return sch

def namespace_array ( ns_key ) : obs_sch = namespace ( ns_key ) obs_sch [ 'title' ] = 'Observation' sch = copy . deepcopy ( JAMS_SCHEMA [ 'definitions' ] [ 'SparseObservationList' ] ) sch [ 'items' ] = obs_sch return sch

Construct a validation schema for arrays of a given namespace .


def values(ns_key):
    '''Return the allowed values for an enumerated namespace.

    Parameters
    ----------
    ns_key : str
        Namespace key identifier

    Returns
    -------
    values : list

    Raises
    ------
    NamespaceError
        If `ns_key` is not found, or does not have enumerated values

    Examples
    --------
    >>> jams.schema.values('tag_gtzan')
    ['blues', 'classical', 'country', 'disco', 'hip-hop', 'jazz',
     'metal', 'pop', 'reggae', 'rock']
    '''

    if ns_key not in __NAMESPACE__:
        raise NamespaceError('Unknown namespace: {:s}'.format(ns_key))

    if 'enum' not in __NAMESPACE__[ns_key]['value']:
        raise NamespaceError('Namespace {:s} is not enumerated'.format(ns_key))

    return copy.copy(__NAMESPACE__[ns_key]['value']['enum'])

def
values
(
ns_key
)
:
if
ns_key
not
in
__NAMESPACE__
:
raise
NamespaceError
(
'Unknown namespace: {:s}'
.
format
(
ns_key
)
)
if
'enum'
not
in
__NAMESPACE__
[
ns_key
]
[
'value'
]
:
raise
NamespaceError
(
'Namespace {:s} is not enumerated'
.
format
(
ns_key
)
)
return
copy
.
copy
(
__NAMESPACE__
[
ns_key
]
[
'value'
]
[
'enum'
]
)

Return the allowed values for an enumerated namespace .


def get_dtypes(ns_key):
    '''Get the dtypes associated with the value and confidence fields
    for a given namespace.

    Parameters
    ----------
    ns_key : str
        The namespace key in question

    Returns
    -------
    value_dtype, confidence_dtype : numpy.dtype
        Type identifiers for value and confidence fields.
    '''

    # First, get the schema
    if ns_key not in __NAMESPACE__:
        raise NamespaceError('Unknown namespace: {:s}'.format(ns_key))

    value_dtype = __get_dtype(__NAMESPACE__[ns_key].get('value', {}))
    confidence_dtype = __get_dtype(__NAMESPACE__[ns_key].get('confidence', {}))

    return value_dtype, confidence_dtype

def
get_dtypes
(
ns_key
)
:
# First, get the schema
if
ns_key
not
in
__NAMESPACE__
:
raise
NamespaceError
(
'Unknown namespace: {:s}'
.
format
(
ns_key
)
)
value_dtype
=
__get_dtype
(
__NAMESPACE__
[
ns_key
]
.
get
(
'value'
,
{
}
)
)
confidence_dtype
=
__get_dtype
(
__NAMESPACE__
[
ns_key
]
.
get
(
'confidence'
,
{
}
)
)
return
value_dtype
,
confidence_dtype

Get the dtypes associated with the value and confidence fields for a given namespace .


def list_namespaces():
    '''Print out a listing of available namespaces'''
    print('{:30s}\t{:40s}'.format('NAME', 'DESCRIPTION'))
    print('-' * 78)
    for sch in sorted(__NAMESPACE__):
        desc = __NAMESPACE__[sch]['description']
        desc = (desc[:44] + '..') if len(desc) > 46 else desc
        print('{:30s}\t{:40s}'.format(sch, desc))

def list_namespaces ( ) : print ( '{:30s}\t{:40s}' . format ( 'NAME' , 'DESCRIPTION' ) ) print ( '-' * 78 ) for sch in sorted ( __NAMESPACE__ ) : desc = __NAMESPACE__ [ sch ] [ 'description' ] desc = ( desc [ : 44 ] + '..' ) if len ( desc ) > 46 else desc print ( '{:30s}\t{:40s}' . format ( sch , desc ) )


def __get_dtype(typespec):
    '''Get the dtype associated with a jsonschema type definition

    Parameters
    ----------
    typespec : dict
        The schema definition

    Returns
    -------
    dtype : numpy.dtype
        The associated dtype
    '''

    if 'type' in typespec:
        return __TYPE_MAP__.get(typespec['type'], np.object_)

    elif 'enum' in typespec:
        # Enums map to objects
        return np.object_

    elif 'oneOf' in typespec:
        # Recurse
        types = [__get_dtype(v) for v in typespec['oneOf']]

        # If they're not all equal, return object
        if all([t == types[0] for t in types]):
            return types[0]

    return np.object_


def __load_jams_schema():
    '''Load the schema file from the package.'''

    schema_file = os.path.join(SCHEMA_DIR, 'jams_schema.json')

    jams_schema = None
    with open(resource_filename(__name__, schema_file), mode='r') as fdesc:
        jams_schema = json.load(fdesc)

    if jams_schema is None:
        raise JamsError('Unable to load JAMS schema')

    return jams_schema

def
__load_jams_schema
(
)
:
schema_file
=
os
.
path
.
join
(
SCHEMA_DIR
,
'jams_schema.json'
)
jams_schema
=
with
open
(
resource_filename
(
__name__
,
schema_file
)
,
mode
=
'r'
)
as
fdesc
:
jams_schema
=
json
.
load
(
fdesc
)
if
jams_schema
is
:
raise
JamsError
(
'Unable to load JAMS schema'
)
return
jams_schema


def import_lab(namespace, filename, infer_duration=True, **parse_options):
    r'''Load a .lab file as an Annotation object.

    .lab files are assumed to have the following format:

        ``TIME_START\tTIME_END\tANNOTATION``

    By default, .lab files are assumed to have columns separated by one
    or more white-space characters, and have no header or index column
    information.

    If the .lab file contains only two columns, then an empty duration
    field is inferred.

    If the .lab file contains more than three columns, each row's
    annotation value is assigned the contents of last non-empty column.

    Parameters
    ----------
    namespace : str
        The namespace for the new annotation

    filename : str
        Path to the .lab file

    infer_duration : bool
        If `True`, interval durations are inferred from `(start, end)` columns,
        or difference between successive times.

        If `False`, interval durations are assumed to be explicitly coded as
        `(start, duration)` columns.  If only one time column is given, then
        durations are set to 0.

        For instantaneous event annotations (e.g., beats or onsets), this
        should be set to `False`.

    parse_options : additional keyword arguments
        Passed to ``pandas.DataFrame.read_csv``

    Returns
    -------
    annotation : Annotation
        The newly constructed annotation object

    See Also
    --------
    pandas.DataFrame.read_csv
    '''

    # Create a new annotation object
    annotation = core.Annotation(namespace)

    parse_options.setdefault('sep', r'\s+')
    parse_options.setdefault('engine', 'python')
    parse_options.setdefault('header', None)
    parse_options.setdefault('index_col', False)

    # This is a hack to handle potentially ragged .lab data
    parse_options.setdefault('names', range(20))

    data = pd.read_csv(filename, **parse_options)

    # Drop all-nan columns
    data = data.dropna(how='all', axis=1)

    # Do we need to add a duration column?
    # This only applies to event annotations
    if len(data.columns) == 2:
        # Insert a column of zeros after the timing
        data.insert(1, 'duration', 0)
        if infer_duration:
            data['duration'][:-1] = data.loc[:, 0].diff()[1:].values

    else:
        # Convert from time to duration
        if infer_duration:
            data.loc[:, 1] -= data[0]

    for row in data.itertuples():
        time, duration = row[1:3]

        value = [x for x in row[3:] if x is not None][-1]

        annotation.append(time=time,
                          duration=duration,
                          confidence=1.0,
                          value=value)

    return annotation

r Load a . lab file as an Annotation object .


def expand_filepaths(base_dir, rel_paths):
    """Expand a list of relative paths to a give base directory.

    Parameters
    ----------
    base_dir : str
        The target base directory

    rel_paths : list (or list-like)
        Collection of relative path strings

    Returns
    -------
    expanded_paths : list
        `rel_paths` rooted at `base_dir`

    Examples
    --------
    >>> jams.util.expand_filepaths('/data', ['audio', 'beat', 'seglab'])
    ['/data/audio', '/data/beat', '/data/seglab']

    """
    return [os.path.join(base_dir, os.path.normpath(rp)) for rp in rel_paths]

def expand_filepaths ( base_dir , rel_paths ) : return [ os . path . join ( base_dir , os . path . normpath ( rp ) ) for rp in rel_paths ]

Expand a list of relative paths to a give base directory .


def smkdirs(dpath, mode=0o777):
    """Safely make a full directory path if it doesn't exist.

    Parameters
    ----------
    dpath : str
        Path of directory/directories to create

    mode : int [default=0777]
        Permissions for the new directories

    See also
    --------
    os.makedirs
    """
    if not os.path.exists(dpath):
        os.makedirs(dpath, mode=mode)

def smkdirs ( dpath , mode = 0o777 ) : if not os . path . exists ( dpath ) : os . makedirs ( dpath , mode = mode )

Safely make a full directory path if it doesn t exist .


def find_with_extension(in_dir, ext, depth=3, sort=True):
    """Naive depth-search into a directory for files with a given extension.

    Parameters
    ----------
    in_dir : str
        Path to search.
    ext : str
        File extension to match.
    depth : int
        Depth of directories to search.
    sort : bool
        Sort the list alphabetically

    Returns
    -------
    matched : list
        Collection of matching file paths.

    Examples
    --------
    >>> jams.util.find_with_extension('Audio', 'wav')
    ['Audio/LizNelson_Rainfall/LizNelson_Rainfall_MIX.wav',
     'Audio/LizNelson_Rainfall/LizNelson_Rainfall_RAW/LizNelson_Rainfall_RAW_01_01.wav',
     'Audio/LizNelson_Rainfall/LizNelson_Rainfall_RAW/LizNelson_Rainfall_RAW_02_01.wav',
     ...
     'Audio/Phoenix_ScotchMorris/Phoenix_ScotchMorris_STEMS/Phoenix_ScotchMorris_STEM_02.wav',
     'Audio/Phoenix_ScotchMorris/Phoenix_ScotchMorris_STEMS/Phoenix_ScotchMorris_STEM_03.wav',
    'Audio/Phoenix_ScotchMorris/Phoenix_ScotchMorris_STEMS/Phoenix_ScotchMorris_STEM_04.wav']

    """
    assert depth >= 1
    ext = ext.strip(os.extsep)
    match = list()
    for n in range(1, depth+1):
        wildcard = os.path.sep.join(["*"]*n)
        search_path = os.path.join(in_dir, os.extsep.join([wildcard, ext]))
        match += glob.glob(search_path)

    if sort:
        match.sort()
    return match

def find_with_extension ( in_dir , ext , depth = 3 , sort = True ) : assert depth >= 1 ext = ext . strip ( os . extsep ) match = list ( ) for n in range ( 1 , depth + 1 ) : wildcard = os . path . sep . join ( [ "*" ] * n ) search_path = os . path . join ( in_dir , os . extsep . join ( [ wildcard , ext ] ) ) match += glob . glob ( search_path ) if sort : match . sort ( ) return match

Naive depth - search into a directory for files with a given extension .


def get_comments(jam, ann):
    '''Get the metadata from a jam and an annotation, combined as a string.

    Parameters
    ----------
    jam : JAMS
        The jams object

    ann : Annotation
        An annotation object

    Returns
    -------
    comments : str
        The jam.file_metadata and ann.annotation_metadata, combined and serialized
    '''
    jam_comments = jam.file_metadata.__json__
    ann_comments = ann.annotation_metadata.__json__
    return json.dumps({'metadata': jam_comments,
                       'annotation metadata': ann_comments},
                      indent=2)

def
get_comments
(
jam
,
ann
)
:
jam_comments
=
jam
.
file_metadata
.
__json__
ann_comments
=
ann
.
annotation_metadata
.
__json__
return
json
.
dumps
(
{
'metadata'
:
jam_comments
,
'annotation metadata'
:
ann_comments
}
,
indent
=
2
)

Get the metadata from a jam and an annotation combined as a string .


def convert_jams(jams_file, output_prefix, csv=False, comment_char='#', namespaces=None):
    '''Convert jams to labs.

    Parameters
    ----------
    jams_file : str
        The path on disk to the jams file in question

    output_prefix : str
        The file path prefix of the outputs

    csv : bool
        Whether to output in csv (True) or lab (False) format

    comment_char : str
        The character used to denote comments

    namespaces : list-like
        The set of namespace patterns to match for output
    '''

    if namespaces is None:
        raise ValueError('No namespaces provided. Try ".*" for all namespaces.')

    jam = jams.load(jams_file)

    # Get all the annotations
    # Filter down to the unique ones
    # For each annotation
    #     generate the comment string
    #     generate the output filename
    #     dump to csv

    # Make a counter object for each namespace type
    counter = collections.Counter()

    annotations = []

    for query in namespaces:
        annotations.extend(jam.search(namespace=query))

    if csv:
        suffix = 'csv'
        sep = ','
    else:
        suffix = 'lab'
        sep = '\t'

    for ann in annotations:
        index = counter[ann.namespace]
        counter[ann.namespace] += 1
        filename = os.path.extsep.join([get_output_name(output_prefix,
                                                        ann.namespace,
                                                        index),
                                        suffix])

        comment = get_comments(jam, ann)

        # Dump to disk
        lab_dump(ann, comment, filename, sep, comment_char)


def parse_arguments(args):
    '''Parse arguments from the command line'''
    parser = argparse.ArgumentParser(description='Convert JAMS to .lab files')

    parser.add_argument('-c',
                        '--comma-separated',
                        dest='csv',
                        action='store_true',
                        default=False,
                        help='Output in .csv instead of .lab')

    parser.add_argument('--comment', dest='comment_char', type=str, default='#',
                        help='Comment character')

    parser.add_argument('-n',
                        '--namespace',
                        dest='namespaces',
                        nargs='+',
                        default=['.*'],
                        help='One or more namespaces to output.  Default is all.')

    parser.add_argument('jams_file',
                        help='Path to the input jams file')

    parser.add_argument('output_prefix', help='Prefix for output files')

    return vars(parser.parse_args(args))


def _conversion(target, source):
    '''A decorator to register namespace conversions.

    Usage
    -----
    >>> @conversion('tag_open', 'tag_.*')
    ... def tag_to_open(annotation):
    ...     annotation.namespace = 'tag_open'
    ...     return annotation
    '''

    def register(func):
        '''This decorator registers func as mapping source to target'''
        __CONVERSION__[target][source] = func
        return func

    return register

A decorator to register namespace conversions .