def load_classes_txt(filename, corpus, split=None):
    """Load class file, i.e. the output of a detector and calculate the
    corresponding annotation according to `corpus`.

    The class file must be formatted as follows:
    Class <ID1> (<symbol>)
    name 0.000 1.000
    ...

    Class <ID2> (<symbol>)
    name 1.200 3.400
    ...


    Parameters
    ----------
    filename : string
    corpus : Corpus
        Contains the annotation by which to annotate the classes.

    Returns
    -------
    n : dict from ClassID to tuple of FragmentToken
        Annotated classes.

    """
    raw = load_classfile(filename)  # without annotation
    # add mark annotation
    return annotate_classes(raw, corpus, split=split) def load_corpus_txt(filename):
    """Read file with phone annotation and convert into Corpus object.

    The phone file must be formatted with the following, space-separated
    columns:
    identifier start stop symbol

    Parameters
    ----------
    filename : string

    Returns
    -------
    c : Corpus

    """
    return tokenlists_to_corpus(load_annotation(filename)) def load_split(fname, multiple=False):
    with open(fname) as fid:
        s = fid.read()
    if multiple:
        r = read_split_multiple(s)
    else:
        r = read_split_single(s)
    return r @contextmanager
def verb_print(label, verbose=False, when_done=False,
               timeit=False, with_dots=False):
    if timeit:
        t0 = time.time()
    if verbose:
        msg = label + ('...' if with_dots else '') + ('' if when_done else '\n')
        print msg,
        sys.stdout.flush()
    try:
        yield
    finally:
        if verbose and when_done:
            if timeit:
                print 'done. Time: {0:.3f}s'.format(time.time() - t0)
            else:
                print 'done.'
            sys.stdout.flush()
        elif verbose and timeit:
            print '{1}: time: {0:.3f}s'.format(time.time() - t0, label)
            sys.stdout.flush() def banner(s):
    l = len(s)
    return '-'*l+'\n'+s+'\n'+'-'*l def pretty_score_f(ps, rs, fs, label, nfolds, nsamples):
    r = '{sep}\n'.format(sep=37*'-')
    r += '{label}\n#folds:    {nfolds}\n#samples:  {nsamples}\n'.format(
        label=label, nfolds=nfolds, nsamples=nsamples)
    r += '{sep}\n'.format(sep=37*'-')
    r += '{score:9s}  {mean:5s}  {std:5s}  {min:5s}  {max:5s}\n'.format(
        score="measure", mean="mean", std="std",
        min="min", max="max")
    r += '---------  -----  -----  -----  -----\n'
    r += '{score:9s}  {mean:.3f}  {std:.3f}  {min:.3f}  {max:.3f}\n'.format(
        score="precision", mean=ps.mean(), std=ps.std(),
        min=ps.min(), max=ps.max())
    r += '{score:9s}  {mean:.3f}  {std:.3f}  {min:.3f}  {max:.3f}\n'.format(
        score="recall", mean=rs.mean(), std=rs.std(),
        min=rs.min(), max=rs.max())
    r += '{score:9s}  {mean:.3f}  {std:.3f}  {min:.3f}  {max:.3f}\n'.format(
        score="fscore", mean=fs.mean(), std=fs.std(),
        min=fs.min(), max=fs.max())
    r += '{sep}\n'.format(sep=37*'-')
    return r def pretty_score_nlp(ned_score, coverage_score, label, nfolds, nsamples):
    r = '{sep}\n'.format(sep=37*'-')
    r += '{label}\n#folds:    {nfolds}\n#samples:  {nsamples}\n'.format(
        label=label, nfolds=nfolds, nsamples=nsamples)
    r += '{sep}\n'.format(sep=37*'-')
    r += '{score:9s}  {mean:5s}  {std:5s}  {min:5s}  {max:5s}\n'.format(
        score="measure", mean="mean", std="std", min="min", max="max")
    r += '---------  -----  -----  -----  -----\n'
    r += '{score:9s}  {mean:.3f}  {std:.3f}  {min:.3f}  {max:.3f}\n'.format(
        score="NED", mean=ned_score.mean(), std=ned_score.std(),
        min=ned_score.min(), max=ned_score.max())
    r += '{score:9s}  {mean:.3f}  {std:.3f}  {min:.3f}  {max:.3f}\n'.format(
        score="coverage", mean=coverage_score.mean(), std=coverage_score.std(),
        min=coverage_score.min(), max=coverage_score.max())
    r += '{sep}\n'.format(sep=37*'-')
    return r def truncate_intervals(clsdict, corpus, mapping):
    disc = {}
    interval_errors = []
    filename_errors = []
    for class_id in clsdict:
        fragments = []
        for fragment in clsdict[class_id]:
            qname = fragment.name
            qstart = fragment.interval.start
            qend = fragment.interval.end
            try:
                finterval = mapping.largest_overlap(qname, fragment.interval)
            except KeyError:
                filename_errors.append(fragment.name)
                continue
            except ValueError:
                interval_errors.append(fragment)
            fstart, fend = finterval
            if qstart != fstart or qend != fend:
                newstart = max(qstart, fstart)
                newend = min(qend, fend)
                newinterval = Interval(newstart, newend)
                newmark = corpus.annotation(qname, newinterval)
                fragment = FragmentToken(qname,
                                         newinterval,
                                         newmark)
            fragments.append(fragment)
        disc[class_id] = tuple(fragments)
    return ClassDict(disc), filename_errors, interval_errors def check_intervals(clsdict, interval_db):
    """
    Check whether all the intervals in a ClassDict are also in an IntervalDB

    Parameters
    ----------
    clsdict : ClassDict
    interval_db : IntervalDB

    Returns
    -------
    interval_errors : list of FragmentToken
        List of fragments for which the interval could not be found.
    filename_errors : list of FragmentToken
        List of fragments for which the filename could not be found.
    """

    interval_errors = []
    filename_errors = []
    for fragment in clsdict.iter_fragments():
        try:
            finterval = interval_db.largest_overlap(fragment.name,
                                                    fragment.interval)
        except KeyError:
            import ipdb; ipdb.set_trace()
            filename_errors.append(fragment)
            continue
        except ValueError:
            import ipdb; ipdb.set_trace()
            interval_errors.append(fragment)
            continue
        # fstart, fend = finterval
        # qstart = fragment.interval.start
        # qend = fragment.interval.end
        # if qstart != fstart or qend != fend:
        #     interval_errors.append(fragment)
    return filename_errors, interval_errors def fscore(p, r):
    if p == 0 and r == 0:
        f = 0
    else:
        f = 2 * p * r / (p + r)
    return f def NED(clsdict):
    neds = np.fromiter((ned(f1.mark, f2.mark)
                        for f1, f2 in clsdict.iter_pairs(within=True,
                                                         order=False)),
                       dtype=np.double)
    if len(neds) == 0:
        r = np.nan
    else:
        r = neds.mean()
    return r def coverage(disc_clsdict, gold_clsdict):
    """Compute the coverage of disc_clsdict relative to gold_clsdict

    Parameters
    ----------
    disc_clsdict, gold_clsdict : ClassDict

    Returns
    -------
    float
        Relative coverage
    """
    num = cover(disc_clsdict)
    den = cover(gold_clsdict)
    if np.isclose(den, 0.) or np.isclose(num, 0):
        c = np.nan
    else:
        c = num  / den
    return c def evaluate_group(disc_clsdict, verbose=False, debug=False):
    pclus = make_pclus(disc_clsdict, verbose, debug)
    pgoldclus = make_pgoldclus(disc_clsdict, verbose, debug)

    ts_disc = make_typeset(pclus, verbose, debug)
    ts_gold = make_typeset(pgoldclus, verbose, debug)

    pclus_pgoldclus_nmatch = make_pclus_pgoldclus_nmatch(pclus,
                                                         pgoldclus,
                                                         verbose, debug)
    pclus_nmatch = make_pclus_nmatch(pclus, verbose, debug)
    pgoldclus_nmatch = make_pgoldclus_nmatch(pgoldclus,
                                             verbose, debug)

    ws_disc = make_weights(pclus, verbose, debug)
    ws_gold = make_weights(pgoldclus, verbose, debug)

    if len(pclus_nmatch) == 0:
        prec = np.nan
    else:
        prec = sum(ws_disc[t] * pclus_pgoldclus_nmatch[t] / pclus_nmatch[t]
                   for t in ts_disc if pclus_nmatch[t] > 0)

    if len(pgoldclus_nmatch) == 0:
        rec = np.nan
    else:
        rec = sum(ws_gold[t] * pclus_pgoldclus_nmatch[t] / pgoldclus_nmatch[t]
                  for t in ts_gold if pgoldclus_nmatch[t] > 0)

    return prec, rec class Boundaries(object):
    def __init__(self, container, threshold=0.03):
        self.threshold = threshold
        if hasattr(container, 'iter_fragments'):
            iterator = container.iter_fragments()
        else:
            iterator = (f for f in container)

        bounds = defaultdict(list)
        length = 0
        for fragment in iterator:
            bounds[fragment.name].append(fragment.interval)
            length += 1
        self.length = length
        self.bounds = {}
        for name, intervals in bounds.iteritems():
            points = set(flatten((interval.start, interval.end)
                                  for interval in intervals))
            self.bounds[name] = np.sort(np.array(list(points)))

    def __len__(self):
        return self.length

    def has_close(self, name, query_point):
        if not name in self.bounds:
            b = False
        else:
            points = self.bounds[name]
            ix = np.searchsorted(points, query_point, side='left')
            if ix == 0:
                found_point = points[0]
            elif ix == points.shape[0]:
                found_point = points[-1]
            elif query_point - points[ix-1] < points[ix] - query_point:
                found_point = points[ix-1]
            else:
                found_point = points[ix]
            b = np.abs(found_point - query_point) < self.threshold
        return b

    def __iter__(self):
        for name, points in self.bounds.iteritems():
            for point in points:
                yield name, point def eval_from_bounds(disc, gold):
    gold_close = np.fromiter((gold.has_close(n, p)
                              for n, p in disc),
                             dtype=np.bool)
    if gold_close.shape[0] > 0:
        prec = gold_close.mean()
    else:
        prec = np.nan

    disc_close = np.fromiter((disc.has_close(n, p)
                              for n, p in gold),
                             dtype=np.bool)
    if disc_close.shape[0] > 0:
        rec = disc_close.mean()
    else:
        rec = np.nan
    return prec, rec def eval_from_psets(pdisc, pgold, psubs, verbose=False, debug=False):
    ts_disc = make_typeset(psubs, verbose, debug)
    ts_gold = make_typeset(pgold, verbose, debug)

    psubs_pgold_nmatch = make_psubs_pgold_nmatch(pgold, psubs,
                                                 verbose, debug)

    psubs_nmatch = make_psubs_nmatch(psubs, verbose, debug)
    pgold_nmatch = make_pgold_nmatch(pgold, verbose, debug)

    ws_disc = make_weights(psubs, verbose, debug)
    ws_gold = make_weights(pgold, verbose, debug)

    if len(psubs_nmatch) == 0:
        prec = np.nan
    else:
        prec = sum(ws_disc[t] * psubs_pgold_nmatch[t] / psubs_nmatch[t]
                   for t in ts_disc if psubs_nmatch[t] > 0)

    if len(pgold_nmatch) == 0:
        rec = np.nan
    else:
        rec = sum(ws_gold[t] * psubs_pgold_nmatch[t] / pgold_nmatch[t]
                  for t in ts_gold if pgold_nmatch[t] > 0)

    return prec, rec def make_pdisc(disc_clsdict, verbose, debug):
    with verb_print('constructing pdisc set', verbose, True, True):
        pdisc = list(Pclus(disc_clsdict))
    if debug:
        print banner('PDISC ({0})'.format(len(pdisc)))
        print pretty_pairs(pdisc)
        print
    return pdisc def make_pgold(gold_clsdict, verbose, debug):
    with verb_print('constructing pgold set', verbose, True, True):
        pgold = list(Pclus(gold_clsdict))
    if debug:
        print banner('PGOLD ({0})'.format(len(pgold)))
        print pretty_pairs(pgold)
        print
    return pgold def make_psubs(disc_clsdict, corpus, minlength, maxlength,
               verbose, debug):
    with verb_print('constructing psubs set', verbose, True, True):
        psubs = list(Psubs(disc_clsdict, corpus, minlength=minlength,
                           maxlength=maxlength))
    if debug:
        print banner('PSUBS ({0})'.format(len(psubs)))
        print pretty_pairs(psubs)
        print
    return psubs def evaluate_token_type(disc_clsdict, wrd_corpus,
                        verbose=False, debug=False):
    n_word_tokens = iterator_length(unique(wrd_corpus.iter_fragments()))
    word_types = set(f.mark for f in wrd_corpus.iter_fragments())
    n_word_types = len(word_types)
    n_disc_fragments = iterator_length(disc_clsdict.iter_fragments())

    with verb_print('querying words', verbose, True, True, True):
        types_hit = set()
        types_seen = set()
        hits = 0
        for disc_fragment in disc_clsdict.iter_fragments():
            disc_start = disc_fragment.interval.start
            disc_end = disc_fragment.interval.end
            wrd_tokens = wrd_corpus.tokens(disc_fragment.name,
                                           disc_fragment.interval)
            types_seen.add(tuple(f.mark for f in wrd_tokens))
            if len(wrd_tokens) != 1:
                continue
            goldtok = wrd_tokens[0]
            if abs(goldtok.interval.start - disc_start) > 0.03:
                continue
            if abs(goldtok.interval.end - disc_end) > 0.03:
                continue
            types_hit.add(goldtok.mark)
            hits += 1

    if n_disc_fragments == 0:
        token_prec = np.nan
    else:
        token_prec = hits / n_disc_fragments

    if n_word_tokens == 0:
        token_rec = np.nan
    else:
        token_rec = hits / n_word_tokens

    if len(types_seen) == 0:
        type_prec = np.nan
    else:
        type_prec = len(types_hit) / len(types_seen)

    if n_word_types == 0:
        type_rec = np.nan
    else:
        type_rec = len(types_hit) / n_word_types

    return token_prec, token_rec, type_prec, type_rec
import os
import os.path as path
import sys
import numpy as np
    import argparse
from itertools import izip
from joblib import Parallel, delayed
from tde.util.reader import load_classes_txt, load_corpus_txt, load_split
from tde.util.printing import verb_print, banner, pretty_score_f, \
    pretty_score_nlp
from tde.util.splits import truncate_intervals, check_intervals
from tde.util.functions import fscore
from tde.measures.nlp import NED, coverage
from tde.measures.group import evaluate_group
from tde.measures.boundaries import Boundaries, eval_from_bounds
from tde.measures.match import eval_from_psets, make_pdisc, make_pgold, \
    make_psubs
from tde.measures.token_type import evaluate_token_type


def _load_classes(fname, corpus, split_mapping=None):
    return load_classes_txt(fname, corpus, split=split_mapping)


def load_disc(fname, corpus, split_file, truncate, verbose):
    with verb_print('  loading discovered classes',
                             verbose, True, True, True):
        split_mapping = load_split(split_file)
        disc, errors = _load_classes(fname, corpus, split_mapping)
        if not truncate:
            errors_found = len(errors) > 0
            if len(errors) > 100:
                print 'There were more than 100 interval errors found.'
                print 'Printing only the first 100.'
                print
                errors = errors[:100]
            for fragment in sorted(errors, key=lambda x: (x.name, x.interval.start)):
                print '  error: {0} [{1:.3f}, {2:.3f}]'.format(
                    fragment.name, fragment.interval.start, fragment.interval.end)
            if not truncate and errors_found:
                print 'There were errors in {0}. Use option -f to'\
                    ' automatically skip invalid intervals.'.format(fname)
                sys.exit()

    if truncate:
        with verb_print('  checking discovered classes and truncating'):
            disc, filename_errors, interval_errors = \
                truncate_intervals(disc, corpus,
                                   split_mapping)
    else:
        with verb_print('  checking discovered classes', verbose, True,
                                 True, True):
            filename_errors, interval_errors = \
                check_intervals(disc, split_mapping)
    if not truncate:
        filename_errors = sorted(filename_errors,
                                 key=lambda x: (x.name, x.interval.start))
        interval_errors = sorted(interval_errors,
                                 key=lambda x: (x.name, x.interval.start))
        interval_error = len(interval_errors) > 0
        filename_error = len(filename_errors) > 0
        errors_found = filename_error or interval_error
        if interval_error:
            print banner('intervals found in {0} outside of valid'
                                      ' splits'.format(fname))
            if len(interval_errors) > 100:
                print 'There were more than 100 interval errors found.'
                print 'Printing only the first 100.'
                print
                interval_errors = interval_errors[:100]
            for fragment in sorted(interval_errors,
                                   key=lambda x: (x.name, x.interval.start)):
                print '  error: {0} [{1:.3f}, {2:.3f}]'.format(
                    fragment.name,
                    fragment.interval.start, fragment.interval.end)
        if filename_error:
            print banner('unknown filenames found in {0}'
                                      .format(fname))
            if len(filename_errors) > 100:
                print 'There were more than 100 filename errors found.'
                print 'Printing only the first 100.'
                print
                filename_errors = filename_errors[:100]
            for fragment in sorted(filename_errors,
                                   key=lambda x: (x.name, x.interval.start)):
                print '  error: {0}'.format(fragment.name)
        if not truncate and errors_found:
            print 'There were errors in {0}. Use option -f to automatically skip invalid intervals.'.format(fname)
            sys.exit()
    return disc


def _match_sub(disc_clsdict, gold_clsdict, phn_corpus, names, label,
               verbose, n_jobs):
    em = eval_from_psets
    if verbose:
        print '  matching ({2}): subsampled {0} files in {1} sets'\
            .format(sum(map(len, names)), len(names), label)
    with verb_print('  matching ({0}): prepping psets'.format(label),
                             verbose, True, True, True):
        pdiscs = [make_pdisc(disc_clsdict.restrict(fs, True),
                             False, False)
                  for fs in names]
        pgolds = [make_pgold(gold_clsdict.restrict(fs, True),
                             False, False)
                  for fs in names]
        psubs = [make_psubs(disc_clsdict.restrict(fs, True),
                            phn_corpus, 3, 20, False, False)
                 for fs in names]
    with verb_print('  matching ({0}): calculating scores'
                             .format(label), verbose, False, True, False):
        tp, tr = izip(*Parallel(n_jobs=n_jobs,
                                verbose=5 if verbose else 0,
                                pre_dispatch='n_jobs')
                      (delayed(em)(pdisc, pgold, psub)
                      for pdisc, pgold, psub in zip(pdiscs, pgolds, psubs)))
    tp, tr = np.fromiter(tp, dtype=np.double), np.fromiter(tr, dtype=np.double)
    tp, tr = praggregate(tp, tr)
    return tp, tr


def match(disc_clsdict, gold_clsdict, phn_corpus,
          fragments_within, fragments_cross,
          dest, verbose, n_jobs):
    if verbose:
        print banner('MATCHING')
    pc, rc = _match_sub(disc_clsdict, gold_clsdict, phn_corpus,
                        fragments_cross, 'cross', verbose, n_jobs)
    fc = np.fromiter((fscore(pc[i], rc[i]) for i in xrange(pc.shape[0])), dtype=np.double)

    pw, rw = _match_sub(disc_clsdict, gold_clsdict, phn_corpus,
                        fragments_within, 'within', verbose, n_jobs)
    fw = np.fromiter((fscore(pw[i], rw[i]) for i in xrange(pw.shape[0])), dtype=np.double)
    with open(path.join(dest, 'matching'), 'w') as fid:
        fid.write(pretty_score_f(pc, rc, fc, 'match total',
                                 len(fragments_cross),
                                 sum(map(len, fragments_cross))))
        fid.write('\n')

        fid.write(pretty_score_f(pw, rw, fw, 'match within-speaker only',