class IntervalDB(object):
    """Holds collection of fragments and makes them easily searchable.

    Parameters
    ----------
    mapping : dict from string to sorted list of Intervals

    Attributes
    ----------
    starts, ends : list of floats

    Methods
    -------
    find(filename, interval)
        Find all the intervals that overlap with the query interval.
    is_covered(filename, interval)
        Determine if an interval is covered.
    largest_overlap(filename, interval)
        Find the interval with the largest overlap.

    """
    def __init__(self, mapping):
        starts = {}
        ends = {}
        for fname in mapping:
            starts[fname], ends[fname] = zip(*mapping[fname])
        self.starts = starts
        self.ends = ends

    def __eq__(self, other):
        if isinstance(other, IntervalDB):
            return self.starts == other.starts and self.ends == other.ends
        else:
            return False

    def __ne__(self, other):
        return not self.__eq__(other)

    def __repr__(self):
        return '<IntervalDB {0} entries>'.format(
            sum(map(len, self.starts.itervalues())))

    def __str__(self):
        return 'IntervalDB\n- starts: {0}\n- ends: {1}'.format(
            pformat(self.starts), pformat(self.ends))

    def __len__(self):
        return sum(map(len, self.starts.itervalues()))

    def find(self, fname, interval):
        """
        Find all the intervals that overlap with the query interval.

        Parameters
        ----------
        fname : string
        interval : Interval

        Returns
        -------
        r : iterator over Intervals
            intervals with overlap

        Raises
        ------
        KeyError
            if fname is not a known key
        """
        starts = self.starts[fname]
        ends = self.ends[fname]
        qstart, qend = interval.start, interval.end
        left = max(bisect.bisect_left(starts, qstart) - 1, 0)
        right = bisect.bisect_right(starts, qend)
        return (Interval(start, end)
                for start, end in zip(starts[left:right],
                                      ends[left:right])
                if Interval(start, end).overlap(interval) > 0)

    def is_covered(self, fname, interval):
        """
        Determine if an interval is covered.

        Parameters
        ----------
        fname : string
        interval : Interval

        Returns
        -------
        bool

        """
        try:
            b = any(interval.start >= start and interval.end <= end
                    for start, end in self.find(fname, interval))
        except KeyError:
            b = False
        return b

    def largest_overlap(self, fname, interval):
        """Return the interval that has the largest overlap with the query
        interval.

        Parameters
        ----------
        fname : string
        interval : Interval

        Returns
        -------
        start, end : float
            start and end points of interval with the largest overlap

        Raises
        ------
        KeyError
            if fname is not a known key
        ValueError
            if no interval is found
        """
        return max(self.find(fname, interval),
                   key=lambda x: interval.overlap(x)) def load_annotation(filename):
    """Read in a file with annotations in the Buckeye style.

    The phone file must be formatted with the following, space-separated
    columns:

    identifier start stop symbol

    Parameters
    ----------
    filename : string

    Returns
    -------
    r : list of list of FragmentToken

    """
    with open(filename, 'r') as fid:
        contents = fid.read()
    return read_annotation(contents) class ReadError(Exception):
    pass def load_corpus_txt(filename):
    """Read file with phone annotation and convert into Corpus object.

    The phone file must be formatted with the following, space-separated
    columns:
    identifier start stop symbol

    Parameters
    ----------
    filename : string

    Returns
    -------
    c : Corpus

    """
    return tokenlists_to_corpus(load_annotation(filename)) def read_classfile(contents):
    """Read in class file.

    Parameters
    ----------
    contents : string

    Returns
    -------
    r : dict from ClassID to list of FragmentToken

    """
    classp = re.compile(r"^Class (?P<classID>\d+)(?: (?P<mark>.+))?$")
    r = {}
    curr = []  # list of FragmentTokens without mark
    curr_class = None

    for lineno, line in enumerate(contents.split('\n')):
        m = re.match(classp, line)
        if m:  # on a line with a class label
            if curr_class is None:
                curr_class = ClassID(int(m.group('classID')),
                                     m.group('mark'))
            else:
                raise ValueError('new class while reading class')
        else:  # on an interval line or a whitespace line
            if len(line.strip()) > 0:
                split = line.strip().split(' ')
                name = split[0]
                start = float(split[1])
                end = float(split[2])
                interval = Interval(start, end)
                curr.append(FragmentToken(name, interval, None))
            else:  # whitespace line, reset
                if curr_class is None:
                    continue
                    # if lineno == 0:
                    #     continue
                    # print lineno, line
                    # raise ValueError('attempting to end reading class '
                    #                  'while not reading class in line {0}'
                    #                  .format(lineno))
                r[curr_class] = tuple(curr)
                curr = []
                curr_class = None
    if not curr_class is None:
        r[curr_class] = tuple(curr)
    return r def annotate_classes(clsdict, corpus, split=None):
    new = {}  # with annotation
    errors = []
    check_split = not (split is None)
    for classID, tokenlist in clsdict.iteritems():
        newtokens = []
        for token in tokenlist:
            filename = token.name
            interval = token.interval
            if check_split and not split.is_covered(filename, interval):
                errors.append(token)
                try:
                    finterval = split.largest_overlap(filename, interval)
                    qstart, qend = interval
                    fstart, fend = finterval
                    if fstart != qstart or fstart != qend:
                        newstart = max(fstart, qstart)
                        newend = min(fend, qend)
                        interval = Interval(newstart, newend)
                except KeyError:
                    continue
                except ValueError:
                    continue
            try:
                annot = tuple(corpus.annotation(filename, interval))
            except:
                continue
            newtokens.append(FragmentToken(filename, interval, annot))
        if len(newtokens) > 0:
            newtokens = tuple(newtokens)
            new[classID] = newtokens
    return ClassDict(new), errors def tokenlists_to_corpus(tokenlists):
    """Convert a list of tokens to Corpus object

    Parameters
    ----------
    tokenlists : list of list of FragmentToken

    Returns :
    c : Corpus

    """
    fas = []  # FileAnnotations
    for tokenlist in tokenlists:
        fname = tokenlist[0].name
        fas.append(SegmentAnnotation(fname, tokenlist))
    return Corpus(fas) def read_split_single(s):
    mapping = defaultdict(list)
    for line in s.split('\n'):
        if line == '':
            continue
        name, start, end = line.strip().split(' ')
        mapping[name].append((float(start), float(end)))
    return IntervalDB({k: sorted(v) for k, v in mapping.iteritems()}) def read_split_multiple(s):
    return [read_split_single(s0) for s0 in s.split('\n\n')] def read_annotation(contents):
    ID_prev = None
    interval_prev = None
    r = []
    tokenlist_curr = []
    for line_idx, line in enumerate(contents.split('\n')):
        if line == '':
            continue
        try:
            ID_curr, start, stop, mark = line.strip().split(' ')
        except ValueError:
            raise ReadError('badly formatted line {1}: {0}'
                            .format(line, line_idx))
        try:
            start = float(start)
            stop = float(stop)
        except ValueError:
            raise ReadError('could not convert string to float in line {1}: {0}'
                            .format(line, line_idx))
        try:
            interval_curr = Interval(start, stop)
        except ValueError:
            raise ReadError('invalid interval in line {0}: ({1:.3f} {2:.3f})'
                            .format(line_idx, start, stop))

        token = FragmentToken(ID_curr, interval_curr, mark)

        if ID_prev is None:
            tokenlist_curr = [token]
            ID_prev = ID_curr
        elif ID_prev == ID_curr:
            if interval_prev.is_left_adjacent_to(interval_curr):
                tokenlist_curr.append(token)
            else:
                r.append(tokenlist_curr)
                tokenlist_curr = [token]
        else:  # ID_prev != ID_curr
            r.append(tokenlist_curr)
            tokenlist_curr = [token]
            ID_prev = ID_curr
        interval_prev = interval_curr
    r.append(tokenlist_curr)
    return r class ClassDict(collections.Mapping):
    def __init__(self, clsdict):
    def __contains__(self, key):
    def __getitem__(self, key):
    def __iter__(self):
    def __len__(self):
    def __str__(self):
    def __eq__(self, other):
    def __ne__(self, other):
    def pretty(self):
    def iter_fragments(self, with_class=False):
    def iter_pairs(self, within, order):
    def restrict(self, interval_db, remove_singletons=False): def token_cmp(token1, token2): class SegmentAnnotation(collections.Sequence):
    """
    Annotation for contiguous time segment.

    This class represents a contiguous sequence of annotations. It contains
    methods for efficiently finding the annotation corresponding to a temporal
    interval.

    Parameters
    ----------
    name : string
        Identifier for the annotation.
    tokens : list of Fragment objects
        The sequence of annotations.

    Attributes
    ----------
    interval : Interval
        The temporal interval that is covered.

    Raises
    ------
    ValueError
        If tokens are not contiguous or don't all have the same name.

    """
    def __init__(self, name, tokens):
        self.name = name
        self.tokens = SortedList(tokens, key=cmp_to_key(token_cmp))
        if len(self.tokens) == 0:
            self.interval = None
        else:
            self.interval = Interval(self.tokens[0].interval.start,
                                     self.tokens[-1].interval.end)
        if not all(t1.interval.end == t2.interval.start
                   for t1, t2 in zip(self.tokens[:-1], self.tokens[1:])):
            raise ValueError('Non-contiguous tokens.')

    def __len__(self):
        return len(self.tokens)

    def __iter__(self):
        return iter(self.tokens)

    def __getitem__(self, i):
        return self.tokens[i]

    def __repr__(self):
        return '%s(%r)' % (self.__class__.__name__, self.__dict__)

    def __str__(self):
        return '<SegmentAnnotation {0} {1} {2} tokens>'.format(
            self.name, str(self.interval), len(self.tokens))

    def __eq__(self, other):
        if self.name != other.name:
            return False
        if self.interval != other.interval:
            return False
        if len(self.tokens) != len(other.tokens):
            return False
        return all(t1 == t2 for t1, t2 in zip(self.tokens, other.tokens))

    def __ne__(self, other):
        return not self.__eq__(other)

    def restrict(self, interval_db):
        """
        Restrict the SegmentAnnotation to a set of Intervals.

        Returns a new SegmentAnnotation object with only the FragmentTokens
        that are fully covered in `interval_db`.

        Parameters
        ----------
        interval_db : IntervalDB

        Returns
        -------
        SegmentAnnotation
            New SegmentAnnotation object.

        """
        return SegmentAnnotation(self.name,
                                 [f for f in self.tokens
                                  if interval_db.is_covered(f.name, f.interval)])

    def annotation_at_interval(self, interval):
        """
        Get the annotation corresponding to an interval.

        Parameters
        ----------
        interval : Interval

        Returns
        -------
        s : list of strings
            Annotation symbols covered by the interval.
        """
        return tuple([x.mark for x in self.tokens_at_interval(interval)])

    def tokens_at_interval(self, interval):
        """
        Get the annotation tokens corresponding to an interval.

        Parameters
        ----------
        interval : Interval

        Returns
        -------
        tuple of FragmentTokens
            FragmentTokens covered by the interval.
        """
        if len(self.tokens) > 0:
            name = self.tokens[0].name
        else:
            return tuple()
        dummy_token = FragmentToken(name, interval, None)
        try:
            start = self.tokens.index_ge(dummy_token)
        except ValueError:
            return tuple()
        try:
            stop = self.tokens.index_gt(dummy_token)
        except ValueError:
            stop = len(self.tokens)
        return tuple([x for x in self.tokens[start:stop]]) class Interval(object):
    """
    Time interval.

    This class represents a temporal interval. It contains methods for
    determining whether two intervals overlap and for calculating that overlap.

    The default values for the keyword arguments assume that time is measured
    in seconds and set the minimum overlap to 30 milliseconds and the minimum
    overlap fraction to 0.5.

    Parameters
    ----------
    start : float
        Start of the interval.
    end : float
        End  of the interval.
    minimum_overlap : float, optional
        Minimum time for two intervals to be considered to overlap.
    minimum_overlap_fraction : float, optional
        Minimum fraction of either interval to be considered an overlap.


    Methods
    -------
    length()
        The length of the interval.
    overlap(interval)
        Calculate overlap with another interval.
    overlaps_with(interval)
        Determine whether there is an overlap.
    contains(interval)
        Determine whether another interval is contained in this one.
    is_adjacent(interval)
        Determine whether another interval is adjacent to this one.
    is_left_adjacent_to(interval)
        Determine whether this interval is immediately to the left of another.
    is_right_adjacent_to(interval)
        Determine whether this interval is immediately to the right of another.

    Raises
    ------
    ValueError
        If start > end or start < 0 or end < 0

    """
    def __init__(self, start, end,
                 minimum_overlap=0.03, minimum_overlap_fraction=0.5):
        if end < start:
            raise ValueError('end must be greater than start')
        if start < 0 or end < 0:
            raise ValueError('start and end must be non-negative')
        self.start = start
        self.end = end
        self.minimum_overlap = minimum_overlap
        self.minimum_overlap_fraction = minimum_overlap_fraction
        self._length = self.end - self.start

    def __repr__(self):
        return '[{0},{1}]'.format(self.start, self.end)

    def __str__(self):
        return '[{0},{1}]'.format(self.start, self.end)

    def __eq__(self, other):
        if isinstance(other, Interval):
            return self.start == other.start and self.end == other.end
        else:
            return False

    def __hash__(self):
        return (hash(self.start) << 1) ^ hash(self.end)

    def __ne__(self, other):
        return not self.__eq__(other)

    def __iter__(self):
        yield self.start
        yield self.end

    def length(self):
        """
        Calculate the length of the interval.

        Returns
        -------
        float

        """
        return self._length

    def overlap(self, other):
        """
        Calculate overlap with another interval.

        Parameters
        ----------
        other : Interval

        Returns
        -------
        float

        """
        if self.end < other.start:
            return 0.
        if self.start > other.end:
            return 0.
        return min(self.end, other.end) - max(self.start, other.start)

    def overlaps_with(self, other):
        """
        Determine whether there is an overlap.

        Overlap is calculated in accordance with the minimum
        and maximum overlap parameters. An overlap is confirmed if the
        overlapping time is at least `minimum_overlap` or at least
        `minimum_overlap_fraction` of either interval.


        Parameters
        ----------
        other : Interval

        Returns
        -------
        bool

        Examples
        --------
        >>> i1 = Interval(0, 2)
        >>> i2 = Interval(1, 3)
        >>> i1.overlaps_with(i2)
        True
        >>> i1 = Interval(0.0, 1.0, minimum_overlap=0.5, minimum_overlap_fraction=0.5)
        >>> i2 = Interval(0.9, 3.0, minimum_overlap=0.5, minimum_overlap_fraction=0.5)
        >>> i1.overlaps_with(i2)
        False

        """
        if self.minimum_overlap != other.minimum_overlap:
            raise ValueError('Attempting to calculate overlap on incomparable '
                             'intervals. Make sure that `minimum_overlap` on '
                             'both intervals is the same.')
        if self.minimum_overlap_fraction != other.minimum_overlap_fraction:
            raise ValueError('Attempting to calculate overlap on incomparable '
                             'intervals. Make sure that `minimum_overlap_fraction`'
                             ' on both intervals is the same.')

        over = self.overlap(other)
        if np.isclose(over, 0.0):
            return False
        if over > self.minimum_overlap:
            return True
        if over > self.minimum_overlap_fraction * other.length():
            return True
        if over > self.minimum_overlap_fraction * self.length():
            return True
        return False

    def contains(self, other):
        """
        Determine whether another interval is contained in this one.

        Parameters
        ----------
        other : Interval

        Returns
        -------
        bool

        """
        return self.start <= other.start and self.end >= other.end

    def is_adjacent(self, other):
        """
        Determine whether another interval is adjacent to this one.

        Parameters
        ----------
        other : Interval

        Returns
        -------
        bool

        """
        return self.is_left_adjacent_to(other) or \
            self.is_right_adjacent_to(other)


    def is_left_adjacent_to(self, other):
        """
        Determine whether this interval is immediately to the left of another.

        Parameters
        ----------
        other : Interval

        Returns
        -------
        bool

        """
        return np.isclose(self.end, other.start)

    def is_right_adjacent_to(self, other):
        """
        Determine whether this interval is immediately to the right of another.

        Parameters
        ----------
        other : Interval

        Returns
        -------
        bool

        """
        return np.isclose(other.end, self.start) class Corpus(collections.Mapping):
    """
    Corpus annotation.

    Holds full segmental annotation of a corpus with fast search on annotation
    for intervals.

    Parameters
    ----------
    segment_annotations : list of SegmentAnnotations

    """
    def __init__(self, segment_annotations=None):
        self.segment_annotations = {}
        if segment_annotations is None:
            segment_annotations = []
        for fa in segment_annotations:
            try:
                self.segment_annotations[fa.name].insert(fa)
            except KeyError:
                self.segment_annotations[fa.name] = \
                    SortedList([fa], key=cmp_to_key(annotation_cmp))
        self._cache = {}

    def __eq__(self, other):
        if not isinstance(other, Corpus):
            return False
        if sorted(self.keys()) != sorted(other.keys()):
            return False
        for k in sorted(self.keys()):
            s = self.segment_annotations[k]
            o = other.segment_annotations[k]
            if len(s) != len(o):
                return False
            for i in xrange(len(s)):
                if s[i] != o[i]:
                    return False
        return True

    def __len__(self):
        return len(self.segment_annotations)

    def __getitem__(self, key):
        return self.segment_annotations[key]

    def __iter__(self):
        return iter(self.segment_annotations)

    def __repr__(self):
        return '<Corpus - {0} segments, {1} names>'.format(
            sum(map(len, self.values())), len(self))

    def __str__(self):
        return pformat(self.segment_annotations)

    def clear(self):
        self._cache = {}

    def iter_fragments(self):
        return flatten(self.iter_segments())

    def iter_segments(self):
        return flatten(self.itervalues())

    def restrict(self, interval_db):
        """
        Return a new Corpus, containing only the annotations for the
        specified names.

        Parameters
        ----------
        names : list of strings
            List of identifiers to restrict the Corpus to. Supplied identifiers
            must be valid (i.e. must be in self.keys()).

        Returns
        -------
        Corpus
            New Corpus object.
        """
        sa = []
        for k in self:
            for s in self[k]:
                r = s.restrict(interval_db)
                if len(r.tokens) > 0:
                    sa.append(r)
        return Corpus(sa)

    def annotation(self, name, interval):
        """
        Find the annotation covering an interval.

        Parameters
        ----------
        name : string
            Identifier.
        interval : Interval
            Time segment.

        Returns
        -------
        list of strings
            Symbols covered by the interval.
        """
        return tuple(x.mark for x in self.tokens(name, interval))

    def annotation_exact(self, name, interval):
        return tuple(x.mark for x in self.tokens_exact(name, interval))

    def tokens_exact(self, name, interval):
        tokens = self.tokens(name, interval)
        if tokens[0].interval.start != interval.start and \
           tokens[-1].interval.end != interval.end:
            raise ValueError('exact tokens not found')
        return tokens

    def tokens(self, name, interval):
        """
        Find the FragmentTokens covering an interval.

        Parameters
        ----------
        name : string
            Identifier.
        interval : Interval
            Time segment.

        Returns
        -------
        list of tokens
            FragmentTokens covered by the interval.
        """
        key = (name, interval)
        if not key in self._cache:
            try:
                fa_for_filename = self[name]
            except KeyError:
                raise KeyError('no such name: {0}'.format(name))
            dummy_token = FragmentToken(name, interval, None)
            try:
                fa = fa_for_filename.find_le(dummy_token)
            except ValueError:
                raise ValueError('interval not found: {0}'.format(str(interval)))
            if (fa.interval.overlap(interval)) > 0:
                self._cache[key] = fa.tokens_at_interval(interval)
            else:
                raise ValueError('interval not found: {0}'.format(str(interval)))
        return self._cache[key]
import cPickle as pickle
import pytest
from tde.data.interval import IntervalDB
from tde.util.reader import load_annotation, ReadError, \
    load_corpus_txt, read_classfile, annotate_classes, \
    tokenlists_to_corpus, read_split_single, read_split_multiple, read_annotation
from tde.data.classes import ClassID
from tde.data.fragment import FragmentToken
from tde.data.segment_annotation import SegmentAnnotation
from tde.data.interval import Interval
from tde.data.corpus import Corpus
                     SegmentAnnotation('f2', tokens[4:])])

    def test_small(self):
        assert (self.clsdict_e == read_classfile(self.tiny_classes))

    def test_corpus(self):
        assert (self.corpus ==
                tokenlists_to_corpus(read_annotation(self.tiny_corpus)))

    def test_annotate(self):
        assert (self.clsdict_a ==
                annotate_classes(read_classfile(self.tiny_classes),
                                 tokenlists_to_corpus(read_annotation(self.tiny_corpus))))


class TestReadSplit(object):
    multi = """a 0.000 1.000
a 2.000 3.000
a 4.000 5.000

b 0.000 1.000
b 2.000 3.000
b 4.000 5.000
"""
    single = """a 0.000 1.000
a 2.000 3.000
a 4.000 5.000
b 0.000 1.000
b 2.000 3.000
b 4.000 5.000
"""

    multi_split = [IntervalDB({'a': [(0.0, 1.0), (2.0, 3.0), (4.0, 5.0)]}),
                   IntervalDB({'b': [(0.0, 1.0), (2.0, 3.0), (4.0, 5.0)]})]
    single_split = IntervalDB({'a': [(0.0, 1.0), (2.0, 3.0), (4.0, 5.0)],
                           'b': [(0.0, 1.0), (2.0, 3.0), (4.0, 5.0)]})

    def test_multi_multi(self):
        assert (read_split_multiple(self.multi) ==
                self.multi_split)
    def test_multi_single(self):
        assert (read_split_multiple(self.single) ==
                [self.single_split])
    def test_single_multi(self):
        assert (read_split_single(self.multi) ==
                self.single_split)
    def test_single_single(self):
        assert (read_split_single(self.single) ==
                self.single_split)


SKIP_HUGE = True

class TestReadAnnotationFile(object):
    def test_read_small(self):
        contents = """f1 0.000 0.100 a
f1 0.100 0.200 r
f1 0.200 0.300 m
f1 0.300 0.400 s
f1 0.400 0.500 a
f1 0.700 0.800 w
f1 0.800 0.900 o
f1 0.900 1.000 r
f1 1.000 1.100 m
f1 1.100 1.200 s
f1 1.200 1.300 a
f2 0.100 0.200 w
f2 0.200 0.300 o
f2 0.300 0.400 r
f2 0.400 0.500 d
f2 0.500 0.600 s
"""
        tokens = [FragmentToken('f1', Interval(0.0, 0.1), 'a'),
                  FragmentToken('f1', Interval(0.1, 0.2), 'r'),
                  FragmentToken('f1', Interval(0.2, 0.3), 'm'),
                  FragmentToken('f1', Interval(0.3, 0.4), 's'),
                  FragmentToken('f1', Interval(0.4, 0.5), 'a'),
                  FragmentToken('f1', Interval(0.7, 0.8), 'w'),
                  FragmentToken('f1', Interval(0.8, 0.9), 'o'),
                  FragmentToken('f1', Interval(0.9, 1.0), 'r'),
                  FragmentToken('f1', Interval(1.0, 1.1), 'm'),
                  FragmentToken('f1', Interval(1.1, 1.2), 's'),
                  FragmentToken('f1', Interval(1.2, 1.3), 'a'),
                  FragmentToken('f2', Interval(0.1, 0.2), 'w'),
                  FragmentToken('f2', Interval(0.2, 0.3), 'o'),
                  FragmentToken('f2', Interval(0.3, 0.4), 'r'),
                  FragmentToken('f2', Interval(0.4, 0.5), 'd'),
                  FragmentToken('f2', Interval(0.5, 0.6), 's')]
        corpus = Corpus([SegmentAnnotation('f1', tokens[0:5]),
                         SegmentAnnotation('f1', tokens[5:11]),
                         SegmentAnnotation('f2', tokens[11:])])

        assert ([tokens[0:5], tokens[5:11], tokens[11:]] ==
                read_annotation(contents))
        assert (tokenlists_to_corpus(read_annotation(contents)) ==
                corpus)

    def test_huge(self):
        with open('tests/mockdata/dev.phn.annotation.pkl', 'rb') as fid:
            corpus = pickle.load(fid)
        assert (corpus == load_annotation('tests/mockdata/dev.phn'))

    def test_badline(self):
        with pytest.raises(ReadError):
            load_annotation('tests/mockdata/mockcorpus_small_badline.phn')

    def test_badinterval(self):
        with pytest.raises(ReadError):
            load_annotation('tests/mockdata/mockcorpus_small_badinterval.phn')

    def test_badfloat(self):
        with pytest.raises(ReadError):
            load_annotation('tests/mockdata/mockcorpus_small_badfloat.phn')

class TestCorpusAnnotationFromPhoneFile(object):


    def test_dev(self):
        fname = 'tests/mockdata/dev.phn'
        corpus1 = tokenlists_to_corpus(load_annotation(fname))

        corpus2 = load_corpus_txt(fname)