def load_conll(cls, fname):
        """
        The CONLL file must have a tab delimited header, for example::

            # description   tags
            Alice
            Hello   t1
            my      t2
            name    t3
            is      t4
            alice   t5

            Bob
            I'm     t1
            bob     t2

        Here, the fields are `description` and `tags`. The first instance has the label `Alice` and the
        description `['Hello', 'my', 'name', 'is', 'alice']` and the tags `['t1', 't2', 't3', 't4', 't5']`.
        The second instance has the label `Bob` and the description `["I'm", 'bob']` and the tags `['t1', 't2']`.

        :param fname: The CONLL formatted file from which to load the dataset

        :return: loaded Dataset instance
        """
        def process_cache(cache, fields):
            cache = [l.split() for l in cache if l]
            if not cache:
                return None
            fields['label'].append(cache[0][0])
            instance = {k: [] for k in fields if k != 'label'}
            for l in cache[1:]:
                for i, k in enumerate(fields):
                    if k != 'label':
                        instance[k].append(None if l[i] == '-' else l[i])
            for k, v in instance.items():
                fields[k].append(v)

        cache = []

        with open(fname) as f:
            header = f.next().strip().split('\t')
            header[0] = header[0].lstrip('# ')
            fields = OrderedDict([(head, []) for head in header])
            fields['label'] = []
            for line in f:
                line = line.strip()
                if line:
                    cache.append(line)
                else:
                    # met empty line, process cache
                    process_cache(cache, fields)
                    cache = []
            if cache:
                process_cache(cache, fields)
        return cls(fields)
def load_conll ( cls , fname ) : def process_cache ( cache , fields ) : cache = [ l . split ( ) for l in cache if l ] if not cache : return None fields [ 'label' ] . append ( cache [ 0 ] [ 0 ] ) instance = { k : [ ] for k in fields if k != 'label' } for l in cache [ 1 : ] : for i , k in enumerate ( fields ) : if k != 'label' : instance [ k ] . append ( None if l [ i ] == '-' else l [ i ] ) for k , v in instance . items ( ) : fields [ k ] . append ( v ) cache = [ ] with open ( fname ) as f : header = f . next ( ) . strip ( ) . split ( '\t' ) header [ 0 ] = header [ 0 ] . lstrip ( '# ' ) fields = OrderedDict ( [ ( head , [ ] ) for head in header ] ) fields [ 'label' ] = [ ] for line in f : line = line . strip ( ) if line : cache . append ( line ) else : process_cache ( cache , fields ) cache = [ ] if cache : process_cache ( cache , fields ) return cls ( fields )
The CONLL file must have a tab delimited header, for example::

            # description   tags
            Alice
            Hello   t1
            my      t2
            name    t3
            is      t4
            alice   t5

            Bob
            I'm     t1
            bob     t2

        Here, the fields are `description` and `tags`. The first instance has the label `Alice` and the
        description `['Hello', 'my', 'name', 'is', 'alice']` and the tags `['t1', 't2', 't3', 't4', 't5']`.
        The second instance has the label `Bob` and the description `["I'm", 'bob']` and the tags `['t1', 't2']`.

        :param fname: The CONLL formatted file from which to load the dataset

        :return: loaded Dataset instance