def simple_tokenize(name):
    """Simple tokenizer function to be used with the normalizers."""
    last_names, first_names = name.split(',')
    last_names = _RE_NAME_TOKEN_SEPARATOR.split(last_names)
    first_names = _RE_NAME_TOKEN_SEPARATOR.split(first_names)

    first_names = [NameToken(n) if len(n) > 1 else NameInitial(n)
                   for n in first_names if n]
    last_names = [NameToken(n) if len(n) > 1 else NameInitial(n)
                  for n in last_names if n]
    return {'lastnames': last_names,
            'nonlastnames': first_names}
def simple_tokenize ( name ) : last_names , first_names = name . split ( ',' ) last_names = _RE_NAME_TOKEN_SEPARATOR . split ( last_names ) first_names = _RE_NAME_TOKEN_SEPARATOR . split ( first_names ) first_names = [ NameToken ( n ) if len ( n ) > 1 else NameInitial ( n ) for n in first_names if n ] last_names = [ NameToken ( n ) if len ( n ) > 1 else NameInitial ( n ) for n in last_names if n ] return { 'lastnames' : last_names , 'nonlastnames' : first_names }
Simple tokenizer function to be used with the normalizers.