postal/normalize.py
def normalized_tokens(s, string_options=DEFAULT_STRING_OPTIONS,
                      token_options=DEFAULT_TOKEN_OPTIONS,
                      strip_parentheticals=True, whitespace=False,
                      languages=None):
    '''
    Normalizes a string, tokenizes, and normalizes each token
    with string and token-level options.

    This version only uses libpostal's deterministic normalizations
    i.e. methods with a single output. The string tree version will
    return multiple normalized strings, each with tokens.

    Usage:
        normalized_tokens(u'St.-Barthélemy')
    '''
    s = safe_decode(s)
    normalized_tokens = _normalize.normalized_tokens(s, string_options, token_options, whitespace, languages=languages)

    if strip_parentheticals:
        normalized_tokens = remove_parens(normalized_tokens)

    return [(s, token_types.from_id(token_type)) for s, token_type in normalized_tokens]
def normalized_tokens(s, string_options=DEFAULT_STRING_OPTIONS,
                      token_options=DEFAULT_TOKEN_OPTIONS,
                      strip_parentheticals=True, whitespace=False,
                      languages=None):
    '''
    Normalizes a string, tokenizes, and normalizes each token
    with string and token-level options.

    This version only uses libpostal's deterministic normalizations
    i.e. methods with a single output. The string tree version will
    return multiple normalized strings, each with tokens.

    Usage:
        normalized_tokens(u'St.-Barthélemy')
    '''
    s = safe_decode(s)
    normalized_tokens = _normalize.normalized_tokens(s, string_options, token_options, whitespace, languages=languages)

    if strip_parentheticals:
        normalized_tokens = remove_parens(normalized_tokens)

    return [(s, token_types.from_id(token_type)) for s, token_type in normalized_tokens]
def normalized_tokens ( s , string_options = DEFAULT_STRING_OPTIONS , token_options = DEFAULT_TOKEN_OPTIONS , strip_parentheticals = True , whitespace = False , languages = None ) : s = safe_decode ( s ) normalized_tokens = _normalize . normalized_tokens ( s , string_options , token_options , whitespace , languages = languages ) if strip_parentheticals : normalized_tokens = remove_parens ( normalized_tokens ) return [ ( s , token_types . from_id ( token_type ) ) for s , token_type in normalized_tokens ]
Normalizes a string, tokenizes, and normalizes each token
    with string and token-level options.

    This version only uses libpostal's deterministic normalizations
    i.e. methods with a single output. The string tree version will
    return multiple normalized strings, each with tokens.

    Usage:
        normalized_tokens(u'St.-Barthélemy')
Normalizes a string tokenizes and normalizes each token with string and token - level options .