<s> from __future__ import unicode_literals <EOL> from itertools import chain <EOL> from operator import methodcaller <EOL> import regex as re <EOL> from six . moves import zip_longest <EOL> from dateparser . utils import normalize_unicode <EOL> DATEUTIL_PARSER_HARDCODED_TOKENS = [ "<STR_LIT::>" , "<STR_LIT:.>" , "<STR_LIT:U+0020>" , "<STR_LIT:->" , "<STR_LIT:/>" ] <EOL> DATEUTIL_PARSERINFO_KNOWN_TOKENS = [ "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT:a>" , "<STR_LIT:p>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" ] <EOL> ALWAYS_KEEP_TOKENS = [ "<STR_LIT:+>" ] + DATEUTIL_PARSER_HARDCODED_TOKENS <EOL> class UnknownTokenError ( Exception ) : <EOL> pass <EOL> class Dictionary ( object ) : <EOL> _split_regex_cache = { } <EOL> _sorted_words_cache = { } <EOL> def __init__ ( self , language_info , settings = None ) : <EOL> dictionary = { } <EOL> self . _settings = settings <EOL> self . info = language_info <EOL> if '<STR_LIT>' in language_info : <EOL> skip = map ( methodcaller ( '<STR_LIT>' ) , language_info [ '<STR_LIT>' ] ) <EOL> dictionary . update ( zip_longest ( skip , [ ] , fillvalue = None ) ) <EOL> if '<STR_LIT>' in language_info : <EOL> pertain = map ( methodcaller ( '<STR_LIT>' ) , language_info [ '<STR_LIT>' ] ) <EOL> dictionary . update ( zip_longest ( pertain , [ ] , fillvalue = None ) ) <EOL> for word in [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' ] : <EOL> translations = map ( methodcaller ( '<STR_LIT>' ) , language_info [ word ] ) <EOL> dictionary . update ( zip_longest ( translations , [ ] , fillvalue = word ) ) <EOL> dictionary . update ( zip_longest ( ALWAYS_KEEP_TOKENS , ALWAYS_KEEP_TOKENS ) ) <EOL> dictionary . update ( zip_longest ( map ( methodcaller ( '<STR_LIT>' ) , <EOL> DATEUTIL_PARSERINFO_KNOWN_TOKENS ) , <EOL> DATEUTIL_PARSERINFO_KNOWN_TOKENS ) ) <EOL> self . _dictionary = dictionary <EOL> self . _no_word_spacing = language_info . get ( '<STR_LIT>' , False ) <EOL> def __contains__ ( self , key ) : <EOL> if key in self . _settings . SKIP_TOKENS : <EOL> return True <EOL> return self . _dictionary . __contains__ ( key ) <EOL> def __getitem__ ( self , key ) : <EOL> if key in self . _settings . SKIP_TOKENS : <EOL> return None <EOL> return self . _dictionary . __getitem__ ( key ) <EOL> def __iter__ ( self ) : <EOL> return chain ( self . _settings . SKIP_TOKENS , iter ( self . _dictionary ) ) <EOL> def split ( self , string , keep_formatting ) : <EOL> """<STR_LIT>""" <EOL> if not string : <EOL> return string <EOL> regex = self . _get_split_regex_cache ( ) <EOL> match = regex . match ( string ) <EOL> if not match : <EOL> return [ string ] if self . _should_capture ( string , keep_formatting ) else [ ] <EOL> unparsed , known , unknown = match . groups ( ) <EOL> splitted = [ known ] if self . _should_capture ( known , keep_formatting ) else [ ] <EOL> if unparsed and self . _should_capture ( unparsed , keep_formatting ) : <EOL> splitted = [ unparsed ] + splitted <EOL> if unknown : <EOL> splitted . extend ( self . split ( unknown , keep_formatting ) ) <EOL> return splitted <EOL> def _should_capture ( self , token , keep_formatting ) : <EOL> return ( <EOL> keep_formatting or <EOL> ( token in ALWAYS_KEEP_TOKENS ) or <EOL> re . match ( r"<STR_LIT>" , token , re . U ) <EOL> ) <EOL> def _get_sorted_words_from_cache ( self ) : <EOL> if ( <EOL> self . _settings . registry_key not in self . _sorted_words_cache or <EOL> self . info [ '<STR_LIT:name>' ] not in self . _sorted_words_cache [ self . _settings . registry_key ] <EOL> ) : <EOL> self . _sorted_words_cache [ self . _settings . registry_key ] = { <EOL> self . info [ '<STR_LIT:name>' ] : sorted ( [ key for key in self ] , key = len , reverse = True ) <EOL> } <EOL> return self . _sorted_words_cache [ self . _settings . registry_key ] [ self . info [ '<STR_LIT:name>' ] ] <EOL> def _get_split_regex_cache ( self ) : <EOL> if ( <EOL> self . _settings . registry_key not in self . _split_regex_cache or <EOL> self . info [ '<STR_LIT:name>' ] not in self . _split_regex_cache [ self . _settings . registry_key ] <EOL> ) : <EOL> self . _construct_split_regex ( ) <EOL> return self . _split_regex_cache [ self . _settings . registry_key ] [ self . info [ '<STR_LIT:name>' ] ] <EOL> def _construct_split_regex ( self ) : <EOL> known_words_group = u"<STR_LIT:|>" . join ( map ( re . escape , self . _get_sorted_words_from_cache ( ) ) ) <EOL> if self . _no_word_spacing : <EOL> regex = r"<STR_LIT>" . format ( known_words_group ) <EOL> else : <EOL> regex = r"<STR_LIT>" . format ( known_words_group ) <EOL> self . _split_regex_cache [ self . _settings . registry_key ] = { <EOL> self . info [ '<STR_LIT:name>' ] : re . compile ( regex , re . UNICODE | re . IGNORECASE ) <EOL> } <EOL> class NormalizedDictionary ( Dictionary ) : <EOL> def __init__ ( self , language_info , settings = None ) : <EOL> super ( NormalizedDictionary , self ) . __init__ ( language_info , settings ) <EOL> self . _normalize ( ) <EOL> def _normalize ( self ) : <EOL> new_dict = { } <EOL> conflicting_keys = [ ] <EOL> for key , value in self . _dictionary . items ( ) : <EOL> normalized = normalize_unicode ( key ) <EOL> if key != normalized and normalized in self . _dictionary : <EOL> conflicting_keys . append ( key ) <EOL> else : <EOL> new_dict [ normalized ] = value <EOL> for key in conflicting_keys : <EOL> normalized = normalize_unicode ( key ) <EOL> if key in ( self . info . get ( '<STR_LIT>' , [ ] ) + self . info . get ( '<STR_LIT>' , [ ] ) ) : <EOL> new_dict [ normalized ] = self . _dictionary [ key ] <EOL> self . _dictionary = new_dict </s>
