<s> from bisect import bisect_left <EOL> import operator <EOL> import java . lang . Character <EOL> _codepoints = { } <EOL> _eaw = { } <EOL> _names = { } <EOL> _segments = [ ] <EOL> _eaw_segments = [ ] <EOL> Nonesuch = object ( ) <EOL> def get_int ( col ) : <EOL> try : <EOL> return int ( col ) <EOL> except ValueError : <EOL> return None <EOL> def get_yn ( col ) : <EOL> if col == '<STR_LIT:Y>' : return <NUM_LIT:1> <EOL> else : return <NUM_LIT:0> <EOL> def get_numeric ( col ) : <EOL> try : <EOL> return float ( col ) <EOL> except ValueError : <EOL> try : <EOL> a , b = col . split ( '<STR_LIT:/>' ) <EOL> return float ( a ) / float ( b ) <EOL> except : <EOL> return None <EOL> def init_unicodedata ( data ) : <EOL> for row in data : <EOL> cols = row . split ( '<STR_LIT:;>' ) <EOL> codepoint = int ( cols [ <NUM_LIT:0> ] , <NUM_LIT:16> ) <EOL> name = cols [ <NUM_LIT:1> ] <EOL> if name == '<STR_LIT>' : <EOL> lookup_name = '<STR_LIT>' <EOL> else : <EOL> lookup_name = name <EOL> data = ( <EOL> cols [ <NUM_LIT:2> ] , <EOL> get_int ( cols [ <NUM_LIT:3> ] ) , <EOL> cols [ <NUM_LIT:4> ] , <EOL> cols [ <NUM_LIT:5> ] , <EOL> get_int ( cols [ <NUM_LIT:6> ] ) , <EOL> get_int ( cols [ <NUM_LIT:7> ] ) , <EOL> get_numeric ( cols [ <NUM_LIT:8> ] ) , <EOL> get_yn ( cols [ <NUM_LIT:9> ] ) , <EOL> lookup_name , <EOL> ) <EOL> if name . find ( '<STR_LIT>' ) >= <NUM_LIT:0> : <EOL> start = codepoint <EOL> elif name . find ( '<STR_LIT>' ) >= <NUM_LIT:0> : <EOL> _segments . append ( ( start , ( start , codepoint ) , data ) ) <EOL> else : <EOL> _names [ name ] = unichr ( codepoint ) <EOL> _codepoints [ codepoint ] = data <EOL> def init_east_asian_width ( data ) : <EOL> for row in data : <EOL> if row . startswith ( '<STR_LIT:#>' ) : <EOL> continue <EOL> row = row . partition ( '<STR_LIT:#>' ) [ <NUM_LIT:0> ] <EOL> cols = row . split ( '<STR_LIT:;>' ) <EOL> if len ( cols ) < <NUM_LIT:2> : <EOL> continue <EOL> cr = cols [ <NUM_LIT:0> ] . split ( '<STR_LIT:..>' ) <EOL> width = cols [ <NUM_LIT:1> ] . rstrip ( ) <EOL> if len ( cr ) == <NUM_LIT:1> : <EOL> codepoint = int ( cr [ <NUM_LIT:0> ] , <NUM_LIT:16> ) <EOL> _eaw [ codepoint ] = width <EOL> else : <EOL> start = int ( cr [ <NUM_LIT:0> ] , <NUM_LIT:16> ) <EOL> end = int ( cr [ <NUM_LIT:1> ] , <NUM_LIT:16> ) <EOL> _eaw_segments . append ( ( start , ( start , end ) , width ) ) <EOL> def name ( unichr , default = None ) : <EOL> codepoint = get_codepoint ( unichr , "<STR_LIT:name>" ) <EOL> v = _codepoints . get ( codepoint , None ) <EOL> if v is None : <EOL> v = check_segments ( codepoint , _segments ) <EOL> if v is not None : <EOL> return "<STR_LIT>" % ( v [ <NUM_LIT:8> ] , codepoint ) <EOL> if v is None : <EOL> if default is not Nonesuch : <EOL> return default <EOL> raise ValueError ( ) <EOL> return v [ <NUM_LIT:8> ] <EOL> def lookup ( name ) : <EOL> return _names [ name ] <EOL> def check_segments ( codepoint , segments ) : <EOL> i = bisect_left ( segments , ( codepoint , ) ) <EOL> if i < len ( segments ) : <EOL> segment = segments [ i - <NUM_LIT:1> ] <EOL> if codepoint <= segment [ <NUM_LIT:1> ] [ <NUM_LIT:1> ] : <EOL> return segment [ <NUM_LIT:2> ] <EOL> return None <EOL> def get_codepoint ( unichr , fn = None ) : <EOL> if not ( isinstance ( unichr , unicode ) ) : <EOL> raise TypeError ( fn , "<STR_LIT>" + type ( unichr ) ) <EOL> if len ( unichr ) > <NUM_LIT:1> or len ( unichr ) == <NUM_LIT:0> : <EOL> raise TypeError ( "<STR_LIT>" ) <EOL> return ord ( unichr ) <EOL> def get_eaw ( unichr , default , fn ) : <EOL> codepoint = get_codepoint ( unichr , fn ) <EOL> v = _eaw . get ( codepoint , None ) <EOL> if v is None : <EOL> v = check_segments ( codepoint , _eaw_segments ) <EOL> if v is None : <EOL> if default is not Nonesuch : <EOL> return default <EOL> raise ValueError ( ) <EOL> return v <EOL> def get ( unichr , default , fn , getter ) : <EOL> codepoint = get_codepoint ( unichr , fn ) <EOL> data = _codepoints . get ( codepoint , None ) <EOL> if data is None : <EOL> data = check_segments ( codepoint , _segments ) <EOL> if data is None : <EOL> if default is not Nonesuch : <EOL> return default <EOL> raise ValueError ( ) <EOL> v = getter ( data ) <EOL> if v is None : <EOL> if default is not Nonesuch : <EOL> return default <EOL> raise ValueError ( ) <EOL> else : <EOL> return v <EOL> category_getter = operator . itemgetter ( <NUM_LIT:0> ) <EOL> combining_getter = operator . itemgetter ( <NUM_LIT:1> ) <EOL> bidirectional_getter = operator . itemgetter ( <NUM_LIT:2> ) <EOL> decomposition_getter = operator . itemgetter ( <NUM_LIT:3> ) <EOL> decimal_getter = operator . itemgetter ( <NUM_LIT:4> ) <EOL> digit_getter = operator . itemgetter ( <NUM_LIT:5> ) <EOL> numeric_getter = operator . itemgetter ( <NUM_LIT:6> ) <EOL> mirrored_getter = operator . itemgetter ( <NUM_LIT:7> ) <EOL> def decimal ( unichr , default = Nonesuch ) : <EOL> return get ( unichr , default , '<STR_LIT>' , decimal_getter ) <EOL> def decomposition ( unichr , default = '<STR_LIT>' ) : <EOL> return get ( unichr , default , '<STR_LIT>' , decomposition_getter ) <EOL> def digit ( unichr , default = Nonesuch ) : <EOL> return get ( unichr , default , '<STR_LIT>' , digit_getter ) <EOL> def numeric ( unichr , default = Nonesuch ) : <EOL> return get ( unichr , default , '<STR_LIT>' , numeric_getter ) <EOL> def category ( unichr ) : <EOL> return get ( unichr , '<STR_LIT>' , '<STR_LIT>' , category_getter ) <EOL> def bidirectional ( unichr ) : <EOL> return get ( unichr , '<STR_LIT>' , '<STR_LIT>' , bidirectional_getter ) <EOL> def combining ( unichr ) : <EOL> return get ( unichr , <NUM_LIT:0> , '<STR_LIT>' , combining_getter ) <EOL> def mirrored ( unichr ) : <EOL> return get ( unichr , <NUM_LIT:0> , '<STR_LIT>' , mirrored_getter ) <EOL> def east_asian_width ( unichr ) : <EOL> return get_eaw ( unichr , '<STR_LIT:N>' , '<STR_LIT>' ) <EOL> def jymirrored ( unichr ) : <EOL> return java . lang . Character . isMirrored ( get_codepoint ( unichr , '<STR_LIT>' ) ) <EOL> try : <EOL> from java . text import Normalizer <EOL> _forms = { <EOL> '<STR_LIT>' : Normalizer . Form . NFC , <EOL> '<STR_LIT>' : Normalizer . Form . NFKC , <EOL> '<STR_LIT>' : Normalizer . Form . NFD , <EOL> '<STR_LIT>' : Normalizer . Form . NFKD <EOL> } <EOL> def normalize ( form , unistr ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> normalizer_form = _forms [ form ] <EOL> except KeyError : <EOL> raise ValueError ( '<STR_LIT>' ) <EOL> return Normalizer . normalize ( unistr , normalizer_form ) <EOL> except ImportError : <EOL> pass <EOL> def init ( ) : <EOL> import pkgutil <EOL> import os . path <EOL> import StringIO <EOL> import sys <EOL> my_path = os . path . dirname ( __file__ ) <EOL> loader = pkgutil . get_loader ( '<STR_LIT>' ) <EOL> init_unicodedata ( StringIO . StringIO ( loader . get_data ( os . path . join ( my_path , '<STR_LIT>' ) ) ) ) <EOL> init_east_asian_width ( StringIO . StringIO ( loader . get_data ( os . path . join ( my_path , '<STR_LIT>' ) ) ) ) <EOL> init ( ) </s>
