<s> """<STR_LIT>""" <EOL> import collections <EOL> import glob <EOL> import os <EOL> from os import path <EOL> import sys <EOL> import xml . etree . cElementTree as ET <EOL> from nototools import cldr_data <EOL> from nototools import tool_utils <EOL> from nototools import unicode_data <EOL> TOOLS_DIR = path . abspath ( path . join ( path . dirname ( __file__ ) , os . pardir ) ) <EOL> CLDR_DIR = path . join ( TOOLS_DIR , '<STR_LIT>' , '<STR_LIT>' ) <EOL> def _add_text ( chars , text ) : <EOL> skip = False <EOL> for i , cp in enumerate ( text ) : <EOL> if cp == '<STR_LIT:{>' : <EOL> skip = True <EOL> continue <EOL> if cp == '<STR_LIT:}>' : <EOL> skip = False <EOL> continue <EOL> if not skip : <EOL> if cp == '<STR_LIT:U+0020>' : <EOL> continue <EOL> script = unicode_data . script ( cp ) <EOL> if script == '<STR_LIT>' : <EOL> chars . add ( cp ) <EOL> def _collect_punct_data ( tree ) : <EOL> chars = set ( ) <EOL> for tag in tree . findall ( '<STR_LIT>' ) : <EOL> _add_text ( chars , tag . text ) <EOL> for tag in tree . findall ( '<STR_LIT>' ) : <EOL> _add_text ( chars , tag . text ) <EOL> for tag in tree . findall ( '<STR_LIT>' ) : <EOL> for t in tag : <EOL> _add_text ( chars , t . text ) <EOL> for tag in tree . findall ( '<STR_LIT>' ) : <EOL> _add_text ( chars , tag . text ) <EOL> return chars <EOL> def _collect_currency_data ( tree ) : <EOL> currencies = set ( ) <EOL> for tag in tree . findall ( '<STR_LIT>' ) : <EOL> if len ( tag . text ) == <NUM_LIT:1> : <EOL> currencies . add ( tag . text ) <EOL> return currencies <EOL> def _get_cldr_files ( cldr_dirs ) : <EOL> files = set ( ) <EOL> for cldr_dir in cldr_dirs : <EOL> search = path . join ( CLDR_DIR , '<STR_LIT>' , cldr_dir , '<STR_LIT>' ) <EOL> files . update ( glob . glob ( search ) ) <EOL> return files <EOL> def _collect_script_to_punct ( files ) : <EOL> """<STR_LIT>""" <EOL> script_to_punct = collections . defaultdict ( set ) <EOL> curly_quotes_to_standard = [ <EOL> ( frozenset ( [ unichr ( <NUM_LIT> ) , unichr ( <NUM_LIT> ) ] ) , frozenset ( [ '<STR_LIT>' ] ) ) , <EOL> ( frozenset ( [ unichr ( <NUM_LIT> ) , unichr ( <NUM_LIT> ) ] ) , frozenset ( [ '<STR_LIT:">' ] ) ) , <EOL> ] <EOL> for f in files : <EOL> tree = ET . parse ( f ) <EOL> punct = _collect_punct_data ( tree ) <EOL> if punct : <EOL> filename = path . splitext ( path . basename ( f ) ) [ <NUM_LIT:0> ] <EOL> script = cldr_data . get_likely_script ( filename ) <EOL> if script == '<STR_LIT>' : <EOL> if filename != '<STR_LIT:root>' : <EOL> print >> sys . stderr , '<STR_LIT>' % filename <EOL> else : <EOL> script_to_punct [ script ] |= punct <EOL> script_to_punct [ '<STR_LIT>' ] = set ( <EOL> script_to_punct [ '<STR_LIT>' ] | <EOL> script_to_punct [ '<STR_LIT>' ] | <EOL> script_to_punct [ '<STR_LIT>' ] ) <EOL> for script in script_to_punct : <EOL> punct = script_to_punct [ script ] <EOL> for curly , standard in curly_quotes_to_standard : <EOL> if curly & punct : <EOL> punct . update ( standard ) <EOL> return script_to_punct <EOL> def _build_script_to_punct ( ) : <EOL> files = _get_cldr_files ( [ '<STR_LIT>' ] ) <EOL> script_to_punct = _collect_script_to_punct ( files ) <EOL> return { <EOL> script : frozenset ( script_to_punct [ script ] ) <EOL> for script in script_to_punct <EOL> } <EOL> _script_to_punct = None <EOL> def script_to_punct ( ) : <EOL> global _script_to_punct <EOL> if not _script_to_punct : <EOL> _script_to_punct = _build_script_to_punct ( ) <EOL> return _script_to_punct <EOL> def _write_script_to_punct ( script_to_punct ) : <EOL> print '<STR_LIT>' <EOL> for script in sorted ( script_to_punct ) : <EOL> chars = script_to_punct [ script ] <EOL> int_chars = [ ord ( cp ) for cp in chars ] <EOL> print '<STR_LIT>' % ( '<STR_LIT:|>' . join ( sorted ( chars ) ) ) <EOL> print "<STR_LIT>" % ( script , tool_utils . write_int_ranges ( int_chars ) ) <EOL> print '<STR_LIT:}>' <EOL> def main ( ) : <EOL> _write_script_to_punct ( _build_script_to_punct ( ) ) <EOL> if __name__ == "<STR_LIT:__main__>" : <EOL> main ( ) </s>
