<s> """<STR_LIT>""" <EOL> import os <EOL> try : <EOL> set <EOL> except NameError : <EOL> from sets import Set as set <EOL> import sys <EOL> from tokenize import generate_tokens , COMMENT , NAME , OP , STRING <EOL> from babel . util import parse_encoding , pathmatch , relpath <EOL> from textwrap import dedent <EOL> __all__ = [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] <EOL> __docformat__ = '<STR_LIT>' <EOL> GROUP_NAME = '<STR_LIT>' <EOL> DEFAULT_KEYWORDS = { <EOL> '<STR_LIT:_>' : None , <EOL> '<STR_LIT>' : None , <EOL> '<STR_LIT>' : ( <NUM_LIT:1> , <NUM_LIT:2> ) , <EOL> '<STR_LIT>' : None , <EOL> '<STR_LIT>' : ( <NUM_LIT:1> , <NUM_LIT:2> ) , <EOL> '<STR_LIT>' : ( <NUM_LIT:2> , ) , <EOL> '<STR_LIT>' : ( <NUM_LIT:2> , <NUM_LIT:3> ) , <EOL> '<STR_LIT>' : None <EOL> } <EOL> DEFAULT_MAPPING = [ ( '<STR_LIT>' , '<STR_LIT>' ) ] <EOL> empty_msgid_warning = ( <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> def _strip_comment_tags ( comments , tags ) : <EOL> """<STR_LIT>""" <EOL> def _strip ( line ) : <EOL> for tag in tags : <EOL> if line . startswith ( tag ) : <EOL> return line [ len ( tag ) : ] . strip ( ) <EOL> return line <EOL> comments [ : ] = map ( _strip , comments ) <EOL> def extract_from_dir ( dirname = os . getcwd ( ) , method_map = DEFAULT_MAPPING , <EOL> options_map = None , keywords = DEFAULT_KEYWORDS , <EOL> comment_tags = ( ) , callback = None , strip_comment_tags = False ) : <EOL> """<STR_LIT>""" <EOL> if options_map is None : <EOL> options_map = { } <EOL> absname = os . path . abspath ( dirname ) <EOL> for root , dirnames , filenames in os . walk ( absname ) : <EOL> for subdir in dirnames : <EOL> if subdir . startswith ( '<STR_LIT:.>' ) or subdir . startswith ( '<STR_LIT:_>' ) : <EOL> dirnames . remove ( subdir ) <EOL> dirnames . sort ( ) <EOL> filenames . sort ( ) <EOL> for filename in filenames : <EOL> filename = relpath ( <EOL> os . path . join ( root , filename ) . replace ( os . sep , '<STR_LIT:/>' ) , <EOL> dirname <EOL> ) <EOL> for pattern , method in method_map : <EOL> if pathmatch ( pattern , filename ) : <EOL> filepath = os . path . join ( absname , filename ) <EOL> options = { } <EOL> for opattern , odict in options_map . items ( ) : <EOL> if pathmatch ( opattern , filename ) : <EOL> options = odict <EOL> if callback : <EOL> callback ( filename , method , options ) <EOL> for lineno , message , comments in extract_from_file ( method , filepath , <EOL> keywords = keywords , <EOL> comment_tags = comment_tags , <EOL> options = options , <EOL> strip_comment_tags = <EOL> strip_comment_tags ) : <EOL> yield filename , lineno , message , comments <EOL> break <EOL> def extract_from_file ( method , filename , keywords = DEFAULT_KEYWORDS , <EOL> comment_tags = ( ) , options = None , strip_comment_tags = False ) : <EOL> """<STR_LIT>""" <EOL> fileobj = open ( filename , '<STR_LIT>' ) <EOL> try : <EOL> return list ( extract ( method , fileobj , keywords , comment_tags , options , <EOL> strip_comment_tags ) ) <EOL> finally : <EOL> fileobj . close ( ) <EOL> def extract ( method , fileobj , keywords = DEFAULT_KEYWORDS , comment_tags = ( ) , <EOL> options = None , strip_comment_tags = False ) : <EOL> """<STR_LIT>""" <EOL> func = None <EOL> if '<STR_LIT::>' in method or '<STR_LIT:.>' in method : <EOL> if '<STR_LIT::>' not in method : <EOL> lastdot = method . rfind ( '<STR_LIT:.>' ) <EOL> module , attrname = method [ : lastdot ] , method [ lastdot + <NUM_LIT:1> : ] <EOL> else : <EOL> module , attrname = method . split ( '<STR_LIT::>' , <NUM_LIT:1> ) <EOL> func = getattr ( __import__ ( module , { } , { } , [ attrname ] ) , attrname ) <EOL> else : <EOL> try : <EOL> from pkg_resources import working_set <EOL> except ImportError : <EOL> builtin = { '<STR_LIT:ignore>' : extract_nothing , '<STR_LIT>' : extract_python } <EOL> func = builtin . get ( method ) <EOL> else : <EOL> for entry_point in working_set . iter_entry_points ( GROUP_NAME , <EOL> method ) : <EOL> func = entry_point . load ( require = True ) <EOL> break <EOL> if func is None : <EOL> raise ValueError ( '<STR_LIT>' % method ) <EOL> results = func ( fileobj , keywords . keys ( ) , comment_tags , <EOL> options = options or { } ) <EOL> for lineno , funcname , messages , comments in results : <EOL> if funcname : <EOL> spec = keywords [ funcname ] or ( <NUM_LIT:1> , ) <EOL> else : <EOL> spec = ( <NUM_LIT:1> , ) <EOL> if not isinstance ( messages , ( list , tuple ) ) : <EOL> messages = [ messages ] <EOL> if not messages : <EOL> continue <EOL> msgs = [ ] <EOL> invalid = False <EOL> last_index = len ( messages ) <EOL> for index in spec : <EOL> if last_index < index : <EOL> invalid = True <EOL> break <EOL> message = messages [ index - <NUM_LIT:1> ] <EOL> if message is None : <EOL> invalid = True <EOL> break <EOL> msgs . append ( message ) <EOL> if invalid : <EOL> continue <EOL> first_msg_index = spec [ <NUM_LIT:0> ] - <NUM_LIT:1> <EOL> if not messages [ first_msg_index ] : <EOL> where = '<STR_LIT>' % ( hasattr ( fileobj , '<STR_LIT:name>' ) and fileobj . name or '<STR_LIT>' , lineno ) <EOL> print >> sys . stderr , empty_msgid_warning % where <EOL> continue <EOL> messages = tuple ( msgs ) <EOL> if len ( messages ) == <NUM_LIT:1> : <EOL> messages = messages [ <NUM_LIT:0> ] <EOL> if strip_comment_tags : <EOL> _strip_comment_tags ( comments , comment_tags ) <EOL> yield lineno , messages , comments <EOL> def extract_nothing ( fileobj , keywords , comment_tags , options ) : <EOL> """<STR_LIT>""" <EOL> return [ ] <EOL> def extract_python ( fileobj , keywords , comment_tags , options ) : <EOL> """<STR_LIT>""" <EOL> funcname = lineno = message_lineno = None <EOL> call_stack = - <NUM_LIT:1> <EOL> buf = [ ] <EOL> messages = [ ] <EOL> translator_comments = [ ] <EOL> in_def = in_translator_comments = False <EOL> comment_tag = None <EOL> encoding = parse_encoding ( fileobj ) or options . get ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> tokens = generate_tokens ( fileobj . readline ) <EOL> for tok , value , ( lineno , _ ) , _ , _ in tokens : <EOL> if call_stack == - <NUM_LIT:1> and tok == NAME and value in ( '<STR_LIT>' , '<STR_LIT:class>' ) : <EOL> in_def = True <EOL> elif tok == OP and value == '<STR_LIT:(>' : <EOL> if in_def : <EOL> in_def = False <EOL> continue <EOL> if funcname : <EOL> message_lineno = lineno <EOL> call_stack += <NUM_LIT:1> <EOL> elif in_def and tok == OP and value == '<STR_LIT::>' : <EOL> in_def = False <EOL> continue <EOL> elif call_stack == - <NUM_LIT:1> and tok == COMMENT : <EOL> value = value . decode ( encoding ) [ <NUM_LIT:1> : ] . strip ( ) <EOL> if in_translator_comments and translator_comments [ - <NUM_LIT:1> ] [ <NUM_LIT:0> ] == lineno - <NUM_LIT:1> : <EOL> translator_comments . append ( ( lineno , value ) ) <EOL> continue <EOL> for comment_tag in comment_tags : <EOL> if value . startswith ( comment_tag ) : <EOL> in_translator_comments = True <EOL> translator_comments . append ( ( lineno , value ) ) <EOL> break <EOL> elif funcname and call_stack == <NUM_LIT:0> : <EOL> if tok == OP and value == '<STR_LIT:)>' : <EOL> if buf : <EOL> messages . append ( '<STR_LIT>' . join ( buf ) ) <EOL> del buf [ : ] <EOL> else : <EOL> messages . append ( None ) <EOL> if len ( messages ) > <NUM_LIT:1> : <EOL> messages = tuple ( messages ) <EOL> else : <EOL> messages = messages [ <NUM_LIT:0> ] <EOL> if translator_comments and translator_comments [ - <NUM_LIT:1> ] [ <NUM_LIT:0> ] < message_lineno - <NUM_LIT:1> : <EOL> translator_comments = [ ] <EOL> yield ( message_lineno , funcname , messages , <EOL> [ comment [ <NUM_LIT:1> ] for comment in translator_comments ] ) <EOL> funcname = lineno = message_lineno = None <EOL> call_stack = - <NUM_LIT:1> <EOL> messages = [ ] <EOL> translator_comments = [ ] <EOL> in_translator_comments = False <EOL> elif tok == STRING : <EOL> value = eval ( '<STR_LIT>' % ( encoding , value ) , <EOL> { '<STR_LIT>' : { } } , { } ) <EOL> if isinstance ( value , str ) : <EOL> value = value . decode ( encoding ) <EOL> buf . append ( value ) <EOL> elif tok == OP and value == '<STR_LIT:U+002C>' : <EOL> if buf : <EOL> messages . append ( '<STR_LIT>' . join ( buf ) ) <EOL> del buf [ : ] <EOL> else : <EOL> messages . append ( None ) <EOL> if translator_comments : <EOL> old_lineno , old_comment = translator_comments . pop ( ) <EOL> translator_comments . append ( ( old_lineno + <NUM_LIT:1> , old_comment ) ) <EOL> elif call_stack > <NUM_LIT:0> and tok == OP and value == '<STR_LIT:)>' : <EOL> call_stack -= <NUM_LIT:1> <EOL> elif funcname and call_stack == - <NUM_LIT:1> : <EOL> funcname = None <EOL> elif tok == NAME and value in keywords : <EOL> funcname = value <EOL> def extract_javascript ( fileobj , keywords , comment_tags , options ) : <EOL> """<STR_LIT>""" <EOL> from babel . messages . jslexer import tokenize , unquote_string <EOL> funcname = message_lineno = None <EOL> messages = [ ] <EOL> last_argument = None <EOL> translator_comments = [ ] <EOL> concatenate_next = False <EOL> encoding = options . get ( '<STR_LIT>' , '<STR_LIT:utf-8>' ) <EOL> last_token = None <EOL> call_stack = - <NUM_LIT:1> <EOL> for token in tokenize ( fileobj . read ( ) . decode ( encoding ) ) : <EOL> if token . type == '<STR_LIT>' and token . value == '<STR_LIT:(>' : <EOL> if funcname : <EOL> message_lineno = token . lineno <EOL> call_stack += <NUM_LIT:1> <EOL> elif call_stack == - <NUM_LIT:1> and token . type == '<STR_LIT>' : <EOL> value = token . value [ <NUM_LIT:2> : ] . strip ( ) <EOL> if translator_comments and translator_comments [ - <NUM_LIT:1> ] [ <NUM_LIT:0> ] == token . lineno - <NUM_LIT:1> : <EOL> translator_comments . append ( ( token . lineno , value ) ) <EOL> continue <EOL> for comment_tag in comment_tags : <EOL> if value . startswith ( comment_tag ) : <EOL> translator_comments . append ( ( token . lineno , value . strip ( ) ) ) <EOL> break <EOL> elif token . type == '<STR_LIT>' : <EOL> translator_comments = [ ] <EOL> value = token . value [ <NUM_LIT:2> : - <NUM_LIT:2> ] . strip ( ) <EOL> for comment_tag in comment_tags : <EOL> if value . startswith ( comment_tag ) : <EOL> lines = value . splitlines ( ) <EOL> if lines : <EOL> lines [ <NUM_LIT:0> ] = lines [ <NUM_LIT:0> ] . strip ( ) <EOL> lines [ <NUM_LIT:1> : ] = dedent ( '<STR_LIT:\n>' . join ( lines [ <NUM_LIT:1> : ] ) ) . splitlines ( ) <EOL> for offset , line in enumerate ( lines ) : <EOL> translator_comments . append ( ( token . lineno + offset , <EOL> line ) ) <EOL> break <EOL> elif funcname and call_stack == <NUM_LIT:0> : <EOL> if token . type == '<STR_LIT>' and token . value == '<STR_LIT:)>' : <EOL> if last_argument is not None : <EOL> messages . append ( last_argument ) <EOL> if len ( messages ) > <NUM_LIT:1> : <EOL> messages = tuple ( messages ) <EOL> elif messages : <EOL> messages = messages [ <NUM_LIT:0> ] <EOL> else : <EOL> messages = None <EOL> if translator_comments and translator_comments [ - <NUM_LIT:1> ] [ <NUM_LIT:0> ] < message_lineno - <NUM_LIT:1> : <EOL> translator_comments = [ ] <EOL> if messages is not None : <EOL> yield ( message_lineno , funcname , messages , <EOL> [ comment [ <NUM_LIT:1> ] for comment in translator_comments ] ) <EOL> funcname = message_lineno = last_argument = None <EOL> concatenate_next = False <EOL> translator_comments = [ ] <EOL> messages = [ ] <EOL> call_stack = - <NUM_LIT:1> <EOL> elif token . type == '<STR_LIT:string>' : <EOL> new_value = unquote_string ( token . value ) <EOL> if concatenate_next : <EOL> last_argument = ( last_argument or '<STR_LIT>' ) + new_value <EOL> concatenate_next = False <EOL> else : <EOL> last_argument = new_value <EOL> elif token . type == '<STR_LIT>' : <EOL> if token . value == '<STR_LIT:U+002C>' : <EOL> if last_argument is not None : <EOL> messages . append ( last_argument ) <EOL> last_argument = None <EOL> else : <EOL> messages . append ( None ) <EOL> concatenate_next = False <EOL> elif token . value == '<STR_LIT:+>' : <EOL> concatenate_next = True <EOL> elif call_stack > <NUM_LIT:0> and token . type == '<STR_LIT>' and token . value == '<STR_LIT:)>' : <EOL> call_stack -= <NUM_LIT:1> <EOL> elif funcname and call_stack == - <NUM_LIT:1> : <EOL> funcname = None <EOL> elif call_stack == - <NUM_LIT:1> and token . type == '<STR_LIT:name>' and token . value in keywords and ( last_token is None or last_token . type != '<STR_LIT:name>' or <EOL> last_token . value != '<STR_LIT>' ) : <EOL> funcname = token . value <EOL> last_token = token </s>
