<s> '''<STR_LIT>''' <EOL> import tokenize <EOL> import operator <EOL> import collections <EOL> try : <EOL> import StringIO as io <EOL> except ImportError : <EOL> import io <EOL> __all__ = [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] <EOL> COMMENT = tokenize . COMMENT <EOL> OP = tokenize . OP <EOL> NL = tokenize . NL <EOL> EM = tokenize . ENDMARKER <EOL> TOKEN_NUMBER = operator . itemgetter ( <NUM_LIT:0> ) <EOL> Module = collections . namedtuple ( '<STR_LIT>' , [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT:blank>' , <EOL> '<STR_LIT>' ] ) <EOL> def _generate ( code ) : <EOL> '''<STR_LIT>''' <EOL> return list ( tokenize . generate_tokens ( io . StringIO ( code ) . readline ) ) <EOL> def _less_tokens ( tokens , remove ) : <EOL> '''<STR_LIT>''' <EOL> for values in tokens : <EOL> if values [ <NUM_LIT:0> ] in remove : <EOL> continue <EOL> yield values <EOL> def _find ( tokens , token , value ) : <EOL> '''<STR_LIT>''' <EOL> for index , token_values in enumerate ( reversed ( tokens ) ) : <EOL> if ( token , value ) == token_values [ : <NUM_LIT:2> ] : <EOL> return len ( tokens ) - index - <NUM_LIT:1> <EOL> raise ValueError ( '<STR_LIT>' ) <EOL> def _split_tokens ( tokens , token , value ) : <EOL> '''<STR_LIT>''' <EOL> res = [ [ ] ] <EOL> for token_values in tokens : <EOL> if ( token , value ) == token_values [ : <NUM_LIT:2> ] : <EOL> res . append ( [ ] ) <EOL> continue <EOL> res [ - <NUM_LIT:1> ] . append ( token_values ) <EOL> return res <EOL> def _get_all_tokens ( line , lines ) : <EOL> '''<STR_LIT>''' <EOL> sloc_increment = multi_increment = <NUM_LIT:0> <EOL> try : <EOL> tokens = _generate ( line ) <EOL> except tokenize . TokenError : <EOL> while True : <EOL> sloc_increment += <NUM_LIT:1> <EOL> line = '<STR_LIT:\n>' . join ( [ line , next ( lines ) ] ) <EOL> try : <EOL> tokens = _generate ( line ) <EOL> except tokenize . TokenError : <EOL> continue <EOL> if tokens [ <NUM_LIT:0> ] [ <NUM_LIT:0> ] == <NUM_LIT:3> and len ( tokens ) == <NUM_LIT:2> : <EOL> multi_increment += line . count ( '<STR_LIT:\n>' ) + <NUM_LIT:1> <EOL> break <EOL> return tokens , sloc_increment , multi_increment <EOL> def _logical ( tokens ) : <EOL> '''<STR_LIT>''' <EOL> def aux ( sub_tokens ) : <EOL> '''<STR_LIT>''' <EOL> processed = list ( _less_tokens ( sub_tokens , [ COMMENT ] ) ) <EOL> try : <EOL> token_pos = _find ( processed , OP , '<STR_LIT::>' ) <EOL> return <NUM_LIT:2> - ( token_pos == len ( processed ) - <NUM_LIT:2> ) <EOL> except ValueError : <EOL> if not list ( _less_tokens ( processed , [ NL , EM ] ) ) : <EOL> return <NUM_LIT:0> <EOL> return <NUM_LIT:1> <EOL> return sum ( aux ( sub ) for sub in _split_tokens ( tokens , OP , '<STR_LIT:;>' ) ) <EOL> def remove_lines ( doc , lines_to_remove ) : <EOL> '''<STR_LIT>''' <EOL> for line_number in lines_to_remove : <EOL> doc [ line_number ] = [ ] <EOL> return [ line . strip ( ) for line in doc if line ] <EOL> def is_multiline_string ( doc , line_count , quote_type ) : <EOL> '''<STR_LIT>''' <EOL> line = doc [ line_count ] <EOL> previous_line = doc [ line_count - <NUM_LIT:1> ] <EOL> if line . count ( '<STR_LIT:=>' ) > <NUM_LIT:0> and line . index ( '<STR_LIT:=>' ) < line . index ( quote_type ) or line_count != <NUM_LIT:0> and len ( previous_line ) > <NUM_LIT:2> and '<STR_LIT:=>' in previous_line [ - <NUM_LIT:3> ] : <EOL> return True <EOL> return False <EOL> def find_multiline_comments ( lines_to_remove , end , doc , line_count , quote_type ) : <EOL> '''<STR_LIT>''' <EOL> if quote_type and end is False and line_count < len ( doc ) - <NUM_LIT:1> : <EOL> quote_type = quote_type [ <NUM_LIT:0> ] <EOL> end = True <EOL> if is_multiline_string ( doc , line_count , quote_type ) : <EOL> line_count += <NUM_LIT:1> <EOL> while quote_type not in doc [ line_count ] : <EOL> line_count += <NUM_LIT:1> <EOL> if quote_type in doc [ line_count ] : <EOL> line_count += <NUM_LIT:1> <EOL> break <EOL> return lines_to_remove , False , line_count <EOL> lines_to_remove . append ( line_count ) <EOL> return lines_to_remove , end , line_count <EOL> elif end and not quote_type : <EOL> lines_to_remove . append ( line_count ) <EOL> elif end and quote_type : <EOL> lines_to_remove . append ( line_count ) <EOL> end = False <EOL> return lines_to_remove , end , line_count <EOL> def find_comments ( lines_to_remove , line_count , line ) : <EOL> '''<STR_LIT>''' <EOL> if not line : <EOL> return ( lines_to_remove , True ) <EOL> if line [ <NUM_LIT:0> ] == "<STR_LIT:#>" or line . count ( "<STR_LIT>" ) == <NUM_LIT:2> and '<STR_LIT:=>' not in line or line . count ( '<STR_LIT>' ) == <NUM_LIT:2> and '<STR_LIT:=>' not in line : <EOL> lines_to_remove . append ( line_count ) <EOL> return ( lines_to_remove , True ) <EOL> return ( lines_to_remove , False ) <EOL> def remove_python_documentation ( doc ) : <EOL> '''<STR_LIT>''' <EOL> multi_quos = [ "<STR_LIT>" , '<STR_LIT>' ] <EOL> lines_to_remove = [ ] <EOL> end = False <EOL> comments = <NUM_LIT:0> <EOL> multi = <NUM_LIT:1> <EOL> new_count = <NUM_LIT:0> <EOL> for line_count , line in enumerate ( doc ) : <EOL> if new_count > line_count : <EOL> continue <EOL> lines_to_remove , removed = find_comments ( lines_to_remove , <EOL> line_count , <EOL> line ) <EOL> if removed : <EOL> comments += <NUM_LIT:1> <EOL> continue <EOL> quote_type = [ multi_quo for multi_quo in multi_quos <EOL> if multi_quo in doc [ line_count ] ] <EOL> lines_to_remove , end , new_count = find_multiline_comments ( <EOL> lines_to_remove = lines_to_remove , <EOL> end = end , <EOL> doc = doc , <EOL> line_count = line_count , <EOL> quote_type = quote_type ) <EOL> if end : <EOL> multi += <NUM_LIT:1> <EOL> if multi == <NUM_LIT:1> : <EOL> multi = <NUM_LIT:0> <EOL> return len ( remove_lines ( doc , lines_to_remove ) ) , comments , multi <EOL> def analyze ( source ) : <EOL> '''<STR_LIT>''' <EOL> lloc = comments = multi = blank = <NUM_LIT:0> <EOL> source_array = [ line . strip ( ) for line in source . split ( '<STR_LIT:\n>' ) if line ] <EOL> sloc = len ( source_array ) <EOL> loc , single_comments , multi = remove_python_documentation ( source_array ) <EOL> lines = iter ( source . splitlines ( ) ) <EOL> for lineno , line in enumerate ( lines , <NUM_LIT:1> ) : <EOL> line = line . strip ( ) <EOL> if not line : <EOL> blank += <NUM_LIT:1> <EOL> continue <EOL> try : <EOL> tokens , _ , _ = _get_all_tokens ( line , lines ) <EOL> except StopIteration : <EOL> raise SyntaxError ( '<STR_LIT>' . format ( lineno ) ) <EOL> comments += list ( map ( TOKEN_NUMBER , tokens ) ) . count ( COMMENT ) <EOL> for sub_tokens in _split_tokens ( tokens , OP , '<STR_LIT:;>' ) : <EOL> lloc += _logical ( sub_tokens ) <EOL> return Module ( loc , lloc , sloc , comments , multi , blank , single_comments ) </s>
