<s> from __future__ import unicode_literals <EOL> import io <EOL> import re <EOL> import ast <EOL> from . exceptions import MoultScannerError <EOL> from . compat import str_ <EOL> from . import utils , log <EOL> _fallback_re = re . compile ( r'''<STR_LIT>''' , re . VERBOSE | re . MULTILINE | re . UNICODE ) <EOL> def ast_value ( val , scope , return_name = False ) : <EOL> '''<STR_LIT>''' <EOL> try : <EOL> if isinstance ( val , ( ast . Assign , ast . Delete ) ) : <EOL> if hasattr ( val , '<STR_LIT:value>' ) : <EOL> value = ast_value ( val . value , scope ) <EOL> else : <EOL> value = None <EOL> for t in val . targets : <EOL> name = ast_value ( t , scope , return_name = True ) <EOL> if isinstance ( t . ctx , ast . Del ) : <EOL> if name in scope : <EOL> scope . pop ( name ) <EOL> elif isinstance ( t . ctx , ast . Store ) : <EOL> scope [ name ] = value <EOL> return <EOL> elif isinstance ( val , ast . Expr ) and isinstance ( val . value , ast . Name ) : <EOL> return ast_value ( val . value ) <EOL> if isinstance ( val , ast . Name ) : <EOL> if isinstance ( val . ctx , ast . Load ) : <EOL> if val . id == '<STR_LIT:None>' : <EOL> return None <EOL> elif val . id == '<STR_LIT:True>' : <EOL> return True <EOL> elif val . id == '<STR_LIT:False>' : <EOL> return False <EOL> if val . id in scope : <EOL> return scope [ val . id ] <EOL> if return_name : <EOL> return val . id <EOL> elif isinstance ( val . ctx , ast . Store ) : <EOL> if return_name : <EOL> return val . id <EOL> return None <EOL> if isinstance ( val , ast . Subscript ) : <EOL> toslice = ast_value ( val . value , scope ) <EOL> theslice = ast_value ( val . slice , scope ) <EOL> return toslice [ theslice ] <EOL> elif isinstance ( val , ast . Index ) : <EOL> return ast_value ( val . value , scope ) <EOL> elif isinstance ( val , ast . Slice ) : <EOL> lower = ast_value ( val . lower ) <EOL> upper = ast_value ( val . upper ) <EOL> step = ast_value ( val . step ) <EOL> return slice ( lower , upper , step ) <EOL> if isinstance ( val , list ) : <EOL> return [ ast_value ( x , scope ) for x in val ] <EOL> elif isinstance ( val , tuple ) : <EOL> return tuple ( ast_value ( x , scope ) for x in val ) <EOL> if isinstance ( val , ast . Attribute ) : <EOL> name = ast_value ( val . value , scope , return_name = True ) <EOL> if isinstance ( val . ctx , ast . Load ) : <EOL> return '<STR_LIT:.>' . join ( ( name , val . attr ) ) <EOL> if return_name : <EOL> return name <EOL> elif isinstance ( val , ast . keyword ) : <EOL> return { val . arg : ast_value ( val . value , scope ) } <EOL> elif isinstance ( val , ast . List ) : <EOL> return [ ast_value ( x , scope ) for x in val . elts ] <EOL> elif isinstance ( val , ast . Tuple ) : <EOL> return tuple ( ast_value ( x , scope ) for x in val . elts ) <EOL> elif isinstance ( val , ast . Dict ) : <EOL> return dict ( zip ( [ ast_value ( x , scope ) for x in val . keys ] , <EOL> [ ast_value ( x , scope ) for x in val . values ] ) ) <EOL> elif isinstance ( val , ast . Num ) : <EOL> return val . n <EOL> elif isinstance ( val , ast . Str ) : <EOL> return val . s <EOL> elif hasattr ( ast , '<STR_LIT>' ) and isinstance ( val , ast . Bytes ) : <EOL> return bytes ( val . s ) <EOL> except Exception : <EOL> pass <EOL> return None <EOL> def flatten_call_args ( args , kwlist , starargs , kwargs ) : <EOL> if starargs : <EOL> args . extend ( starargs ) <EOL> keywords = { } <EOL> for kw in kwlist : <EOL> keywords . update ( kw ) <EOL> if kwargs : <EOL> keywords . update ( keywords ) <EOL> return args , keywords <EOL> def get_args ( args , kwargs , arg_names ) : <EOL> '''<STR_LIT>''' <EOL> n_args = len ( arg_names ) <EOL> if len ( args ) + len ( kwargs ) > n_args : <EOL> raise MoultScannerError ( '<STR_LIT>' . format ( n_args ) ) <EOL> out_args = { } <EOL> for i , a in enumerate ( args ) : <EOL> out_args [ arg_names [ i ] ] = a <EOL> for a in arg_names : <EOL> if a not in out_args : <EOL> out_args [ a ] = None <EOL> out_args . update ( kwargs ) <EOL> return out_args <EOL> def parse_programmatic_import ( node , scope ) : <EOL> name = ast_value ( node . func , scope , return_name = True ) <EOL> if not name : <EOL> return [ ] <EOL> args , kwargs = flatten_call_args ( ast_value ( node . args , scope ) , <EOL> ast_value ( node . keywords , scope ) , <EOL> ast_value ( node . starargs , scope ) , <EOL> ast_value ( node . kwargs , scope ) ) <EOL> imports = [ ] <EOL> if name . endswith ( '<STR_LIT>' ) : <EOL> func_args = get_args ( args , kwargs , [ '<STR_LIT:name>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' ] ) <EOL> log . debug ( '<STR_LIT>' . format ( func_args ) ) <EOL> if not func_args [ '<STR_LIT:name>' ] : <EOL> raise MoultScannerError ( '<STR_LIT>' ) <EOL> if func_args [ '<STR_LIT>' ] : <EOL> if not hasattr ( func_args [ '<STR_LIT>' ] , '<STR_LIT>' ) : <EOL> raise MoultScannerError ( '<STR_LIT>' ) <EOL> for fromname in func_args [ '<STR_LIT>' ] : <EOL> imports . append ( ( func_args [ '<STR_LIT:name>' ] , fromname ) ) <EOL> else : <EOL> imports . append ( ( None , func_args [ '<STR_LIT:name>' ] ) ) <EOL> elif name . endswith ( '<STR_LIT>' ) : <EOL> func_args = get_args ( args , kwargs , [ '<STR_LIT:name>' , '<STR_LIT>' ] ) <EOL> log . debug ( '<STR_LIT>' . format ( func_args ) ) <EOL> if not func_args [ '<STR_LIT:name>' ] : <EOL> raise MoultScannerError ( '<STR_LIT>' ) <EOL> if func_args [ '<STR_LIT>' ] and not isinstance ( func_args [ '<STR_LIT>' ] , ( bytes , str_ ) ) : <EOL> raise MoultScannerError ( '<STR_LIT>' ) <EOL> imports . append ( ( func_args [ '<STR_LIT>' ] , func_args [ '<STR_LIT:name>' ] ) ) <EOL> return imports <EOL> class ResolvedImport ( object ) : <EOL> def __init__ ( self , import_path , import_root ) : <EOL> module = import_path . split ( '<STR_LIT:.>' , <NUM_LIT:1> ) [ <NUM_LIT:0> ] <EOL> self . module = module <EOL> self . import_path = import_path <EOL> self . is_stdlib = utils . is_stdlib ( module ) <EOL> self . filename = None <EOL> if not self . is_stdlib : <EOL> self . filename = utils . file_containing_import ( import_path , import_root ) <EOL> def __repr__ ( self ) : <EOL> return '<STR_LIT>' . format ( self . import_path , self . filename ) <EOL> class ImportNodeVisitor ( ast . NodeVisitor ) : <EOL> '''<STR_LIT>''' <EOL> def reset ( self , filename ) : <EOL> self . filename = filename <EOL> self . import_path , self . import_root = utils . import_path_from_file ( filename ) <EOL> def add_import ( self , * names ) : <EOL> for module , name in names : <EOL> if module and module . startswith ( '<STR_LIT:.>' ) : <EOL> module = utils . resolve_import ( module , self . import_path ) <EOL> elif not module : <EOL> module = '<STR_LIT>' <EOL> module = '<STR_LIT:.>' . join ( ( module , name . strip ( '<STR_LIT:.>' ) ) ) . strip ( '<STR_LIT:.>' ) <EOL> if module not in self . _imports : <EOL> self . _imports . add ( module ) <EOL> self . imports . append ( ResolvedImport ( module , self . import_root ) ) <EOL> def visit_Module ( self , node ) : <EOL> log . debug ( '<STR_LIT>' , self . import_path ) <EOL> self . _imports = set ( ) <EOL> self . imports = [ ] <EOL> self . scope = { } <EOL> if node : <EOL> self . generic_visit ( node ) <EOL> def visit_Import ( self , node ) : <EOL> for n in node . names : <EOL> self . add_import ( ( n . name , '<STR_LIT>' ) ) <EOL> self . generic_visit ( node ) <EOL> def visit_ImportFrom ( self , node ) : <EOL> module = '<STR_LIT>' . format ( '<STR_LIT:.>' * node . level , str_ ( node . module or '<STR_LIT>' ) ) <EOL> for n in node . names : <EOL> self . add_import ( ( module , n . name ) ) <EOL> self . generic_visit ( node ) <EOL> def visit_Expr ( self , node ) : <EOL> if isinstance ( node . value , ast . Call ) : <EOL> try : <EOL> self . add_import ( * parse_programmatic_import ( node . value , self . scope ) ) <EOL> except MoultScannerError as e : <EOL> log . debug ( '<STR_LIT>' , e , self . filename ) <EOL> elif isinstance ( node . value , ast . Name ) : <EOL> ast_value ( node . value , self . scope ) <EOL> self . generic_visit ( node ) <EOL> def visit_Assign ( self , node ) : <EOL> ast_value ( node , self . scope ) <EOL> def visit_Delete ( self , node ) : <EOL> ast_value ( node , self . scope ) <EOL> def visit ( self , node ) : <EOL> super ( ImportNodeVisitor , self ) . visit ( node ) <EOL> ast_visitor = ImportNodeVisitor ( ) <EOL> def _ast_scan_file_re ( filename ) : <EOL> try : <EOL> with io . open ( filename , '<STR_LIT>' , encoding = '<STR_LIT:utf8>' ) as fp : <EOL> script = fp . read ( ) <EOL> normalized = '<STR_LIT>' <EOL> for imp in _fallback_re . finditer ( script ) : <EOL> imp_line = imp . group ( <NUM_LIT:1> ) <EOL> try : <EOL> imp_line = imp_line . decode ( '<STR_LIT:utf8>' ) <EOL> except AttributeError : <EOL> pass <EOL> except UnicodeEncodeError : <EOL> log . warn ( '<STR_LIT>' , imp_line ) <EOL> continue <EOL> imp_line = re . sub ( r'<STR_LIT>' , '<STR_LIT>' , imp_line ) <EOL> normalized += '<STR_LIT:U+0020>' . join ( imp_line . split ( ) ) . strip ( '<STR_LIT:U+002C>' ) + '<STR_LIT:\n>' <EOL> log . debug ( '<STR_LIT>' , normalized ) <EOL> try : <EOL> root = ast . parse ( normalized , filename = filename ) <EOL> except SyntaxError : <EOL> log . error ( '<STR_LIT>' , filename ) <EOL> log . info ( '<STR_LIT>' , exc_info = True ) <EOL> return None , None <EOL> log . debug ( '<STR_LIT>' , filename ) <EOL> ast_visitor . reset ( filename ) <EOL> ast_visitor . visit ( root ) <EOL> return ast_visitor . scope , ast_visitor . imports <EOL> except IOError : <EOL> log . warn ( '<STR_LIT>' , filename ) <EOL> return None , None <EOL> def ast_scan_file ( filename , re_fallback = True ) : <EOL> '''<STR_LIT>''' <EOL> try : <EOL> with io . open ( filename , '<STR_LIT:rb>' ) as fp : <EOL> try : <EOL> root = ast . parse ( fp . read ( ) , filename = filename ) <EOL> except ( SyntaxError , IndentationError ) : <EOL> if re_fallback : <EOL> log . debug ( '<STR_LIT>' ) <EOL> return _ast_scan_file_re ( filename ) <EOL> else : <EOL> log . error ( '<STR_LIT>' , filename ) <EOL> log . info ( '<STR_LIT>' , exc_info = True ) <EOL> return None , None <EOL> log . debug ( '<STR_LIT>' , filename ) <EOL> ast_visitor . reset ( filename ) <EOL> ast_visitor . visit ( root ) <EOL> log . debug ( '<STR_LIT>' , ast_visitor . import_root ) <EOL> return ast_visitor . scope , ast_visitor . imports <EOL> except IOError : <EOL> log . warn ( '<STR_LIT>' , filename ) <EOL> return None , None </s>
