<s> import sys <EOL> import opcode <EOL> import dis <EOL> import imp <EOL> import os <EOL> import __builtin__ <EOL> import time <EOL> """<STR_LIT>""" <EOL> class System : <EOL> def __init__ ( self ) : <EOL> self . modules = { } <EOL> self . pendingmodules = { } <EOL> class Scope ( object ) : <EOL> def __init__ ( self , parent = None ) : <EOL> self . modvars = { } <EOL> self . parent = parent <EOL> self . varsources = { } <EOL> def mod_for_name ( self , name ) : <EOL> if name in self . modvars : <EOL> return self . modvars [ name ] <EOL> elif self . parent is not None : <EOL> return self . parent . mod_for_name ( name ) <EOL> else : <EOL> return None <EOL> def var_source ( self , name ) : <EOL> if name in self . varsources : <EOL> return self . varsources [ name ] <EOL> elif self . parent is not None : <EOL> return self . parent . var_source ( name ) <EOL> else : <EOL> return None , None <EOL> class Module ( object ) : <EOL> def __init__ ( self , name , system ) : <EOL> self . name = name <EOL> self . system = system <EOL> self . _imports = { } <EOL> self . definitions = [ '<STR_LIT>' ] <EOL> if name == '<STR_LIT>' : <EOL> self . definitions . extend ( [ <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] ) <EOL> self . toplevelscope = Scope ( ) <EOL> self . importers = [ ] <EOL> def import_ ( self , modname ) : <EOL> if modname not in self . _imports : <EOL> if recursive and modname not in self . system . modules : <EOL> self . system . pendingmodules [ modname ] = None <EOL> self . _imports [ modname ] = { } <EOL> return self . _imports [ modname ] <EOL> def iteropcodes ( codestring ) : <EOL> n = len ( codestring ) <EOL> i = <NUM_LIT:0> <EOL> while i < n : <EOL> op = ord ( codestring [ i ] ) <EOL> i += <NUM_LIT:1> <EOL> oparg = None <EOL> assert op != opcode . EXTENDED_ARG , '<STR_LIT>' <EOL> if op >= opcode . HAVE_ARGUMENT : <EOL> oparg = ord ( codestring [ i ] ) + ord ( codestring [ i + <NUM_LIT:1> ] ) * <NUM_LIT> <EOL> i += <NUM_LIT:2> <EOL> yield op , oparg <EOL> class _Op ( object ) : <EOL> def __getattr__ ( self , name ) : <EOL> if name in opcode . opmap : <EOL> return opcode . opmap [ name ] <EOL> else : <EOL> raise AttributeError , name <EOL> _op_ = _Op ( ) <EOL> loadops = [ _op_ . LOAD_NAME , _op_ . LOAD_GLOBAL , _op_ . LOAD_FAST , _op_ . LOAD_DEREF ] <EOL> storeops = [ _op_ . STORE_NAME , _op_ . STORE_FAST , _op_ . STORE_DEREF , _op_ . STORE_GLOBAL ] <EOL> def name_for_op ( code , op , oparg ) : <EOL> if op in [ _op_ . LOAD_GLOBAL , _op_ . STORE_GLOBAL , _op_ . LOAD_NAME , _op_ . STORE_NAME ] : <EOL> return code . co_names [ oparg ] <EOL> elif op in [ _op_ . LOAD_FAST , _op_ . STORE_FAST ] : <EOL> return code . co_varnames [ oparg ] <EOL> elif op in [ _op_ . LOAD_DEREF , _op_ . STORE_DEREF ] : <EOL> if oparg < len ( code . co_cellvars ) : <EOL> return code . co_cellvars [ oparg ] <EOL> else : <EOL> return code . co_freevars [ oparg - len ( code . co_cellvars ) ] <EOL> else : <EOL> assert <NUM_LIT:0> , "<STR_LIT>" % ( opcode . opname [ op ] , ) <EOL> def process ( r , codeob , scope , toplevel = False ) : <EOL> opcodes = list ( iteropcodes ( codeob . co_code ) ) <EOL> i = <NUM_LIT:0> <EOL> codeobjs = [ ] <EOL> while i < len ( opcodes ) : <EOL> op , oparg = opcodes [ i ] <EOL> if op == _op_ . IMPORT_NAME : <EOL> preop , preoparg = opcodes [ i - <NUM_LIT:1> ] <EOL> assert preop == _op_ . LOAD_CONST , '<STR_LIT>' <EOL> fromlist = codeob . co_consts [ preoparg ] <EOL> modname = codeob . co_names [ oparg ] <EOL> if fromlist is None : <EOL> r . import_ ( modname ) <EOL> seenloadattr = False <EOL> while <NUM_LIT:1> : <EOL> postop , postoparg = opcodes [ i + <NUM_LIT:1> ] <EOL> i += <NUM_LIT:1> <EOL> if postop != _op_ . LOAD_ATTR : <EOL> break <EOL> seenloadattr = True <EOL> assert postop in storeops , '<STR_LIT>' <EOL> storename = name_for_op ( codeob , postop , postoparg ) <EOL> if seenloadattr : <EOL> scope . modvars [ storename ] = modname <EOL> else : <EOL> scope . modvars [ storename ] = modname . split ( '<STR_LIT:.>' ) [ <NUM_LIT:0> ] <EOL> elif fromlist == ( '<STR_LIT:*>' , ) : <EOL> assert toplevel , '<STR_LIT>' <EOL> if modname . startswith ( '<STR_LIT>' ) : <EOL> if modname not in r . system . modules : <EOL> if modname in r . system . pendingmodules : <EOL> del r . system . pendingmodules [ modname ] <EOL> process_module ( modname , r . system ) <EOL> M = r . system . modules [ modname ] <EOL> for d in M . definitions + list ( M . toplevelscope . modvars ) + [ a [ <NUM_LIT:1> ] for a in M . toplevelscope . varsources . itervalues ( ) ] : <EOL> if d [ <NUM_LIT:0> ] != '<STR_LIT:_>' : <EOL> scope . varsources [ d ] = modname , d <EOL> r . import_ ( modname ) [ d ] = - <NUM_LIT:1> <EOL> r . import_ ( modname ) [ '<STR_LIT:*>' ] = True <EOL> else : <EOL> path = None <EOL> try : <EOL> for part in modname . split ( '<STR_LIT:.>' ) : <EOL> path = [ imp . find_module ( part , path ) [ <NUM_LIT:1> ] ] <EOL> except ImportError : <EOL> path = - <NUM_LIT:1> <EOL> i += <NUM_LIT:1> <EOL> for f in fromlist : <EOL> op , oparg = opcodes [ i ] <EOL> assert op == _op_ . IMPORT_FROM , '<STR_LIT>' <EOL> assert codeob . co_names [ oparg ] == f , '<STR_LIT:f>' <EOL> i += <NUM_LIT:1> <EOL> if path == - <NUM_LIT:1> : <EOL> i += <NUM_LIT:1> <EOL> continue <EOL> op , oparg = opcodes [ i ] <EOL> i += <NUM_LIT:1> <EOL> assert op in storeops , '<STR_LIT>' <EOL> storename = name_for_op ( codeob , op , oparg ) <EOL> try : <EOL> imp . find_module ( f , path ) <EOL> except ImportError : <EOL> r . import_ ( modname ) [ f ] = False <EOL> scope . varsources [ storename ] = modname , f <EOL> else : <EOL> submod = modname + '<STR_LIT:.>' + f <EOL> r . import_ ( submod ) <EOL> scope . modvars [ storename ] = submod <EOL> op , oparg = opcodes [ i ] <EOL> assert op == _op_ . POP_TOP , '<STR_LIT>' <EOL> elif op == _op_ . STORE_NAME and toplevel or op == _op_ . STORE_GLOBAL : <EOL> r . definitions . append ( codeob . co_names [ oparg ] ) <EOL> elif op == _op_ . LOAD_ATTR : <EOL> preop , preoparg = opcodes [ i - <NUM_LIT:1> ] <EOL> if preop in loadops : <EOL> name = name_for_op ( codeob , preop , preoparg ) <EOL> m = scope . mod_for_name ( name ) <EOL> if m : <EOL> r . import_ ( m ) [ codeob . co_names [ oparg ] ] = True <EOL> elif op in loadops : <EOL> name = name_for_op ( codeob , op , oparg ) <EOL> m , a = scope . var_source ( name ) <EOL> if m : <EOL> assert a in r . import_ ( m ) , '<STR_LIT:a>' <EOL> r . import_ ( m ) [ a ] = True <EOL> elif op in [ _op_ . MAKE_FUNCTION , _op_ . MAKE_CLOSURE ] : <EOL> preop , preoparg = opcodes [ i - <NUM_LIT:1> ] <EOL> assert preop == _op_ . LOAD_CONST , '<STR_LIT>' <EOL> codeobjs . append ( codeob . co_consts [ preoparg ] ) <EOL> i += <NUM_LIT:1> <EOL> for c in codeobjs : <EOL> process ( r , c , Scope ( scope ) ) <EOL> def find_from_dotted_name ( modname ) : <EOL> path = None <EOL> for part in modname . split ( '<STR_LIT:.>' ) : <EOL> try : <EOL> path = [ imp . find_module ( part , path ) [ <NUM_LIT:1> ] ] <EOL> except ImportError : <EOL> print modname <EOL> raise <EOL> return path [ <NUM_LIT:0> ] <EOL> def process_module ( dottedname , system ) : <EOL> if dottedname . endswith ( '<STR_LIT>' ) : <EOL> path = dottedname <EOL> dottedname = path . lstrip ( '<STR_LIT>' ) . rstrip ( ) [ : - <NUM_LIT:3> ] . replace ( '<STR_LIT:/>' , '<STR_LIT:.>' ) <EOL> else : <EOL> path = find_from_dotted_name ( dottedname ) <EOL> ispackage = False <EOL> if os . path . isdir ( path ) : <EOL> ispackage = True <EOL> path += '<STR_LIT>' <EOL> r = Module ( dottedname , system ) <EOL> r . ispackage = ispackage <EOL> if dottedname in system . modules : <EOL> return system . modules [ dottedname ] <EOL> try : <EOL> code = compile ( open ( path , "<STR_LIT>" ) . read ( ) , path , '<STR_LIT>' ) <EOL> process ( r , code , r . toplevelscope , True ) <EOL> except ( ImportError , SyntaxError ) , e : <EOL> print "<STR_LIT>" , e <EOL> else : <EOL> if dottedname in system . pendingmodules : <EOL> print <EOL> del system . pendingmodules [ dottedname ] <EOL> system . modules [ dottedname ] = r <EOL> return r <EOL> def report_unused_symbols ( system ) : <EOL> for name , mod in sorted ( system . modules . iteritems ( ) ) : <EOL> printed = False <EOL> if not '<STR_LIT>' in name or '<STR_LIT>' in name : <EOL> continue <EOL> u = { } <EOL> for n in mod . _imports : <EOL> if n in ( '<STR_LIT>' , '<STR_LIT>' ) : <EOL> continue <EOL> usedany = False <EOL> for field , used in mod . _imports [ n ] . iteritems ( ) : <EOL> if n in system . modules : <EOL> M = system . modules [ n ] <EOL> if not M . ispackage and field != '<STR_LIT:*>' and field not in M . definitions and used != - <NUM_LIT:1> : <EOL> if not printed : <EOL> print '<STR_LIT:*>' , name <EOL> printed = True <EOL> sourcemod , nam = M . toplevelscope . var_source ( field ) <EOL> print '<STR_LIT:U+0020>' , field , '<STR_LIT>' , n , '<STR_LIT>' , sourcemod <EOL> if not used : <EOL> u . setdefault ( n , [ ] ) . append ( field ) <EOL> else : <EOL> usedany = True <EOL> if not usedany : <EOL> if n in u : <EOL> u [ n ] . append ( '<STR_LIT>' ) <EOL> else : <EOL> u [ n ] = '<STR_LIT>' <EOL> if u : <EOL> if not printed : <EOL> print '<STR_LIT:*>' , name <EOL> printed = True <EOL> for k , v in u . iteritems ( ) : <EOL> print '<STR_LIT:U+0020>' , k , v <EOL> def find_cycles ( system ) : <EOL> from pypy . tool . algo import graphlib <EOL> vertices = dict . fromkeys ( system . modules ) <EOL> edges = { } <EOL> for m in system . modules : <EOL> edges [ m ] = [ ] <EOL> for n in system . modules [ m ] . _imports : <EOL> edges [ m ] . append ( graphlib . Edge ( m , n ) ) <EOL> cycles = [ ] <EOL> for component in graphlib . strong_components ( vertices , edges ) : <EOL> random_vertex = component . iterkeys ( ) . next ( ) <EOL> cycles . extend ( graphlib . all_cycles ( random_vertex , component , edges ) ) <EOL> ncycles = [ ] <EOL> for cycle in cycles : <EOL> packs = { } <EOL> for edge in cycle : <EOL> package = edge . source . rsplit ( '<STR_LIT:.>' , <NUM_LIT:1> ) [ <NUM_LIT:0> ] <EOL> packs [ package ] = True <EOL> if len ( packs ) > <NUM_LIT:1> : <EOL> ncycles . append ( cycle ) <EOL> cycles = ncycles <EOL> for cycle in cycles : <EOL> l = len ( cycle [ <NUM_LIT:0> ] . source ) <EOL> print cycle [ <NUM_LIT:0> ] . source , '<STR_LIT>' , cycle [ <NUM_LIT:0> ] . target <EOL> for edge in cycle [ <NUM_LIT:1> : ] : <EOL> print '<STR_LIT:U+0020>' * l , '<STR_LIT>' , edge . target <EOL> print len ( cycles ) , '<STR_LIT>' <EOL> def summary ( system ) : <EOL> mcount = float ( len ( system . modules ) ) <EOL> importcount = <NUM_LIT:0> <EOL> importstars = <NUM_LIT:0> <EOL> importstarusage = <NUM_LIT:0> <EOL> defcount = <NUM_LIT:0> <EOL> importedcount = <NUM_LIT:0> <EOL> for m in system . modules : <EOL> m = system . modules [ m ] <EOL> defcount += len ( m . definitions ) <EOL> importedcount += len ( m . importers ) <EOL> importcount += len ( m . _imports ) <EOL> for n in m . _imports : <EOL> if '<STR_LIT:*>' in m . _imports [ n ] : <EOL> importstars += <NUM_LIT:1> <EOL> importstarusage += len ( [ o for ( o , v ) in m . _imports [ n ] . iteritems ( ) if v == True ] ) <EOL> print <EOL> print '<STR_LIT>' <EOL> print '<STR_LIT>' % ( importedcount / mcount ) <EOL> print '<STR_LIT>' % ( importcount / mcount ) <EOL> print '<STR_LIT>' % ( defcount / mcount ) <EOL> print <EOL> print '<STR_LIT>' % ( importstars ) <EOL> print '<STR_LIT>' % ( ( <NUM_LIT:1.0> * importstarusage ) / importstars ) <EOL> def not_imported ( system ) : <EOL> for m , M in sorted ( system . modules . iteritems ( ) ) : <EOL> if not M . importers and '<STR_LIT:test>' not in m and '<STR_LIT>' not in m : <EOL> print m <EOL> def import_stars ( system ) : <EOL> for m in sorted ( system . modules ) : <EOL> m = system . modules [ m ] <EOL> for n in sorted ( m . _imports ) : <EOL> if '<STR_LIT:*>' in m . _imports [ n ] : <EOL> print m . name , '<STR_LIT>' , n <EOL> used = [ o for ( o , v ) in m . _imports [ n ] . iteritems ( ) if v == True and o != '<STR_LIT:*>' ] <EOL> print len ( used ) , '<STR_LIT>' , len ( m . _imports [ n ] ) - <NUM_LIT:1> , '<STR_LIT>' <EOL> print '<STR_LIT:U+0020>' , '<STR_LIT:U+002CU+0020>' . join ( sorted ( used ) ) <EOL> def find_varargs_users ( system ) : <EOL> for m in sorted ( system . modules ) : <EOL> m = system . modules [ m ] <EOL> if '<STR_LIT>' in m . _imports : <EOL> if m . _imports [ '<STR_LIT>' ] . get ( '<STR_LIT>' ) == True : <EOL> print m . name <EOL> def file_for_module ( module ) : <EOL> fname = os . path . join ( '<STR_LIT>' , * module . name . split ( '<STR_LIT:.>' ) ) + '<STR_LIT>' <EOL> dname = os . path . dirname ( fname ) <EOL> if not os . path . isdir ( dname ) : <EOL> os . makedirs ( dname ) <EOL> return open ( fname , '<STR_LIT:w>' ) <EOL> def link_for_module ( fromlink , module ) : <EOL> link = '<STR_LIT:/>' . join ( module . name . split ( '<STR_LIT:.>' ) ) + '<STR_LIT>' <EOL> prefix = '<STR_LIT:/>' . join ( [ '<STR_LIT:..>' ] * fromlink . count ( '<STR_LIT:/>' ) ) <EOL> if prefix : <EOL> return prefix + '<STR_LIT:/>' + link <EOL> else : <EOL> return link <EOL> def file_for_name ( module , name ) : <EOL> fname = os . path . join ( '<STR_LIT>' , * ( module . name . split ( '<STR_LIT:.>' ) + [ name ] ) ) + '<STR_LIT>' <EOL> dname = os . path . dirname ( fname ) <EOL> if not os . path . isdir ( dname ) : <EOL> os . makedirs ( dname ) <EOL> return open ( fname , '<STR_LIT:w>' ) <EOL> def link_for_name ( fromlink , module , name ) : <EOL> link = '<STR_LIT:/>' . join ( module . name . split ( '<STR_LIT:.>' ) + [ name ] ) + '<STR_LIT>' <EOL> prefix = '<STR_LIT:/>' . join ( [ '<STR_LIT:..>' ] * fromlink . count ( '<STR_LIT:/>' ) ) <EOL> if prefix : <EOL> return prefix + '<STR_LIT:/>' + link <EOL> else : <EOL> return link <EOL> def html_for_module ( module ) : <EOL> from py . xml import html <EOL> out = file_for_module ( module ) <EOL> ourlink = link_for_module ( '<STR_LIT>' , module ) <EOL> head = [ html . title ( module . name ) ] <EOL> body = [ html . h1 ( module . name ) ] <EOL> body . append ( html . p ( '<STR_LIT>' ) ) <EOL> listbody = [ ] <EOL> defuses = { } <EOL> for d in module . definitions : <EOL> uses = [ ] <EOL> for n in sorted ( module . importers ) : <EOL> N = module . system . modules [ n ] <EOL> if N . _imports [ module . name ] . get ( d ) == True : <EOL> uses . append ( n ) <EOL> if not d . startswith ( '<STR_LIT:_>' ) : <EOL> if uses : <EOL> listbody . append ( html . li ( <EOL> html . a ( d , href = link_for_name ( ourlink , module , d ) ) ) ) <EOL> defuses [ d ] = uses <EOL> else : <EOL> listbody . append ( html . li ( d ) ) <EOL> body . append ( html . ul ( listbody ) ) <EOL> body . append ( html . p ( '<STR_LIT>' ) ) <EOL> listbody1 = [ ] <EOL> for n in sorted ( module . _imports ) : <EOL> if n in ( '<STR_LIT>' , '<STR_LIT>' ) : <EOL> continue <EOL> if n in module . system . modules : <EOL> listbody2 = [ html . a ( <EOL> n , href = link_for_module ( ourlink , module . system . modules [ n ] ) ) ] <EOL> else : <EOL> listbody2 = [ n ] <EOL> listbody3 = [ ] <EOL> for o in sorted ( module . _imports [ n ] ) : <EOL> if module . _imports [ n ] [ o ] == True : <EOL> if n in module . system . modules : <EOL> listbody3 . append ( <EOL> html . li ( html . a ( <EOL> o , href = link_for_name ( ourlink , module . system . modules [ n ] , o ) ) ) ) <EOL> else : <EOL> listbody3 . append ( html . li ( o ) ) <EOL> if listbody3 : <EOL> listbody2 . append ( html . ul ( listbody3 ) ) <EOL> listbody1 . append ( html . li ( listbody2 ) ) <EOL> body . append ( html . ul ( listbody1 ) ) <EOL> body . append ( html . p ( '<STR_LIT>' ) ) <EOL> listbody1 = [ ] <EOL> for n in module . importers : <EOL> licontents = [ html . a ( n , href = link_for_module ( ourlink , module . system . modules [ n ] ) ) ] <EOL> contents = [ ] <EOL> for o in sorted ( module . system . modules [ n ] . _imports [ module . name ] ) : <EOL> contents . append ( html . li ( html . a ( o , href = link_for_name ( ourlink , module , o ) ) ) ) <EOL> if contents : <EOL> licontents . append ( html . ul ( contents ) ) <EOL> listbody1 . append ( html . li ( licontents ) ) <EOL> body . append ( html . ul ( listbody1 ) ) <EOL> out . write ( html . html ( head , body ) . unicode ( ) ) <EOL> for d in defuses : <EOL> out = file_for_name ( module , d ) <EOL> ourlink = link_for_name ( '<STR_LIT>' , module , d ) <EOL> head = [ html . title ( module . name + '<STR_LIT:.>' + d ) ] <EOL> body = [ html . h1 ( [ html . a ( module . name , href = link_for_module ( ourlink , module ) ) , '<STR_LIT:.>' + d ] ) ] <EOL> contents = [ ] <EOL> for n in defuses [ d ] : <EOL> N = module . system . modules [ n ] <EOL> contents . append ( html . li ( html . a ( n , href = link_for_module ( ourlink , N ) ) ) ) <EOL> body . append ( html . p ( '<STR_LIT>' ) ) <EOL> body . append ( html . ul ( contents ) ) <EOL> out . write ( html . html ( head , body ) . unicode ( ) ) <EOL> def make_html_report ( system ) : <EOL> if os . path . isdir ( '<STR_LIT>' ) : <EOL> os . system ( '<STR_LIT>' ) <EOL> os . mkdir ( '<STR_LIT>' ) <EOL> for m in system . modules . itervalues ( ) : <EOL> html_for_module ( m ) <EOL> def main ( * paths ) : <EOL> system = System ( ) <EOL> for path in paths : <EOL> system . pendingmodules [ path ] = None <EOL> T = time . time ( ) <EOL> while system . pendingmodules : <EOL> path , d = system . pendingmodules . popitem ( ) <EOL> if sys . stdout . isatty ( ) : <EOL> print '<STR_LIT>' , len ( system . modules ) , '<STR_LIT:/>' , len ( system . pendingmodules ) , path , <EOL> sys . stdout . flush ( ) <EOL> if '<STR_LIT>' in path or '<STR_LIT>' in path : <EOL> continue <EOL> if '<STR_LIT:/>' not in path and not path . startswith ( '<STR_LIT>' ) : <EOL> continue <EOL> process_module ( path , system ) <EOL> print <EOL> print '<STR_LIT>' , len ( system . modules ) , '<STR_LIT>' % ( time . time ( ) - T ) <EOL> print '<STR_LIT>' <EOL> for name , mod in system . modules . iteritems ( ) : <EOL> for n in mod . _imports : <EOL> if n in system . modules : <EOL> system . modules [ n ] . importers . append ( name ) <EOL> make_html_report ( system ) <EOL> if interactive : <EOL> import pdb <EOL> pdb . set_trace ( ) <EOL> recursive = False <EOL> interactive = False <EOL> if __name__ == '<STR_LIT:__main__>' : <EOL> if '<STR_LIT>' in sys . argv : <EOL> recursive = True <EOL> sys . argv . remove ( '<STR_LIT>' ) <EOL> if '<STR_LIT>' in sys . argv : <EOL> interactive = True <EOL> sys . argv . remove ( '<STR_LIT>' ) <EOL> if len ( sys . argv ) > <NUM_LIT:1> : <EOL> main ( * sys . argv [ <NUM_LIT:1> : ] ) <EOL> else : <EOL> paths = [ ] <EOL> for line in os . popen ( "<STR_LIT>" ) : <EOL> paths . append ( line [ : - <NUM_LIT:1> ] ) <EOL> main ( * paths ) </s>
