<s> """<STR_LIT>""" <EOL> import os <EOL> import sys <EOL> import ast <EOL> import json <EOL> import logging <EOL> import re <EOL> from contextlib import contextmanager <EOL> from distutils import sysconfig <EOL> from importmagic . util import parse_ast <EOL> LIB_LOCATIONS = sorted ( set ( ( <EOL> ( sysconfig . get_python_lib ( standard_lib = True ) , '<STR_LIT:S>' ) , <EOL> ( sysconfig . get_python_lib ( plat_specific = True ) , '<STR_LIT:3>' ) , <EOL> ( sysconfig . get_python_lib ( standard_lib = True , prefix = sys . prefix ) , '<STR_LIT:S>' ) , <EOL> ( sysconfig . get_python_lib ( plat_specific = True , prefix = sys . prefix ) , '<STR_LIT:3>' ) , <EOL> ( sysconfig . get_python_lib ( standard_lib = True , prefix = '<STR_LIT>' ) , '<STR_LIT:S>' ) , <EOL> ( sysconfig . get_python_lib ( plat_specific = True , prefix = '<STR_LIT>' ) , '<STR_LIT:3>' ) , <EOL> ) ) , key = lambda l : - len ( l [ <NUM_LIT:0> ] ) ) <EOL> DEFAULT_BLACKLIST_RE = re . compile ( r'<STR_LIT>' , re . I ) <EOL> BUILTIN_MODULES = sys . builtin_module_names + ( '<STR_LIT>' , ) <EOL> _PYTHON_VERSION = '<STR_LIT>' . format ( sys . version_info . major , sys . version_info . minor ) <EOL> LOCATION_BOOSTS = { <EOL> '<STR_LIT:3>' : <NUM_LIT> , <EOL> '<STR_LIT:L>' : <NUM_LIT> , <EOL> } <EOL> logger = logging . getLogger ( __name__ ) <EOL> class JSONEncoder ( json . JSONEncoder ) : <EOL> def default ( self , o ) : <EOL> if isinstance ( o , SymbolIndex ) : <EOL> d = o . _tree . copy ( ) <EOL> d . update ( ( '<STR_LIT:.>' + name , getattr ( o , name ) ) <EOL> for name in SymbolIndex . _SERIALIZED_ATTRIBUTES ) <EOL> if o . _lib_locations is not None : <EOL> d [ '<STR_LIT>' ] = o . _lib_locations <EOL> return d <EOL> return super ( JSONEncoder , self ) . default ( o ) <EOL> class SymbolIndex ( object ) : <EOL> PACKAGE_ALIASES = { <EOL> '<STR_LIT>' : ( os . path . __name__ , <NUM_LIT> ) , <EOL> '<STR_LIT>' : ( '<STR_LIT>' , <NUM_LIT> ) , <EOL> } <EOL> LOCATIONS = { <EOL> '<STR_LIT:F>' : '<STR_LIT>' , <EOL> '<STR_LIT:3>' : '<STR_LIT>' , <EOL> '<STR_LIT:S>' : '<STR_LIT>' , <EOL> '<STR_LIT:L>' : '<STR_LIT>' , <EOL> } <EOL> _PACKAGE_ALIASES = dict ( ( v [ <NUM_LIT:0> ] , ( k , v [ <NUM_LIT:1> ] ) ) for k , v in PACKAGE_ALIASES . items ( ) ) <EOL> _SERIALIZED_ATTRIBUTES = { '<STR_LIT>' : <NUM_LIT:1.0> , '<STR_LIT:location>' : '<STR_LIT:3>' } <EOL> def __init__ ( self , name = None , parent = None , score = <NUM_LIT:1.0> , location = '<STR_LIT:L>' , <EOL> blacklist_re = None , locations = None ) : <EOL> self . _name = name <EOL> self . _tree = { } <EOL> self . _exports = { } <EOL> self . _parent = parent <EOL> if blacklist_re : <EOL> self . _blacklist_re = blacklist_re <EOL> elif parent : <EOL> self . _blacklist_re = parent . _blacklist_re <EOL> else : <EOL> self . _blacklist_re = DEFAULT_BLACKLIST_RE <EOL> self . score = score <EOL> self . location = location <EOL> if parent is None : <EOL> self . _lib_locations = locations or LIB_LOCATIONS <EOL> else : <EOL> self . _lib_locations = None <EOL> if parent is None : <EOL> self . _merge_aliases ( ) <EOL> with self . enter ( '<STR_LIT>' , location = '<STR_LIT:F>' ) : <EOL> pass <EOL> with self . enter ( '<STR_LIT>' , location = '<STR_LIT:S>' ) : <EOL> pass <EOL> @ property <EOL> def lib_locations ( self ) : <EOL> if self . _parent : <EOL> return self . _parent . lib_locations <EOL> return self . _lib_locations <EOL> @ classmethod <EOL> def deserialize ( self , file ) : <EOL> def load ( tree , data , parent_location ) : <EOL> for key , value in data . items ( ) : <EOL> if isinstance ( value , dict ) : <EOL> score = value . pop ( '<STR_LIT>' , <NUM_LIT:1.0> ) <EOL> location = value . pop ( '<STR_LIT>' , parent_location ) <EOL> with tree . enter ( key , score = score , location = location ) as subtree : <EOL> load ( subtree , value , location ) <EOL> else : <EOL> assert isinstance ( value , float ) , '<STR_LIT>' % ( key , value ) <EOL> tree . add ( key , value ) <EOL> data = json . load ( file ) <EOL> data . pop ( '<STR_LIT>' , None ) <EOL> data . pop ( '<STR_LIT>' , None ) <EOL> tree = SymbolIndex ( locations = data . pop ( '<STR_LIT>' , LIB_LOCATIONS ) ) <EOL> load ( tree , data , '<STR_LIT:L>' ) <EOL> return tree <EOL> def index_source ( self , filename , source ) : <EOL> try : <EOL> st = parse_ast ( source , filename ) <EOL> except Exception as e : <EOL> logger . debug ( '<STR_LIT>' , filename , e ) <EOL> return False <EOL> visitor = SymbolVisitor ( self ) <EOL> visitor . visit ( st ) <EOL> return True <EOL> def index_file ( self , module , filename ) : <EOL> if self . _blacklist_re . search ( filename ) : <EOL> return <EOL> logger . debug ( '<STR_LIT>' , filename ) <EOL> with open ( filename , '<STR_LIT:rb>' ) as fd : <EOL> source = fd . read ( ) <EOL> with self . enter ( module , location = self . _determine_location_for ( filename ) ) as subtree : <EOL> success = subtree . index_source ( filename , source ) <EOL> if not success : <EOL> self . _tree . pop ( module , None ) <EOL> def index_path ( self , root ) : <EOL> """<STR_LIT>""" <EOL> basename = os . path . basename ( root ) <EOL> if os . path . splitext ( basename ) [ <NUM_LIT:0> ] != '<STR_LIT>' and basename . startswith ( '<STR_LIT:_>' ) : <EOL> return <EOL> location = self . _determine_location_for ( root ) <EOL> if os . path . isfile ( root ) : <EOL> self . _index_module ( root , location ) <EOL> elif os . path . isdir ( root ) and os . path . exists ( os . path . join ( root , '<STR_LIT>' ) ) : <EOL> self . _index_package ( root , location ) <EOL> def _index_package ( self , root , location ) : <EOL> basename = os . path . basename ( root ) <EOL> with self . enter ( basename , location = location ) as subtree : <EOL> for filename in os . listdir ( root ) : <EOL> subtree . index_path ( os . path . join ( root , filename ) ) <EOL> def _index_module ( self , root , location ) : <EOL> basename , ext = os . path . splitext ( os . path . basename ( root ) ) <EOL> if basename == '<STR_LIT>' : <EOL> basename = None <EOL> ext = ext . lower ( ) <EOL> import_path = '<STR_LIT:.>' . join ( filter ( None , [ self . path ( ) , basename ] ) ) <EOL> if import_path in BUILTIN_MODULES : <EOL> return <EOL> if ext == '<STR_LIT>' : <EOL> self . index_file ( basename , root ) <EOL> elif ext in ( '<STR_LIT>' , '<STR_LIT>' ) : <EOL> self . index_builtin ( import_path , location = location ) <EOL> def index_builtin ( self , name , location ) : <EOL> basename = name . rsplit ( '<STR_LIT:.>' , <NUM_LIT:1> ) [ - <NUM_LIT:1> ] <EOL> if basename . startswith ( '<STR_LIT:_>' ) : <EOL> return <EOL> logger . debug ( '<STR_LIT>' , name ) <EOL> try : <EOL> module = __import__ ( name , fromlist = [ '<STR_LIT:.>' ] ) <EOL> except Exception : <EOL> logger . debug ( '<STR_LIT>' , name ) <EOL> return <EOL> with self . enter ( basename , location = location ) as subtree : <EOL> for key , value in vars ( module ) . items ( ) : <EOL> if not key . startswith ( '<STR_LIT:_>' ) : <EOL> subtree . add ( key , <NUM_LIT> ) <EOL> def build_index ( self , paths ) : <EOL> for builtin in BUILTIN_MODULES : <EOL> self . index_builtin ( builtin , location = '<STR_LIT:S>' ) <EOL> for path in paths : <EOL> path = path or '<STR_LIT:.>' <EOL> if os . path . isdir ( path ) : <EOL> for filename in os . listdir ( path ) : <EOL> filename = os . path . join ( path , filename ) <EOL> self . index_path ( filename ) <EOL> def symbol_scores ( self , symbol ) : <EOL> """<STR_LIT>""" <EOL> scores = [ ] <EOL> path = [ ] <EOL> def fixup ( module , variable ) : <EOL> prefix = module . split ( '<STR_LIT:.>' ) <EOL> if variable is not None : <EOL> prefix . append ( variable ) <EOL> seeking = symbol . split ( '<STR_LIT:.>' ) <EOL> new_module = [ ] <EOL> while prefix and seeking [ <NUM_LIT:0> ] != prefix [ <NUM_LIT:0> ] : <EOL> new_module . append ( prefix . pop ( <NUM_LIT:0> ) ) <EOL> if new_module : <EOL> module , variable = '<STR_LIT:.>' . join ( new_module ) , prefix [ <NUM_LIT:0> ] <EOL> else : <EOL> variable = None <EOL> return module , variable <EOL> def score_walk ( scope , scale ) : <EOL> sub_path , score = self . _score_key ( scope , full_key ) <EOL> if score > <NUM_LIT:0.1> : <EOL> try : <EOL> i = sub_path . index ( None ) <EOL> sub_path , from_symbol = sub_path [ : i ] , '<STR_LIT:.>' . join ( sub_path [ i + <NUM_LIT:1> : ] ) <EOL> except ValueError : <EOL> from_symbol = None <EOL> package_path = '<STR_LIT:.>' . join ( path + sub_path ) <EOL> package_path , from_symbol = fixup ( package_path , from_symbol ) <EOL> scores . append ( ( score * scale , package_path , from_symbol ) ) <EOL> for key , subscope in scope . _tree . items ( ) : <EOL> if type ( subscope ) is not float : <EOL> path . append ( key ) <EOL> score_walk ( subscope , subscope . score * scale - <NUM_LIT:0.1> ) <EOL> path . pop ( ) <EOL> full_key = symbol . split ( '<STR_LIT:.>' ) <EOL> score_walk ( self , <NUM_LIT:1.0> ) <EOL> scores . sort ( reverse = True ) <EOL> return scores <EOL> def depth ( self ) : <EOL> depth = <NUM_LIT:0> <EOL> node = self <EOL> while node . _parent : <EOL> depth += <NUM_LIT:1> <EOL> node = node . _parent <EOL> return depth <EOL> def path ( self ) : <EOL> path = [ ] <EOL> node = self <EOL> while node and node . _name : <EOL> path . append ( node . _name ) <EOL> node = node . _parent <EOL> return '<STR_LIT:.>' . join ( reversed ( path ) ) <EOL> def add_explicit_export ( self , name , score ) : <EOL> self . _exports [ name ] = score <EOL> self . add ( name , score ) <EOL> def find ( self , path ) : <EOL> """<STR_LIT>""" <EOL> path = path . split ( '<STR_LIT:.>' ) <EOL> node = self <EOL> while node . _parent : <EOL> node = node . _parent <EOL> for name in path : <EOL> node = node . _tree . get ( name , None ) <EOL> if node is None or type ( node ) is float : <EOL> return None <EOL> return node <EOL> def location_for ( self , path ) : <EOL> """<STR_LIT>""" <EOL> path = path . split ( '<STR_LIT:.>' ) <EOL> node = self <EOL> while node . _parent : <EOL> node = node . _parent <EOL> location = node . location <EOL> for name in path : <EOL> tree = node . _tree . get ( name , None ) <EOL> if tree is None or type ( tree ) is float : <EOL> return location <EOL> location = tree . location <EOL> return location <EOL> def add ( self , name , score ) : <EOL> current_score = self . _tree . get ( name , <NUM_LIT:0.0> ) <EOL> if isinstance ( current_score , float ) and score > current_score : <EOL> self . _tree [ name ] = score <EOL> @ contextmanager <EOL> def enter ( self , name , location = '<STR_LIT:L>' , score = <NUM_LIT:1.0> ) : <EOL> if name is None : <EOL> tree = self <EOL> else : <EOL> tree = self . _tree . get ( name ) <EOL> if not isinstance ( tree , SymbolIndex ) : <EOL> tree = self . _tree [ name ] = SymbolIndex ( name , self , score = score , location = location ) <EOL> if tree . path ( ) in SymbolIndex . _PACKAGE_ALIASES : <EOL> alias_path , _ = SymbolIndex . _PACKAGE_ALIASES [ tree . path ( ) ] <EOL> alias = self . find ( alias_path ) <EOL> alias . _tree = tree . _tree <EOL> yield tree <EOL> if tree . _exports : <EOL> for key in set ( tree . _tree ) - set ( tree . _exports ) : <EOL> del tree . _tree [ key ] <EOL> def serialize ( self , fd = None ) : <EOL> if fd is None : <EOL> return json . dumps ( self , cls = JSONEncoder ) <EOL> return json . dump ( self , fd , cls = JSONEncoder ) <EOL> def boost ( self ) : <EOL> return LOCATION_BOOSTS . get ( self . location , <NUM_LIT:1.0> ) <EOL> def __repr__ ( self ) : <EOL> return '<STR_LIT>' % ( self . location , self . score , self . _tree ) <EOL> def _merge_aliases ( self ) : <EOL> def create ( node , alias , score ) : <EOL> if not alias : <EOL> return <EOL> name = alias . pop ( <NUM_LIT:0> ) <EOL> with node . enter ( name , location = '<STR_LIT:S>' , score = <NUM_LIT:1.0> if alias else score ) as index : <EOL> create ( index , alias , score ) <EOL> for alias in sorted ( SymbolIndex . _PACKAGE_ALIASES ) : <EOL> package , score = SymbolIndex . _PACKAGE_ALIASES [ alias ] <EOL> create ( self , package . split ( '<STR_LIT:.>' ) , score ) <EOL> def _score_key ( self , scope , key ) : <EOL> if not key : <EOL> return [ ] , <NUM_LIT:0.0> <EOL> key_score = value = scope . _tree . get ( key [ <NUM_LIT:0> ] , None ) <EOL> if value is None : <EOL> return [ ] , <NUM_LIT:0.0> <EOL> if type ( value ) is float : <EOL> return [ None , key [ <NUM_LIT:0> ] ] , key_score * scope . boost ( ) <EOL> else : <EOL> path , score = self . _score_key ( value , key [ <NUM_LIT:1> : ] ) <EOL> return [ key [ <NUM_LIT:0> ] ] + path , ( score + value . score ) * scope . boost ( ) <EOL> def _determine_location_for ( self , path ) : <EOL> parts = path . split ( os . path . sep ) <EOL> if '<STR_LIT>' in parts : <EOL> return '<STR_LIT:3>' <EOL> elif _PYTHON_VERSION in parts : <EOL> return '<STR_LIT:S>' <EOL> for dir , location in self . lib_locations : <EOL> if path . startswith ( dir ) : <EOL> return location <EOL> return '<STR_LIT:L>' <EOL> class SymbolVisitor ( ast . NodeVisitor ) : <EOL> def __init__ ( self , tree ) : <EOL> self . _tree = tree <EOL> def visit_ImportFrom ( self , node ) : <EOL> for name in node . names : <EOL> if name . name == '<STR_LIT:*>' or name . name . startswith ( '<STR_LIT:_>' ) : <EOL> continue <EOL> self . _tree . add ( name . name , <NUM_LIT> ) <EOL> def visit_Import ( self , node ) : <EOL> for name in node . names : <EOL> if name . name . startswith ( '<STR_LIT:_>' ) : <EOL> continue <EOL> self . _tree . add ( name . name , <NUM_LIT> ) <EOL> def visit_ClassDef ( self , node ) : <EOL> if not node . name . startswith ( '<STR_LIT:_>' ) : <EOL> self . _tree . add ( node . name , <NUM_LIT> ) <EOL> def visit_FunctionDef ( self , node ) : <EOL> if not node . name . startswith ( '<STR_LIT:_>' ) : <EOL> self . _tree . add ( node . name , <NUM_LIT> ) <EOL> def visit_Assign ( self , node ) : <EOL> is_name = lambda n : isinstance ( n , ast . Name ) <EOL> for name in filter ( is_name , node . targets ) : <EOL> if name . id == '<STR_LIT>' and isinstance ( node . value , ast . List ) : <EOL> for subnode in node . value . elts : <EOL> if isinstance ( subnode , ast . Str ) : <EOL> self . _tree . add_explicit_export ( subnode . s , <NUM_LIT> ) <EOL> elif not name . id . startswith ( '<STR_LIT:_>' ) : <EOL> self . _tree . add ( name . id , <NUM_LIT> ) <EOL> def visit_If ( self , node ) : <EOL> pass <EOL> if __name__ == '<STR_LIT:__main__>' : <EOL> tree = SymbolIndex ( ) <EOL> tree . build_index ( sys . path ) <EOL> tree . serialize ( sys . stdout ) </s>
