<s> from __future__ import ( absolute_import , division , generators , nested_scopes , print_function , <EOL> unicode_literals , with_statement ) <EOL> import json <EOL> import os <EOL> import sys <EOL> from collections import defaultdict , namedtuple <EOL> from pants . backend . jvm . targets . jar_library import JarLibrary <EOL> from pants . backend . jvm . tasks . classpath_util import ClasspathUtil <EOL> from pants . backend . jvm . tasks . jvm_dependency_analyzer import JvmDependencyAnalyzer <EOL> from pants . base . build_environment import get_buildroot <EOL> from pants . build_graph . resources import Resources <EOL> from pants . build_graph . target import Target <EOL> from pants . util . dirutil import fast_relpath <EOL> from pants . util . fileutil import create_size_estimators <EOL> class JvmDependencyUsage ( JvmDependencyAnalyzer ) : <EOL> """<STR_LIT>""" <EOL> size_estimators = create_size_estimators ( ) <EOL> @ classmethod <EOL> def register_options ( cls , register ) : <EOL> super ( JvmDependencyUsage , cls ) . register_options ( register ) <EOL> register ( '<STR_LIT>' , default = True , type = bool , <EOL> help = '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> register ( '<STR_LIT>' , default = True , type = bool , <EOL> help = '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> register ( '<STR_LIT>' , <EOL> choices = list ( cls . size_estimators . keys ( ) ) , default = '<STR_LIT>' , <EOL> help = '<STR_LIT>' ) <EOL> register ( '<STR_LIT>' , default = True , type = bool , <EOL> help = '<STR_LIT>' ) <EOL> register ( '<STR_LIT>' , type = str , <EOL> help = '<STR_LIT>' ) <EOL> register ( '<STR_LIT>' , type = bool , <EOL> help = '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> @ classmethod <EOL> def prepare ( cls , options , round_manager ) : <EOL> if not options . use_cached : <EOL> super ( JvmDependencyUsage , cls ) . prepare ( options , round_manager ) <EOL> round_manager . require_data ( '<STR_LIT>' ) <EOL> round_manager . require_data ( '<STR_LIT>' ) <EOL> round_manager . require_data ( '<STR_LIT>' ) <EOL> else : <EOL> round_manager . require_data ( '<STR_LIT>' ) <EOL> round_manager . require_data ( '<STR_LIT>' ) <EOL> round_manager . require_data ( '<STR_LIT>' ) <EOL> @ classmethod <EOL> def skip ( cls , options ) : <EOL> """<STR_LIT>""" <EOL> return False <EOL> def execute ( self ) : <EOL> graph = self . create_dep_usage_graph ( self . context . targets ( ) if self . get_options ( ) . transitive <EOL> else self . context . target_roots ) <EOL> output_file = self . get_options ( ) . output_file <EOL> if output_file : <EOL> self . context . log . info ( '<STR_LIT>' . format ( output_file ) ) <EOL> with open ( output_file , '<STR_LIT:w>' ) as fh : <EOL> self . _render ( graph , fh ) <EOL> else : <EOL> sys . stdout . write ( b'<STR_LIT:\n>' ) <EOL> self . _render ( graph , sys . stdout ) <EOL> @ classmethod <EOL> def implementation_version ( cls ) : <EOL> return super ( JvmDependencyUsage , cls ) . implementation_version ( ) + [ ( '<STR_LIT>' , <NUM_LIT:4> ) ] <EOL> def _render ( self , graph , fh ) : <EOL> chunks = graph . to_summary ( ) if self . get_options ( ) . summary else graph . to_json ( ) <EOL> for chunk in chunks : <EOL> fh . write ( chunk ) <EOL> fh . flush ( ) <EOL> def _resolve_aliases ( self , target ) : <EOL> """<STR_LIT>""" <EOL> for declared in target . dependencies : <EOL> if type ( declared ) == Target : <EOL> for r in self . _resolve_aliases ( declared ) : <EOL> yield r <EOL> else : <EOL> yield declared <EOL> def _is_declared_dep ( self , target , dep ) : <EOL> """<STR_LIT>""" <EOL> return dep in self . _resolve_aliases ( target ) <EOL> def _select ( self , target ) : <EOL> if self . get_options ( ) . internal_only and isinstance ( target , JarLibrary ) : <EOL> return False <EOL> elif isinstance ( target , Resources ) or type ( target ) == Target : <EOL> return False <EOL> else : <EOL> return True <EOL> def _normalize_product_dep ( self , buildroot , classes_by_source , dep ) : <EOL> """<STR_LIT>""" <EOL> if dep . endswith ( "<STR_LIT>" ) : <EOL> return set ( ) <EOL> elif dep . endswith ( "<STR_LIT>" ) : <EOL> return set ( [ dep ] ) <EOL> else : <EOL> rel_src = fast_relpath ( dep , buildroot ) <EOL> return set ( p for _ , paths in classes_by_source [ rel_src ] . rel_paths ( ) for p in paths ) <EOL> def _count_products ( self , classpath_products , target ) : <EOL> contents = ClasspathUtil . classpath_contents ( ( target , ) , classpath_products ) <EOL> return sum ( <NUM_LIT:1> for _ in contents ) <EOL> def create_dep_usage_graph ( self , targets ) : <EOL> """<STR_LIT>""" <EOL> with self . invalidated ( targets , <EOL> invalidate_dependents = True ) as invalidation_check : <EOL> target_to_vts = { } <EOL> for vts in invalidation_check . all_vts : <EOL> target_to_vts [ vts . target ] = vts <EOL> if not self . get_options ( ) . use_cached : <EOL> node_creator = self . calculating_node_creator ( <EOL> self . context . products . get_data ( '<STR_LIT>' ) , <EOL> self . context . products . get_data ( '<STR_LIT>' ) , <EOL> self . context . products . get_data ( '<STR_LIT>' ) , <EOL> target_to_vts ) <EOL> else : <EOL> node_creator = self . cached_node_creator ( target_to_vts ) <EOL> return DependencyUsageGraph ( self . create_dep_usage_nodes ( targets , node_creator ) , <EOL> self . size_estimators [ self . get_options ( ) . size_estimator ] ) <EOL> def calculating_node_creator ( self , classes_by_source , runtime_classpath , product_deps_by_src , <EOL> target_to_vts ) : <EOL> """<STR_LIT>""" <EOL> def creator ( target ) : <EOL> node = self . create_dep_usage_node ( target , get_buildroot ( ) , <EOL> classes_by_source , runtime_classpath , product_deps_by_src ) <EOL> vt = target_to_vts [ target ] <EOL> with open ( self . nodes_json ( vt . results_dir ) , mode = '<STR_LIT:w>' ) as fp : <EOL> json . dump ( node . to_cacheable_dict ( ) , fp , indent = <NUM_LIT:2> , sort_keys = True ) <EOL> vt . update ( ) <EOL> return node <EOL> return creator <EOL> def cached_node_creator ( self , target_to_vts ) : <EOL> """<STR_LIT>""" <EOL> def creator ( target ) : <EOL> vt = target_to_vts [ target ] <EOL> if vt . valid and os . path . exists ( self . nodes_json ( vt . results_dir ) ) : <EOL> try : <EOL> with open ( self . nodes_json ( vt . results_dir ) ) as fp : <EOL> return Node . from_cacheable_dict ( json . load ( fp ) , <EOL> lambda spec : self . context . resolve ( spec ) . __iter__ ( ) . next ( ) ) <EOL> except Exception : <EOL> self . context . log . warn ( "<STR_LIT>" . format ( target ) ) <EOL> return Node ( target . concrete_derived_from ) <EOL> else : <EOL> self . context . log . warn ( "<STR_LIT>" . format ( target ) ) <EOL> return Node ( target . concrete_derived_from ) <EOL> return creator <EOL> def nodes_json ( self , target_results_dir ) : <EOL> return os . path . join ( target_results_dir , '<STR_LIT>' ) <EOL> def create_dep_usage_nodes ( self , targets , node_creator ) : <EOL> nodes = dict ( ) <EOL> for target in targets : <EOL> if not self . _select ( target ) : <EOL> continue <EOL> concrete_target = target . concrete_derived_from <EOL> node = node_creator ( target ) <EOL> if concrete_target in nodes : <EOL> nodes [ concrete_target ] . combine ( node ) <EOL> else : <EOL> nodes [ concrete_target ] = node <EOL> for concrete_target , node in nodes . items ( ) [ : ] : <EOL> if node . products_total == <NUM_LIT:0> : <EOL> nodes . pop ( concrete_target ) <EOL> return nodes <EOL> def cache_target_dirs ( self ) : <EOL> return True <EOL> def create_dep_usage_node ( self , target , buildroot , classes_by_source , runtime_classpath , product_deps_by_src ) : <EOL> concrete_target = target . concrete_derived_from <EOL> products_total = self . _count_products ( runtime_classpath , target ) <EOL> node = Node ( concrete_target ) <EOL> node . add_derivation ( target , products_total ) <EOL> for dep_tgt in self . _resolve_aliases ( target ) : <EOL> derived_from = dep_tgt . concrete_derived_from <EOL> if self . _select ( derived_from ) : <EOL> node . add_edge ( Edge ( is_declared = True , products_used = set ( ) ) , derived_from ) <EOL> target_product_deps_by_src = product_deps_by_src . get ( target , dict ( ) ) <EOL> for src in target . sources_relative_to_buildroot ( ) : <EOL> for product_dep in target_product_deps_by_src . get ( os . path . join ( buildroot , src ) , [ ] ) : <EOL> for dep_tgt in self . targets_by_file . get ( product_dep , [ ] ) : <EOL> derived_from = dep_tgt . concrete_derived_from <EOL> if not self . _select ( derived_from ) : <EOL> continue <EOL> is_declared = self . _is_declared_dep ( target , dep_tgt ) <EOL> normalized_deps = self . _normalize_product_dep ( buildroot , classes_by_source , product_dep ) <EOL> node . add_edge ( Edge ( is_declared = is_declared , products_used = normalized_deps ) , derived_from ) <EOL> return node <EOL> class Node ( object ) : <EOL> def __init__ ( self , concrete_target ) : <EOL> self . concrete_target = concrete_target <EOL> self . products_total = <NUM_LIT:0> <EOL> self . derivations = set ( ) <EOL> self . dep_edges = defaultdict ( Edge ) <EOL> def add_derivation ( self , derived_target , derived_products ) : <EOL> self . derivations . add ( derived_target ) <EOL> self . products_total += derived_products <EOL> def add_edge ( self , edge , dest ) : <EOL> self . dep_edges [ dest ] += edge <EOL> def combine ( self , other_node ) : <EOL> assert other_node . concrete_target == self . concrete_target <EOL> self . products_total += other_node . products_total <EOL> self . derivations . update ( other_node . derivations ) <EOL> self . dep_edges . update ( other_node . dep_edges ) <EOL> def to_cacheable_dict ( self ) : <EOL> edges = { } <EOL> for dest in self . dep_edges : <EOL> edges [ dest . address . spec ] = { <EOL> '<STR_LIT>' : list ( self . dep_edges [ dest ] . products_used ) , <EOL> '<STR_LIT>' : self . dep_edges [ dest ] . is_declared , <EOL> } <EOL> return { <EOL> '<STR_LIT:target>' : self . concrete_target . address . spec , <EOL> '<STR_LIT>' : self . products_total , <EOL> '<STR_LIT>' : [ derivation . address . spec for derivation in self . derivations ] , <EOL> '<STR_LIT>' : edges <EOL> } <EOL> @ staticmethod <EOL> def from_cacheable_dict ( cached_dict , target_resolve_func ) : <EOL> res = Node ( target_resolve_func ( cached_dict [ '<STR_LIT:target>' ] ) ) <EOL> res . products_total = cached_dict [ '<STR_LIT>' ] <EOL> res . derivations . update ( [ target_resolve_func ( spec ) for spec in cached_dict [ '<STR_LIT>' ] ] ) <EOL> for edge in cached_dict [ '<STR_LIT>' ] : <EOL> res . dep_edges [ target_resolve_func ( edge ) ] = Edge ( <EOL> is_declared = cached_dict [ '<STR_LIT>' ] [ edge ] [ '<STR_LIT>' ] , <EOL> products_used = set ( cached_dict [ '<STR_LIT>' ] [ edge ] [ '<STR_LIT>' ] ) ) <EOL> return res <EOL> class Edge ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , is_declared = False , products_used = None ) : <EOL> self . products_used = products_used or set ( ) <EOL> self . is_declared = is_declared <EOL> def __iadd__ ( self , that ) : <EOL> self . products_used |= that . products_used <EOL> self . is_declared |= that . is_declared <EOL> return self <EOL> class DependencyUsageGraph ( object ) : <EOL> def __init__ ( self , nodes , size_estimator ) : <EOL> self . _nodes = nodes <EOL> self . _size_estimator = size_estimator <EOL> self . _cost_cache = { } <EOL> self . _trans_cost_cache = { } <EOL> def _cost ( self , target ) : <EOL> if target not in self . _cost_cache : <EOL> self . _cost_cache [ target ] = self . _size_estimator ( target . sources_relative_to_buildroot ( ) ) <EOL> return self . _cost_cache [ target ] <EOL> def _trans_cost ( self , target ) : <EOL> if target not in self . _trans_cost_cache : <EOL> dep_sum = sum ( self . _trans_cost ( dep ) for dep in target . dependencies ) <EOL> self . _trans_cost_cache [ target ] = self . _cost ( target ) + dep_sum <EOL> return self . _trans_cost_cache [ target ] <EOL> def _edge_type ( self , target , edge , dep ) : <EOL> if target == dep : <EOL> return '<STR_LIT>' <EOL> elif edge . is_declared : <EOL> return '<STR_LIT>' <EOL> else : <EOL> return '<STR_LIT>' <EOL> def _used_ratio ( self , dep_tgt , edge ) : <EOL> dep_tgt_products_total = max ( self . _nodes [ dep_tgt ] . products_total if dep_tgt in self . _nodes else <NUM_LIT:1> , <NUM_LIT:1> ) <EOL> return len ( edge . products_used ) / dep_tgt_products_total <EOL> def to_summary ( self ) : <EOL> """<STR_LIT>""" <EOL> max_target_usage = defaultdict ( lambda : <NUM_LIT:0.0> ) <EOL> for target , node in self . _nodes . items ( ) : <EOL> for dep_target , edge in node . dep_edges . items ( ) : <EOL> if target == dep_target : <EOL> continue <EOL> used_ratio = self . _used_ratio ( dep_target , edge ) <EOL> max_target_usage [ dep_target ] = max ( max_target_usage [ dep_target ] , used_ratio ) <EOL> Score = namedtuple ( '<STR_LIT>' , ( '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT:target>' ) ) <EOL> scores = [ ] <EOL> for target , max_usage in max_target_usage . items ( ) : <EOL> cost_transitive = self . _trans_cost ( target ) <EOL> score = int ( cost_transitive / ( max_usage if max_usage > <NUM_LIT:0.0> else <NUM_LIT:1.0> ) ) <EOL> scores . append ( Score ( score , max_usage , cost_transitive , target . address . spec ) ) <EOL> yield '<STR_LIT>' <EOL> first = True <EOL> for score in sorted ( scores , key = lambda s : s . badness ) : <EOL> yield '<STR_LIT>' . format ( '<STR_LIT>' if first else '<STR_LIT>' , json . dumps ( score . _asdict ( ) ) ) <EOL> first = False <EOL> yield '<STR_LIT>' <EOL> def to_json ( self ) : <EOL> """<STR_LIT>""" <EOL> res_dict = { } <EOL> def gen_dep_edge ( node , edge , dep_tgt ) : <EOL> return { <EOL> '<STR_LIT:target>' : dep_tgt . address . spec , <EOL> '<STR_LIT>' : self . _edge_type ( node . concrete_target , edge , dep_tgt ) , <EOL> '<STR_LIT>' : len ( edge . products_used ) , <EOL> '<STR_LIT>' : self . _used_ratio ( dep_tgt , edge ) , <EOL> } <EOL> for node in self . _nodes . values ( ) : <EOL> res_dict [ node . concrete_target . address . spec ] = { <EOL> '<STR_LIT>' : self . _cost ( node . concrete_target ) , <EOL> '<STR_LIT>' : self . _trans_cost ( node . concrete_target ) , <EOL> '<STR_LIT>' : node . products_total , <EOL> '<STR_LIT>' : [ gen_dep_edge ( node , edge , dep_tgt ) for dep_tgt , edge in node . dep_edges . items ( ) ] <EOL> } <EOL> yield json . dumps ( res_dict , indent = <NUM_LIT:2> , sort_keys = True ) </s>
