<s> from __future__ import ( absolute_import , division , generators , nested_scopes , print_function , <EOL> unicode_literals , with_statement ) <EOL> import errno <EOL> import json <EOL> import logging <EOL> import os <EOL> import pkgutil <EOL> import threading <EOL> import xml . etree . ElementTree as ET <EOL> from abc import abstractmethod <EOL> from collections import OrderedDict , defaultdict , namedtuple <EOL> import six <EOL> from twitter . common . collections import OrderedSet <EOL> from pants . backend . jvm . jar_dependency_utils import M2Coordinate , ResolvedJar <EOL> from pants . backend . jvm . subsystems . jar_dependency_management import ( JarDependencyManagement , <EOL> PinnedJarArtifactSet ) <EOL> from pants . backend . jvm . targets . exclude import Exclude <EOL> from pants . backend . jvm . targets . jar_dependency import JarDependency <EOL> from pants . backend . jvm . targets . jar_library import JarLibrary <EOL> from pants . base . generator import Generator , TemplateData <EOL> from pants . base . revision import Revision <EOL> from pants . build_graph . target import Target <EOL> from pants . ivy . bootstrapper import Bootstrapper <EOL> from pants . java . util import execute_runner <EOL> from pants . util . dirutil import safe_concurrent_creation , safe_mkdir , safe_open <EOL> from pants . util . fileutil import atomic_copy <EOL> class IvyResolutionStep ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , confs , hash_name , pinned_artifacts , soft_excludes , ivy_cache_dir , <EOL> global_ivy_workdir ) : <EOL> """<STR_LIT>""" <EOL> self . confs = confs <EOL> self . hash_name = hash_name <EOL> self . pinned_artifacts = pinned_artifacts <EOL> self . soft_excludes = soft_excludes <EOL> self . ivy_cache_dir = ivy_cache_dir <EOL> self . global_ivy_workdir = global_ivy_workdir <EOL> self . workdir_reports_by_conf = { c : self . resolve_report_path ( c ) for c in confs } <EOL> @ abstractmethod <EOL> def required_load_files_exist ( self ) : <EOL> """<STR_LIT>""" <EOL> @ abstractmethod <EOL> def required_exec_files_exist ( self ) : <EOL> """<STR_LIT>""" <EOL> @ abstractmethod <EOL> def load ( self , targets ) : <EOL> """<STR_LIT>""" <EOL> @ abstractmethod <EOL> def exec_and_load ( self , executor , extra_args , targets , jvm_options , workunit_name , <EOL> workunit_factory ) : <EOL> """<STR_LIT>""" <EOL> @ property <EOL> def workdir ( self ) : <EOL> return os . path . join ( self . global_ivy_workdir , self . hash_name ) <EOL> @ property <EOL> def symlink_classpath_filename ( self ) : <EOL> return os . path . join ( self . workdir , '<STR_LIT>' ) <EOL> @ property <EOL> def ivy_cache_classpath_filename ( self ) : <EOL> return '<STR_LIT>' . format ( self . symlink_classpath_filename ) <EOL> @ property <EOL> def frozen_resolve_file ( self ) : <EOL> return os . path . join ( self . workdir , '<STR_LIT>' ) <EOL> @ property <EOL> def symlink_dir ( self ) : <EOL> return os . path . join ( self . global_ivy_workdir , '<STR_LIT>' ) <EOL> @ abstractmethod <EOL> def ivy_xml_path ( self ) : <EOL> """<STR_LIT>""" <EOL> @ abstractmethod <EOL> def resolve_report_path ( self , conf ) : <EOL> """<STR_LIT>""" <EOL> def _construct_and_load_symlink_map ( self ) : <EOL> artifact_paths , symlink_map = IvyUtils . construct_and_load_symlink_map ( <EOL> self . symlink_dir , <EOL> self . ivy_cache_dir , <EOL> self . ivy_cache_classpath_filename , <EOL> self . symlink_classpath_filename ) <EOL> return artifact_paths , symlink_map <EOL> def _call_ivy ( self , executor , extra_args , ivyxml , jvm_options , hash_name_for_report , <EOL> workunit_factory , workunit_name ) : <EOL> IvyUtils . do_resolve ( executor , <EOL> extra_args , <EOL> ivyxml , <EOL> jvm_options , <EOL> self . workdir_reports_by_conf , <EOL> self . confs , <EOL> self . ivy_cache_dir , <EOL> self . ivy_cache_classpath_filename , <EOL> hash_name_for_report , <EOL> workunit_factory , <EOL> workunit_name ) <EOL> class IvyFetchStep ( IvyResolutionStep ) : <EOL> """<STR_LIT>""" <EOL> def required_load_files_exist ( self ) : <EOL> return ( all ( os . path . isfile ( report ) for report in self . workdir_reports_by_conf . values ( ) ) and <EOL> os . path . isfile ( self . ivy_cache_classpath_filename ) and <EOL> os . path . isfile ( self . frozen_resolve_file ) ) <EOL> def resolve_report_path ( self , conf ) : <EOL> return os . path . join ( self . workdir , '<STR_LIT>' . format ( conf ) ) <EOL> @ property <EOL> def ivy_xml_path ( self ) : <EOL> return os . path . join ( self . workdir , '<STR_LIT>' ) <EOL> def required_exec_files_exist ( self ) : <EOL> return os . path . isfile ( self . frozen_resolve_file ) <EOL> def load ( self , targets ) : <EOL> try : <EOL> frozen_resolutions = FrozenResolution . load_from_file ( self . frozen_resolve_file , <EOL> targets ) <EOL> except Exception as e : <EOL> logger . debug ( '<STR_LIT>' . format ( self . frozen_resolve_file , e ) ) <EOL> return NO_RESOLVE_RUN_RESULT <EOL> return self . _load_from_fetch ( frozen_resolutions ) <EOL> def exec_and_load ( self , executor , extra_args , targets , jvm_options , workunit_name , <EOL> workunit_factory ) : <EOL> try : <EOL> frozen_resolutions = FrozenResolution . load_from_file ( self . frozen_resolve_file , <EOL> targets ) <EOL> except Exception as e : <EOL> logger . debug ( '<STR_LIT>' . format ( self . frozen_resolve_file , e ) ) <EOL> return NO_RESOLVE_RUN_RESULT <EOL> self . _do_fetch ( executor , extra_args , frozen_resolutions , jvm_options , <EOL> workunit_name , workunit_factory ) <EOL> result = self . _load_from_fetch ( frozen_resolutions ) <EOL> if not result . all_linked_artifacts_exist ( ) : <EOL> raise IvyResolveMappingError ( <EOL> '<STR_LIT>' . format ( self . global_ivy_workdir , <EOL> result ) ) <EOL> return result <EOL> def _load_from_fetch ( self , frozen_resolutions ) : <EOL> artifact_paths , symlink_map = self . _construct_and_load_symlink_map ( ) <EOL> return IvyFetchResolveResult ( artifact_paths , <EOL> symlink_map , <EOL> self . hash_name , <EOL> self . workdir_reports_by_conf , <EOL> frozen_resolutions ) <EOL> def _do_fetch ( self , executor , extra_args , frozen_resolution , jvm_options , workunit_name , <EOL> workunit_factory ) : <EOL> hash_name_for_report = '<STR_LIT>' . format ( self . hash_name ) <EOL> ivyxml = self . ivy_xml_path <EOL> self . _prepare_ivy_xml ( frozen_resolution , ivyxml , hash_name_for_report ) <EOL> self . _call_ivy ( executor , extra_args , ivyxml , jvm_options , hash_name_for_report , <EOL> workunit_factory , workunit_name ) <EOL> def _prepare_ivy_xml ( self , frozen_resolution , ivyxml , resolve_hash_name_for_report ) : <EOL> default_resolution = frozen_resolution . get ( '<STR_LIT:default>' ) <EOL> if default_resolution is None : <EOL> raise IvyUtils . IvyError ( "<STR_LIT>" ) <EOL> try : <EOL> jars = default_resolution . jar_dependencies <EOL> IvyUtils . generate_fetch_ivy ( jars , ivyxml , self . confs , resolve_hash_name_for_report ) <EOL> except Exception as e : <EOL> raise IvyUtils . IvyError ( '<STR_LIT>' . format ( e ) ) <EOL> class IvyResolveStep ( IvyResolutionStep ) : <EOL> """<STR_LIT>""" <EOL> def required_load_files_exist ( self ) : <EOL> return ( all ( os . path . isfile ( report ) for report in self . workdir_reports_by_conf . values ( ) ) and <EOL> os . path . isfile ( self . ivy_cache_classpath_filename ) ) <EOL> def resolve_report_path ( self , conf ) : <EOL> return os . path . join ( self . workdir , '<STR_LIT>' . format ( conf ) ) <EOL> @ property <EOL> def ivy_xml_path ( self ) : <EOL> return os . path . join ( self . workdir , '<STR_LIT>' ) <EOL> def load ( self , targets ) : <EOL> artifact_paths , symlink_map = self . _construct_and_load_symlink_map ( ) <EOL> return IvyResolveResult ( artifact_paths , <EOL> symlink_map , <EOL> self . hash_name , <EOL> self . workdir_reports_by_conf ) <EOL> def exec_and_load ( self , executor , extra_args , targets , jvm_options , <EOL> workunit_name , workunit_factory ) : <EOL> self . _do_resolve ( executor , extra_args , targets , jvm_options , workunit_name , workunit_factory ) <EOL> result = self . load ( targets ) <EOL> if not result . all_linked_artifacts_exist ( ) : <EOL> raise IvyResolveMappingError ( <EOL> '<STR_LIT>' . format ( self . global_ivy_workdir , <EOL> result ) ) <EOL> frozen_resolutions_by_conf = result . get_frozen_resolutions_by_conf ( targets ) <EOL> FrozenResolution . dump_to_file ( self . frozen_resolve_file , frozen_resolutions_by_conf ) <EOL> return result <EOL> def _do_resolve ( self , executor , extra_args , targets , jvm_options , workunit_name , workunit_factory ) : <EOL> safe_mkdir ( self . workdir ) <EOL> ivyxml = self . ivy_xml_path <EOL> hash_name = '<STR_LIT>' . format ( self . hash_name ) <EOL> self . _prepare_ivy_xml ( targets , ivyxml , hash_name ) <EOL> self . _call_ivy ( executor , extra_args , ivyxml , jvm_options , hash_name , <EOL> workunit_factory , workunit_name ) <EOL> def _prepare_ivy_xml ( self , targets , ivyxml , hash_name ) : <EOL> jars , global_excludes = IvyUtils . calculate_classpath ( targets ) <EOL> if self . soft_excludes : <EOL> global_excludes = [ ] <EOL> IvyUtils . generate_ivy ( targets , jars , global_excludes , ivyxml , self . confs , <EOL> hash_name , self . pinned_artifacts ) <EOL> class FrozenResolution ( object ) : <EOL> """<STR_LIT>""" <EOL> class MissingTarget ( Exception ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self ) : <EOL> self . target_to_resolved_coordinates = defaultdict ( OrderedSet ) <EOL> self . all_resolved_coordinates = OrderedSet ( ) <EOL> self . coordinate_to_attributes = OrderedDict ( ) <EOL> @ property <EOL> def jar_dependencies ( self ) : <EOL> return [ <EOL> JarDependency ( c . org , c . name , c . rev , classifier = c . classifier , ext = c . ext , <EOL> ** self . coordinate_to_attributes . get ( c , { } ) ) <EOL> for c in self . all_resolved_coordinates ] <EOL> def add_resolved_jars ( self , target , resolved_jars ) : <EOL> coords = [ j . coordinate for j in resolved_jars ] <EOL> self . add_resolution_coords ( target , coords ) <EOL> for j in target . jar_dependencies : <EOL> if j . url : <EOL> self . coordinate_to_attributes [ j . coordinate ] = { '<STR_LIT:url>' : j . url } <EOL> else : <EOL> self . coordinate_to_attributes [ j . coordinate ] = { } <EOL> def add_resolution_coords ( self , target , coords ) : <EOL> for c in coords : <EOL> self . target_to_resolved_coordinates [ target ] . add ( c ) <EOL> self . all_resolved_coordinates . add ( c ) <EOL> def target_spec_to_coordinate_strings ( self ) : <EOL> return { t . address . spec : [ str ( c ) for c in coordinates ] <EOL> for t , coordinates in self . target_to_resolved_coordinates . items ( ) } <EOL> def __repr__ ( self ) : <EOL> return '<STR_LIT>' . format ( <EOL> '<STR_LIT>' . join ( '<STR_LIT>' . join ( [ t . address . spec , <EOL> '<STR_LIT>' . join ( str ( c ) for c in cs ) ] ) <EOL> for t , cs in self . target_to_resolved_coordinates . items ( ) ) , <EOL> '<STR_LIT>' . join ( str ( c ) for c in self . coordinate_to_attributes . keys ( ) ) <EOL> ) <EOL> def __eq__ ( self , other ) : <EOL> return ( type ( self ) == type ( other ) and <EOL> self . all_resolved_coordinates == other . all_resolved_coordinates and <EOL> self . target_to_resolved_coordinates == other . target_to_resolved_coordinates ) <EOL> def __ne__ ( self , other ) : <EOL> return not self == other <EOL> @ classmethod <EOL> def load_from_file ( cls , filename , targets ) : <EOL> if not os . path . exists ( filename ) : <EOL> return None <EOL> with open ( filename ) as f : <EOL> from_file = json . load ( f , object_pairs_hook = OrderedDict ) <EOL> result = { } <EOL> target_lookup = { t . address . spec : t for t in targets } <EOL> for conf , serialized_resolution in from_file . items ( ) : <EOL> resolution = FrozenResolution ( ) <EOL> def m2_for ( c ) : <EOL> return M2Coordinate . from_string ( c ) <EOL> for coord , attr_dict in serialized_resolution [ '<STR_LIT>' ] . items ( ) : <EOL> m2 = m2_for ( coord ) <EOL> resolution . coordinate_to_attributes [ m2 ] = attr_dict <EOL> for spec , coord_strs in serialized_resolution [ '<STR_LIT>' ] . items ( ) : <EOL> t = target_lookup . get ( spec , None ) <EOL> if t is None : <EOL> raise cls . MissingTarget ( '<STR_LIT>' <EOL> . format ( spec ) ) <EOL> resolution . add_resolution_coords ( t , [ m2_for ( c ) for c in coord_strs ] ) <EOL> result [ conf ] = resolution <EOL> return result <EOL> @ classmethod <EOL> def dump_to_file ( cls , filename , resolutions_by_conf ) : <EOL> res = { } <EOL> for conf , resolution in resolutions_by_conf . items ( ) : <EOL> res [ conf ] = OrderedDict ( [ <EOL> [ '<STR_LIT>' , resolution . target_spec_to_coordinate_strings ( ) ] , <EOL> [ '<STR_LIT>' , OrderedDict ( [ str ( c ) , attrs ] <EOL> for c , attrs in resolution . coordinate_to_attributes . items ( ) ) ] <EOL> ] ) <EOL> with safe_concurrent_creation ( filename ) as tmp_filename : <EOL> with open ( tmp_filename , '<STR_LIT:wb>' ) as f : <EOL> json . dump ( res , f ) <EOL> class IvyResolveResult ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , resolved_artifact_paths , symlink_map , resolve_hash_name , reports_by_conf ) : <EOL> self . _reports_by_conf = reports_by_conf <EOL> self . resolved_artifact_paths = resolved_artifact_paths <EOL> self . resolve_hash_name = resolve_hash_name <EOL> self . _symlink_map = symlink_map <EOL> @ property <EOL> def has_resolved_artifacts ( self ) : <EOL> """<STR_LIT>""" <EOL> return self . resolve_hash_name is not None <EOL> def all_linked_artifacts_exist ( self ) : <EOL> """<STR_LIT>""" <EOL> if not self . has_resolved_artifacts : <EOL> return False <EOL> for path in self . resolved_artifact_paths : <EOL> if not os . path . isfile ( path ) : <EOL> return False <EOL> else : <EOL> return True <EOL> def report_for_conf ( self , conf ) : <EOL> """<STR_LIT>""" <EOL> return self . _reports_by_conf . get ( conf ) <EOL> def get_frozen_resolutions_by_conf ( self , targets ) : <EOL> frozen_resolutions_by_conf = OrderedDict ( ) <EOL> for conf in self . _reports_by_conf : <EOL> frozen_resolution = FrozenResolution ( ) <EOL> for target , resolved_jars in self . resolved_jars_for_each_target ( conf , targets ) : <EOL> frozen_resolution . add_resolved_jars ( target , resolved_jars ) <EOL> frozen_resolutions_by_conf [ conf ] = frozen_resolution <EOL> return frozen_resolutions_by_conf <EOL> def resolved_jars_for_each_target ( self , conf , targets ) : <EOL> """<STR_LIT>""" <EOL> ivy_info = self . _ivy_info_for ( conf ) <EOL> if not ivy_info : <EOL> return <EOL> jar_library_targets = [ t for t in targets if isinstance ( t , JarLibrary ) ] <EOL> ivy_jar_memo = { } <EOL> for target in jar_library_targets : <EOL> resolved_jars = self . _resolved_jars_with_symlinks ( conf , ivy_info , ivy_jar_memo , <EOL> self . _jar_dependencies_for_target ( conf , <EOL> target ) , <EOL> target ) <EOL> yield target , resolved_jars <EOL> def _jar_dependencies_for_target ( self , conf , target ) : <EOL> return target . jar_dependencies <EOL> def _ivy_info_for ( self , conf ) : <EOL> report_path = self . _reports_by_conf . get ( conf ) <EOL> return IvyUtils . parse_xml_report ( conf , report_path ) <EOL> def _new_resolved_jar_with_symlink_path ( self , conf , target , resolved_jar_without_symlink ) : <EOL> def candidate_cache_paths ( ) : <EOL> yield resolved_jar_without_symlink . cache_path <EOL> yield os . path . realpath ( resolved_jar_without_symlink . cache_path ) <EOL> for cache_path in candidate_cache_paths ( ) : <EOL> pants_path = self . _symlink_map . get ( cache_path ) <EOL> if pants_path : <EOL> break <EOL> else : <EOL> raise IvyResolveMappingError ( <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> . format ( spec = target . address . spec , <EOL> resolved_jar = resolved_jar_without_symlink . cache_path , <EOL> conf = conf ) ) <EOL> return ResolvedJar ( coordinate = resolved_jar_without_symlink . coordinate , <EOL> pants_path = pants_path , <EOL> cache_path = resolved_jar_without_symlink . cache_path ) <EOL> def _resolved_jars_with_symlinks ( self , conf , ivy_info , ivy_jar_memo , coordinates , target ) : <EOL> raw_resolved_jars = ivy_info . get_resolved_jars_for_coordinates ( coordinates , <EOL> memo = ivy_jar_memo ) <EOL> resolved_jars = [ self . _new_resolved_jar_with_symlink_path ( conf , target , raw_resolved_jar ) <EOL> for raw_resolved_jar in raw_resolved_jars ] <EOL> return resolved_jars <EOL> class IvyFetchResolveResult ( IvyResolveResult ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , resolved_artifact_paths , symlink_map , resolve_hash_name , reports_by_conf , <EOL> frozen_resolutions ) : <EOL> super ( IvyFetchResolveResult , self ) . __init__ ( resolved_artifact_paths , symlink_map , <EOL> resolve_hash_name , reports_by_conf ) <EOL> self . _frozen_resolutions = frozen_resolutions <EOL> def _jar_dependencies_for_target ( self , conf , target ) : <EOL> return self . _frozen_resolutions [ conf ] . target_to_resolved_coordinates . get ( target , ( ) ) <EOL> NO_RESOLVE_RUN_RESULT = IvyResolveResult ( [ ] , { } , None , { } ) <EOL> IvyModule = namedtuple ( '<STR_LIT>' , [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] ) <EOL> Dependency = namedtuple ( '<STR_LIT>' , <EOL> [ '<STR_LIT>' , '<STR_LIT:name>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] ) <EOL> Artifact = namedtuple ( '<STR_LIT>' , [ '<STR_LIT:name>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT:url>' , '<STR_LIT>' ] ) <EOL> logger = logging . getLogger ( __name__ ) <EOL> class IvyResolveMappingError ( Exception ) : <EOL> """<STR_LIT>""" <EOL> class IvyModuleRef ( object ) : <EOL> """<STR_LIT>""" <EOL> _ANY_REV = '<STR_LIT>' <EOL> def __init__ ( self , org , name , rev , classifier = None , ext = None ) : <EOL> self . org = org <EOL> self . name = name <EOL> self . rev = rev <EOL> self . classifier = classifier <EOL> self . ext = ext or '<STR_LIT>' <EOL> self . _id = ( self . org , self . name , self . rev , self . classifier , self . ext ) <EOL> def __eq__ ( self , other ) : <EOL> return isinstance ( other , IvyModuleRef ) and self . _id == other . _id <EOL> def __ne__ ( self , other ) : <EOL> return not self == other <EOL> def __hash__ ( self ) : <EOL> return hash ( self . _id ) <EOL> def __str__ ( self ) : <EOL> return '<STR_LIT>' . format ( '<STR_LIT::>' . join ( ( x or '<STR_LIT>' ) for x in self . _id ) ) <EOL> def __repr__ ( self ) : <EOL> return ( '<STR_LIT>' <EOL> . format ( * self . _id ) ) <EOL> def __cmp__ ( self , other ) : <EOL> return cmp ( ( self . org , self . name , self . classifier , self . ext , self . rev ) , <EOL> ( other . org , other . name , other . classifier , other . ext , other . rev ) ) <EOL> @ property <EOL> def caller_key ( self ) : <EOL> """<STR_LIT>""" <EOL> return IvyModuleRef ( name = self . name , org = self . org , rev = self . _ANY_REV ) <EOL> @ property <EOL> def unversioned ( self ) : <EOL> """<STR_LIT>""" <EOL> return IvyModuleRef ( name = self . name , org = self . org , rev = self . _ANY_REV , classifier = self . classifier , <EOL> ext = self . ext ) <EOL> class IvyInfo ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , conf ) : <EOL> self . _conf = conf <EOL> self . modules_by_ref = { } <EOL> self . refs_by_unversioned_refs = { } <EOL> self . _deps_by_caller = defaultdict ( OrderedSet ) <EOL> self . _artifacts_by_ref = defaultdict ( OrderedSet ) <EOL> def add_module ( self , module ) : <EOL> if not module . artifact : <EOL> return <EOL> ref_unversioned = module . ref . unversioned <EOL> if ref_unversioned in self . refs_by_unversioned_refs : <EOL> raise IvyResolveMappingError ( '<STR_LIT>' <EOL> . format ( ref_unversioned , module . ref . rev ) ) <EOL> if module . ref in self . modules_by_ref : <EOL> raise IvyResolveMappingError ( '<STR_LIT>' <EOL> . format ( module . ref ) ) <EOL> self . refs_by_unversioned_refs [ ref_unversioned ] = module . ref <EOL> self . modules_by_ref [ module . ref ] = module <EOL> for caller in module . callers : <EOL> self . _deps_by_caller [ caller . caller_key ] . add ( module . ref ) <EOL> self . _artifacts_by_ref [ ref_unversioned ] . add ( module . artifact ) <EOL> def _do_traverse_dependency_graph ( self , ref , collector , memo , visited ) : <EOL> memoized_value = memo . get ( ref ) <EOL> if memoized_value : <EOL> return memoized_value <EOL> if ref in visited : <EOL> return set ( ) <EOL> visited . add ( ref ) <EOL> acc = collector ( ref ) <EOL> deps = sorted ( self . _deps_by_caller . get ( ref . caller_key , ( ) ) ) <EOL> for dep in deps : <EOL> acc . update ( self . _do_traverse_dependency_graph ( dep , collector , memo , visited ) ) <EOL> memo [ ref ] = acc <EOL> return acc <EOL> def traverse_dependency_graph ( self , ref , collector , memo = None ) : <EOL> """<STR_LIT>""" <EOL> resolved_ref = self . refs_by_unversioned_refs . get ( ref . unversioned ) <EOL> if resolved_ref : <EOL> ref = resolved_ref <EOL> if memo is None : <EOL> memo = dict ( ) <EOL> visited = set ( ) <EOL> return self . _do_traverse_dependency_graph ( ref , collector , memo , visited ) <EOL> def get_resolved_jars_for_coordinates ( self , coordinates , memo = None ) : <EOL> """<STR_LIT>""" <EOL> def to_resolved_jar ( jar_ref , jar_path ) : <EOL> return ResolvedJar ( coordinate = M2Coordinate ( org = jar_ref . org , <EOL> name = jar_ref . name , <EOL> rev = jar_ref . rev , <EOL> classifier = jar_ref . classifier , <EOL> ext = jar_ref . ext ) , <EOL> cache_path = jar_path ) <EOL> resolved_jars = OrderedSet ( ) <EOL> def create_collection ( dep ) : <EOL> return OrderedSet ( [ dep ] ) <EOL> for jar in coordinates : <EOL> classifier = jar . classifier if self . _conf == '<STR_LIT:default>' else self . _conf <EOL> jar_module_ref = IvyModuleRef ( jar . org , jar . name , jar . rev , classifier ) <EOL> for module_ref in self . traverse_dependency_graph ( jar_module_ref , create_collection , memo ) : <EOL> for artifact_path in self . _artifacts_by_ref [ module_ref . unversioned ] : <EOL> resolved_jars . add ( to_resolved_jar ( module_ref , artifact_path ) ) <EOL> return resolved_jars <EOL> def __repr__ ( self ) : <EOL> return '<STR_LIT>' . format ( self . _conf , self . modules_by_ref . keys ( ) ) <EOL> class IvyUtils ( object ) : <EOL> """<STR_LIT>""" <EOL> _ivy_lock = threading . RLock ( ) <EOL> _symlink_map_lock = threading . Lock ( ) <EOL> INTERNAL_ORG_NAME = '<STR_LIT>' <EOL> class IvyError ( Exception ) : <EOL> """<STR_LIT>""" <EOL> class IvyResolveReportError ( IvyError ) : <EOL> """<STR_LIT>""" <EOL> class IvyResolveConflictingDepsError ( IvyError ) : <EOL> """<STR_LIT>""" <EOL> class BadRevisionError ( IvyError ) : <EOL> """<STR_LIT>""" <EOL> @ staticmethod <EOL> def _generate_exclude_template ( exclude ) : <EOL> return TemplateData ( org = exclude . org , name = exclude . name ) <EOL> @ staticmethod <EOL> def _generate_override_template ( jar ) : <EOL> return TemplateData ( org = jar . org , module = jar . name , version = jar . rev ) <EOL> @ staticmethod <EOL> def _load_classpath_from_cachepath ( path ) : <EOL> if not os . path . exists ( path ) : <EOL> return [ ] <EOL> else : <EOL> with safe_open ( path , '<STR_LIT:r>' ) as cp : <EOL> return filter ( None , ( path . strip ( ) for path in cp . read ( ) . split ( os . pathsep ) ) ) <EOL> @ classmethod <EOL> def do_resolve ( cls , executor , extra_args , ivyxml , jvm_options , workdir_report_paths_by_conf , <EOL> confs , ivy_cache_dir , ivy_cache_classpath_filename , resolve_hash_name , <EOL> workunit_factory , workunit_name ) : <EOL> """<STR_LIT>""" <EOL> ivy = Bootstrapper . default_ivy ( bootstrap_workunit_factory = workunit_factory ) <EOL> with safe_concurrent_creation ( ivy_cache_classpath_filename ) as raw_target_classpath_file_tmp : <EOL> extra_args = extra_args or [ ] <EOL> args = [ '<STR_LIT>' , raw_target_classpath_file_tmp ] + extra_args <EOL> with cls . _ivy_lock : <EOL> cls . _exec_ivy ( ivy , confs , ivyxml , args , <EOL> jvm_options = jvm_options , <EOL> executor = executor , <EOL> workunit_name = workunit_name , <EOL> workunit_factory = workunit_factory ) <EOL> if not os . path . exists ( raw_target_classpath_file_tmp ) : <EOL> raise cls . IvyError ( '<STR_LIT>' <EOL> . format ( raw_target_classpath_file_tmp ) ) <EOL> cls . _copy_ivy_reports ( workdir_report_paths_by_conf , confs , ivy_cache_dir , resolve_hash_name ) <EOL> logger . debug ( '<STR_LIT>' <EOL> . format ( dest = ivy_cache_classpath_filename ) ) <EOL> @ classmethod <EOL> def _copy_ivy_reports ( cls , workdir_report_paths_by_conf , confs , ivy_cache_dir , resolve_hash_name ) : <EOL> for conf in confs : <EOL> ivy_cache_report_path = IvyUtils . xml_report_path ( ivy_cache_dir , resolve_hash_name , <EOL> conf ) <EOL> workdir_report_path = workdir_report_paths_by_conf [ conf ] <EOL> try : <EOL> atomic_copy ( ivy_cache_report_path , <EOL> workdir_report_path ) <EOL> except IOError as e : <EOL> raise cls . IvyError ( '<STR_LIT>' <EOL> . format ( ivy_cache_report_path , workdir_report_path , e ) ) <EOL> @ classmethod <EOL> def _exec_ivy ( cls , ivy , confs , ivyxml , args , jvm_options , executor , <EOL> workunit_name , workunit_factory ) : <EOL> ivy = ivy or Bootstrapper . default_ivy ( ) <EOL> ivy_args = [ '<STR_LIT>' , ivyxml ] <EOL> ivy_args . append ( '<STR_LIT>' ) <EOL> ivy_args . extend ( confs ) <EOL> ivy_args . extend ( args ) <EOL> ivy_jvm_options = list ( jvm_options ) <EOL> ivy_jvm_options . append ( '<STR_LIT>' ) <EOL> runner = ivy . runner ( jvm_options = ivy_jvm_options , args = ivy_args , executor = executor ) <EOL> try : <EOL> with ivy . resolution_lock : <EOL> result = execute_runner ( runner , workunit_factory = workunit_factory , <EOL> workunit_name = workunit_name ) <EOL> if result != <NUM_LIT:0> : <EOL> raise IvyUtils . IvyError ( '<STR_LIT>' . format ( result = result , <EOL> cmd = runner . cmd ) ) <EOL> except runner . executor . Error as e : <EOL> raise IvyUtils . IvyError ( e ) <EOL> @ classmethod <EOL> def construct_and_load_symlink_map ( cls , symlink_dir , ivy_cache_dir , <EOL> ivy_cache_classpath_filename , symlink_classpath_filename ) : <EOL> with IvyUtils . _symlink_map_lock : <EOL> symlink_map = cls . _symlink_cachepath ( ivy_cache_dir , <EOL> ivy_cache_classpath_filename , <EOL> symlink_dir , <EOL> symlink_classpath_filename ) <EOL> classpath = cls . _load_classpath_from_cachepath ( symlink_classpath_filename ) <EOL> return classpath , symlink_map <EOL> @ classmethod <EOL> def _symlink_cachepath ( cls , ivy_cache_dir , inpath , symlink_dir , outpath ) : <EOL> """<STR_LIT>""" <EOL> safe_mkdir ( symlink_dir ) <EOL> real_ivy_cache_dir = os . path . realpath ( ivy_cache_dir ) <EOL> symlink_map = OrderedDict ( ) <EOL> inpaths = cls . _load_classpath_from_cachepath ( inpath ) <EOL> paths = OrderedSet ( [ os . path . realpath ( path ) for path in inpaths ] ) <EOL> for path in paths : <EOL> if path . startswith ( real_ivy_cache_dir ) : <EOL> symlink_map [ path ] = os . path . join ( symlink_dir , os . path . relpath ( path , real_ivy_cache_dir ) ) <EOL> else : <EOL> symlink_map [ path ] = path <EOL> for path , symlink in six . iteritems ( symlink_map ) : <EOL> if path == symlink : <EOL> continue <EOL> safe_mkdir ( os . path . dirname ( symlink ) ) <EOL> try : <EOL> os . symlink ( path , symlink ) <EOL> except OSError as e : <EOL> if e . errno != errno . EEXIST : <EOL> raise <EOL> with safe_open ( outpath , '<STR_LIT:w>' ) as outfile : <EOL> outfile . write ( '<STR_LIT::>' . join ( OrderedSet ( symlink_map . values ( ) ) ) ) <EOL> return dict ( symlink_map ) <EOL> @ classmethod <EOL> def xml_report_path ( cls , cache_dir , resolve_hash_name , conf ) : <EOL> """<STR_LIT>""" <EOL> return os . path . join ( cache_dir , '<STR_LIT>' . format ( IvyUtils . INTERNAL_ORG_NAME , <EOL> resolve_hash_name , conf ) ) <EOL> @ classmethod <EOL> def parse_xml_report ( cls , conf , path ) : <EOL> """<STR_LIT>""" <EOL> if not os . path . exists ( path ) : <EOL> raise cls . IvyResolveReportError ( '<STR_LIT>' . format ( path ) ) <EOL> logger . debug ( "<STR_LIT>" . format ( path ) ) <EOL> ret = IvyInfo ( conf ) <EOL> etree = ET . parse ( path ) <EOL> doc = etree . getroot ( ) <EOL> for module in doc . findall ( '<STR_LIT>' ) : <EOL> org = module . get ( '<STR_LIT>' ) <EOL> name = module . get ( '<STR_LIT:name>' ) <EOL> for revision in module . findall ( '<STR_LIT>' ) : <EOL> rev = revision . get ( '<STR_LIT:name>' ) <EOL> callers = [ ] <EOL> for caller in revision . findall ( '<STR_LIT>' ) : <EOL> callers . append ( IvyModuleRef ( caller . get ( '<STR_LIT>' ) , <EOL> caller . get ( '<STR_LIT:name>' ) , <EOL> caller . get ( '<STR_LIT>' ) ) ) <EOL> for artifact in revision . findall ( '<STR_LIT>' ) : <EOL> classifier = artifact . get ( '<STR_LIT>' ) <EOL> ext = artifact . get ( '<STR_LIT>' ) <EOL> ivy_module_ref = IvyModuleRef ( org = org , name = name , rev = rev , <EOL> classifier = classifier , ext = ext ) <EOL> artifact_cache_path = artifact . get ( '<STR_LIT:location>' ) <EOL> ivy_module = IvyModule ( ivy_module_ref , artifact_cache_path , tuple ( callers ) ) <EOL> ret . add_module ( ivy_module ) <EOL> return ret <EOL> @ classmethod <EOL> def generate_ivy ( cls , targets , jars , excludes , ivyxml , confs , resolve_hash_name = None , <EOL> pinned_artifacts = None , jar_dep_manager = None ) : <EOL> if not resolve_hash_name : <EOL> resolve_hash_name = Target . maybe_readable_identify ( targets ) <EOL> return cls . _generate_resolve_ivy ( jars , excludes , ivyxml , confs , resolve_hash_name , pinned_artifacts , <EOL> jar_dep_manager ) <EOL> @ classmethod <EOL> def _generate_resolve_ivy ( cls , jars , excludes , ivyxml , confs , resolve_hash_name , pinned_artifacts = None , <EOL> jar_dep_manager = None ) : <EOL> org = IvyUtils . INTERNAL_ORG_NAME <EOL> name = resolve_hash_name <EOL> extra_configurations = [ conf for conf in confs if conf and conf != '<STR_LIT:default>' ] <EOL> jars_by_key = OrderedDict ( ) <EOL> for jar in jars : <EOL> jars = jars_by_key . setdefault ( ( jar . org , jar . name ) , [ ] ) <EOL> jars . append ( jar ) <EOL> manager = jar_dep_manager or JarDependencyManagement . global_instance ( ) <EOL> artifact_set = PinnedJarArtifactSet ( pinned_artifacts ) <EOL> for jars in jars_by_key . values ( ) : <EOL> for i , dep in enumerate ( jars ) : <EOL> direct_coord = M2Coordinate . create ( dep ) <EOL> managed_coord = artifact_set [ direct_coord ] <EOL> if direct_coord . rev != managed_coord . rev : <EOL> coord = manager . resolve_version_conflict ( managed_coord , direct_coord , force = dep . force ) <EOL> jars [ i ] = dep . copy ( rev = coord . rev ) <EOL> elif dep . force : <EOL> artifact_set . put ( direct_coord ) <EOL> dependencies = [ cls . _generate_jar_template ( jars ) for jars in jars_by_key . values ( ) ] <EOL> overrides = [ cls . _generate_override_template ( _coord ) for _coord in artifact_set ] <EOL> excludes = [ cls . _generate_exclude_template ( exclude ) for exclude in excludes ] <EOL> template_data = TemplateData ( <EOL> org = org , <EOL> module = name , <EOL> extra_configurations = extra_configurations , <EOL> dependencies = dependencies , <EOL> excludes = excludes , <EOL> overrides = overrides ) <EOL> template_relpath = os . path . join ( '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ) <EOL> cls . _write_ivy_xml_file ( ivyxml , template_data , template_relpath ) <EOL> @ classmethod <EOL> def generate_fetch_ivy ( cls , jars , ivyxml , confs , resolve_hash_name ) : <EOL> """<STR_LIT>""" <EOL> org = IvyUtils . INTERNAL_ORG_NAME <EOL> name = resolve_hash_name <EOL> extra_configurations = [ conf for conf in confs if conf and conf != '<STR_LIT:default>' ] <EOL> jars_by_key = OrderedDict ( ) <EOL> for jar in jars : <EOL> jars_by_key . setdefault ( ( jar . org , jar . name , jar . rev ) , [ ] ) . append ( jar ) <EOL> dependencies = [ cls . _generate_fetch_jar_template ( _jars ) for _jars in jars_by_key . values ( ) ] <EOL> template_data = TemplateData ( org = org , <EOL> module = name , <EOL> extra_configurations = extra_configurations , <EOL> dependencies = dependencies ) <EOL> template_relpath = os . path . join ( '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ) <EOL> cls . _write_ivy_xml_file ( ivyxml , template_data , template_relpath ) <EOL> @ classmethod <EOL> def _write_ivy_xml_file ( cls , ivyxml , template_data , template_relpath ) : <EOL> template_text = pkgutil . get_data ( __name__ , template_relpath ) <EOL> generator = Generator ( template_text , lib = template_data ) <EOL> with safe_open ( ivyxml , '<STR_LIT:w>' ) as output : <EOL> generator . write ( output ) <EOL> @ classmethod <EOL> def calculate_classpath ( cls , targets ) : <EOL> """<STR_LIT>""" <EOL> jars = OrderedDict ( ) <EOL> global_excludes = set ( ) <EOL> provide_excludes = set ( ) <EOL> targets_processed = set ( ) <EOL> def add_jar ( jar ) : <EOL> coordinate = ( jar . org , jar . name , jar . classifier ) <EOL> existing = jars . get ( coordinate ) <EOL> jars [ coordinate ] = jar if not existing else cls . _resolve_conflict ( existing = existing , <EOL> proposed = jar ) <EOL> def collect_jars ( target ) : <EOL> if isinstance ( target , JarLibrary ) : <EOL> for jar in target . jar_dependencies : <EOL> add_jar ( jar ) <EOL> def collect_excludes ( target ) : <EOL> target_excludes = target . payload . get_field_value ( '<STR_LIT>' ) <EOL> if target_excludes : <EOL> global_excludes . update ( target_excludes ) <EOL> def collect_provide_excludes ( target ) : <EOL> if not target . is_exported : <EOL> return <EOL> logger . debug ( '<STR_LIT>' . format ( <EOL> target . provides . org , target . provides . name , target ) ) <EOL> provide_excludes . add ( Exclude ( org = target . provides . org , name = target . provides . name ) ) <EOL> def collect_elements ( target ) : <EOL> targets_processed . add ( target ) <EOL> collect_jars ( target ) <EOL> collect_excludes ( target ) <EOL> collect_provide_excludes ( target ) <EOL> for target in targets : <EOL> target . walk ( collect_elements , predicate = lambda target : target not in targets_processed ) <EOL> if provide_excludes : <EOL> additional_excludes = tuple ( provide_excludes ) <EOL> new_jars = OrderedDict ( ) <EOL> for coordinate , jar in jars . items ( ) : <EOL> new_jars [ coordinate ] = jar . copy ( excludes = jar . excludes + additional_excludes ) <EOL> jars = new_jars <EOL> return jars . values ( ) , global_excludes <EOL> @ classmethod <EOL> def _resolve_conflict ( cls , existing , proposed ) : <EOL> if existing . rev is None : <EOL> return proposed <EOL> if proposed . rev is None : <EOL> return existing <EOL> if proposed == existing : <EOL> if proposed . force : <EOL> return proposed <EOL> return existing <EOL> elif existing . force and proposed . force : <EOL> raise cls . IvyResolveConflictingDepsError ( '<STR_LIT>' . format ( <EOL> proposed . org , proposed . name , proposed . classifier or '<STR_LIT>' , existing . rev , proposed . rev <EOL> ) ) <EOL> elif existing . force : <EOL> logger . debug ( '<STR_LIT>' . format ( <EOL> proposed . rev , proposed . org , proposed . name , proposed . classifier or '<STR_LIT>' , existing . rev <EOL> ) ) <EOL> return existing <EOL> elif proposed . force : <EOL> logger . debug ( '<STR_LIT>' . format ( <EOL> proposed . org , proposed . name , proposed . classifier or '<STR_LIT>' , existing . rev , proposed . rev <EOL> ) ) <EOL> return proposed <EOL> else : <EOL> if Revision . lenient ( proposed . rev ) > Revision . lenient ( existing . rev ) : <EOL> logger . debug ( '<STR_LIT>' . format ( <EOL> proposed . org , proposed . name , proposed . classifier or '<STR_LIT>' , existing . rev , proposed . rev , <EOL> ) ) <EOL> return proposed <EOL> else : <EOL> return existing <EOL> @ classmethod <EOL> def _generate_jar_template ( cls , jars ) : <EOL> global_dep_attributes = set ( Dependency ( org = jar . org , <EOL> name = jar . name , <EOL> rev = jar . rev , <EOL> mutable = jar . mutable , <EOL> force = jar . force , <EOL> transitive = jar . transitive ) <EOL> for jar in jars ) <EOL> if len ( global_dep_attributes ) != <NUM_LIT:1> : <EOL> conflicting_dependencies = sorted ( str ( g ) for g in global_dep_attributes ) <EOL> raise cls . IvyResolveConflictingDepsError ( '<STR_LIT>' <EOL> . format ( '<STR_LIT>' . join ( conflicting_dependencies ) ) ) <EOL> jar_attributes = global_dep_attributes . pop ( ) <EOL> excludes = set ( ) <EOL> for jar in jars : <EOL> excludes . update ( jar . excludes ) <EOL> any_have_url = False <EOL> artifacts = OrderedDict ( ) <EOL> for jar in jars : <EOL> ext = jar . ext <EOL> url = jar . url <EOL> if url : <EOL> any_have_url = True <EOL> classifier = jar . classifier <EOL> artifact = Artifact ( name = jar . name , <EOL> type_ = ext or '<STR_LIT>' , <EOL> ext = ext , <EOL> url = url , <EOL> classifier = classifier ) <EOL> artifacts [ ( ext , url , classifier ) ] = artifact <EOL> template = TemplateData ( <EOL> org = jar_attributes . org , <EOL> module = jar_attributes . name , <EOL> version = jar_attributes . rev , <EOL> mutable = jar_attributes . mutable , <EOL> force = jar_attributes . force , <EOL> transitive = jar_attributes . transitive , <EOL> artifacts = artifacts . values ( ) , <EOL> any_have_url = any_have_url , <EOL> excludes = [ cls . _generate_exclude_template ( exclude ) for exclude in excludes ] ) <EOL> return template <EOL> @ classmethod <EOL> def _generate_fetch_jar_template ( cls , jars ) : <EOL> global_dep_attributes = set ( Dependency ( org = jar . org , <EOL> name = jar . name , <EOL> rev = jar . rev , <EOL> transitive = False , <EOL> mutable = jar . mutable , <EOL> force = True ) <EOL> for jar in jars ) <EOL> if len ( global_dep_attributes ) != <NUM_LIT:1> : <EOL> conflicting_dependencies = sorted ( str ( g ) for g in global_dep_attributes ) <EOL> raise cls . IvyResolveConflictingDepsError ( '<STR_LIT>' <EOL> . format ( '<STR_LIT>' . join ( conflicting_dependencies ) ) ) <EOL> jar_attributes = global_dep_attributes . pop ( ) <EOL> any_have_url = False <EOL> artifacts = OrderedDict ( ) <EOL> for jar in jars : <EOL> ext = jar . ext <EOL> url = jar . url <EOL> if url : <EOL> any_have_url = True <EOL> classifier = jar . classifier <EOL> artifact = Artifact ( name = jar . name , <EOL> type_ = ext or '<STR_LIT>' , <EOL> ext = ext , <EOL> url = url , <EOL> classifier = classifier ) <EOL> artifacts [ ( ext , url , classifier ) ] = artifact <EOL> template = TemplateData ( <EOL> org = jar_attributes . org , <EOL> module = jar_attributes . name , <EOL> version = jar_attributes . rev , <EOL> mutable = jar_attributes . mutable , <EOL> artifacts = artifacts . values ( ) , <EOL> any_have_url = any_have_url , <EOL> excludes = [ ] ) <EOL> return template </s>
