<s> """<STR_LIT>""" <EOL> from __future__ import absolute_import <EOL> import glob <EOL> import logging <EOL> import os <EOL> import re <EOL> import shutil <EOL> import tempfile <EOL> import yaml <EOL> import zipfile <EOL> from pyleus import __version__ <EOL> from pyleus . cli . topology_spec import TopologySpec <EOL> from pyleus . cli . virtualenv_proxy import VirtualenvProxy <EOL> from pyleus . compat import StringIO <EOL> from pyleus . storm . component import DESCRIBE_OPT <EOL> from pyleus . exception import InvalidTopologyError <EOL> from pyleus . exception import JarError <EOL> from pyleus . utils import expand_path <EOL> RESOURCES_PATH = "<STR_LIT>" <EOL> YAML_FILENAME = "<STR_LIT>" <EOL> DEFAULT_REQUIREMENTS_FILENAME = "<STR_LIT>" <EOL> VIRTUALENV_NAME = "<STR_LIT>" <EOL> log = logging . getLogger ( __name__ ) <EOL> def _open_jar ( base_jar ) : <EOL> """<STR_LIT>""" <EOL> if not os . path . exists ( base_jar ) : <EOL> raise JarError ( "<STR_LIT>" ) <EOL> if not zipfile . is_zipfile ( base_jar ) : <EOL> raise JarError ( "<STR_LIT>" ) <EOL> zip_file = zipfile . ZipFile ( base_jar , "<STR_LIT:r>" ) <EOL> return zip_file <EOL> def _zip_dir ( src , arc ) : <EOL> """<STR_LIT>""" <EOL> src_re = re . compile ( src + "<STR_LIT>" ) <EOL> for root , dirs , files in os . walk ( src ) : <EOL> prefix = re . sub ( src_re , "<STR_LIT>" , root ) <EOL> for f in files : <EOL> arc . write ( os . path . join ( root , f ) , os . path . join ( prefix , f ) , <EOL> zipfile . ZIP_DEFLATED ) <EOL> def _pack_jar ( tmp_dir , output_jar ) : <EOL> """<STR_LIT>""" <EOL> zf = zipfile . ZipFile ( output_jar , "<STR_LIT:w>" ) <EOL> try : <EOL> _zip_dir ( tmp_dir , zf ) <EOL> finally : <EOL> zf . close ( ) <EOL> def _validate_venv ( topology_dir , venv ) : <EOL> """<STR_LIT>""" <EOL> if os . path . exists ( venv ) : <EOL> raise InvalidTopologyError ( "<STR_LIT>" <EOL> "<STR_LIT>" . format ( venv ) ) <EOL> def _path_contained_by ( containing_path , path ) : <EOL> """<STR_LIT>""" <EOL> real_containing_path = os . path . join ( os . path . realpath ( containing_path ) , '<STR_LIT>' ) <EOL> real_path = os . path . realpath ( path ) <EOL> common_prefix = os . path . commonprefix ( [ real_containing_path , real_path ] ) <EOL> return common_prefix == real_containing_path <EOL> def _remove_pyleus_base_jar ( venv ) : <EOL> """<STR_LIT>""" <EOL> base_jar_path = venv . execute_module ( "<STR_LIT>" , <EOL> cwd = venv . path ) . strip ( ) <EOL> if _path_contained_by ( venv . path , base_jar_path ) : <EOL> os . remove ( base_jar_path ) <EOL> def _set_up_virtualenv ( venv_name , tmp_dir , req , <EOL> include_packages , system_site_packages , <EOL> pypi_index_url , python_interpreter , verbose ) : <EOL> """<STR_LIT>""" <EOL> venv = VirtualenvProxy ( <EOL> os . path . join ( tmp_dir , venv_name ) , <EOL> system_site_packages = system_site_packages , <EOL> pypi_index_url = pypi_index_url , <EOL> python_interpreter = python_interpreter , <EOL> verbose = verbose <EOL> ) <EOL> packages = [ "<STR_LIT>" . format ( __version__ ) ] <EOL> if include_packages is not None : <EOL> packages += include_packages <EOL> for package in packages : <EOL> venv . install_package ( package ) <EOL> if req is not None : <EOL> venv . install_from_requirements ( req ) <EOL> _remove_pyleus_base_jar ( venv ) <EOL> return venv <EOL> def _assemble_full_topology_yaml ( spec , venv , resources_dir ) : <EOL> """<STR_LIT>""" <EOL> for component in spec . topology : <EOL> if component . type == "<STR_LIT>" : <EOL> log . debug ( '<STR_LIT>' . format ( component . module ) ) <EOL> description = venv . execute_module ( module = component . module , <EOL> args = [ DESCRIBE_OPT ] , <EOL> cwd = resources_dir ) <EOL> module_spec = yaml . load ( description ) <EOL> component . update_from_module ( module_spec ) <EOL> spec . verify_groupings ( ) <EOL> new_yaml = StringIO ( ) <EOL> yaml . dump ( spec . asdict ( ) , new_yaml ) <EOL> return new_yaml . getvalue ( ) <EOL> def _content_to_copy ( src , exclude ) : <EOL> """<STR_LIT>""" <EOL> content = set ( glob . glob ( os . path . join ( src , "<STR_LIT:*>" ) ) ) <EOL> content -= set ( exclude ) <EOL> return content <EOL> def _copy_dir_content ( src , dst , exclude ) : <EOL> """<STR_LIT>""" <EOL> content = _content_to_copy ( src , exclude ) <EOL> for t in content : <EOL> if os . path . isdir ( t ) : <EOL> shutil . copytree ( t , os . path . join ( dst , os . path . basename ( t ) ) , <EOL> symlinks = True ) <EOL> else : <EOL> shutil . copy2 ( t , dst ) <EOL> def _create_pyleus_jar ( original_topology_spec , topology_dir , base_jar , <EOL> output_jar , zip_file , tmp_dir , include_packages , <EOL> system_site_packages , pypi_index_url , verbose ) : <EOL> """<STR_LIT>""" <EOL> requirements_filename = original_topology_spec . requirements_filename <EOL> if not requirements_filename : <EOL> requirements_filename = DEFAULT_REQUIREMENTS_FILENAME <EOL> python_interpreter = original_topology_spec . python_interpreter <EOL> venv = os . path . join ( topology_dir , VIRTUALENV_NAME ) <EOL> req = os . path . join ( topology_dir , requirements_filename ) <EOL> if not os . path . isfile ( req ) : <EOL> req = None <EOL> _validate_venv ( topology_dir , venv ) <EOL> zip_file . extractall ( tmp_dir ) <EOL> resources_dir = os . path . join ( tmp_dir , RESOURCES_PATH ) <EOL> os . mkdir ( resources_dir ) <EOL> _copy_dir_content ( <EOL> src = topology_dir , <EOL> dst = resources_dir , <EOL> exclude = [ venv , req , output_jar ] , <EOL> ) <EOL> venv = _set_up_virtualenv ( <EOL> venv_name = VIRTUALENV_NAME , <EOL> tmp_dir = resources_dir , <EOL> req = req , <EOL> include_packages = include_packages , <EOL> system_site_packages = system_site_packages , <EOL> pypi_index_url = pypi_index_url , <EOL> python_interpreter = python_interpreter , <EOL> verbose = verbose ) <EOL> new_yaml = _assemble_full_topology_yaml ( <EOL> spec = original_topology_spec , <EOL> venv = venv , <EOL> resources_dir = resources_dir ) <EOL> jar_yaml = os . path . join ( resources_dir , YAML_FILENAME ) <EOL> with open ( jar_yaml , '<STR_LIT:w>' ) as f : <EOL> f . write ( new_yaml ) <EOL> _pack_jar ( tmp_dir , output_jar ) <EOL> def _build_output_path ( output_arg , topology_name ) : <EOL> """<STR_LIT>""" <EOL> if output_arg is not None : <EOL> return expand_path ( output_arg ) <EOL> else : <EOL> return expand_path ( topology_name + "<STR_LIT>" ) <EOL> def parse_original_topology ( topology_path ) : <EOL> with open ( topology_path ) as f : <EOL> yaml_spec = yaml . load ( f ) <EOL> return TopologySpec ( yaml_spec ) <EOL> def build_topology_jar ( configs ) : <EOL> """<STR_LIT>""" <EOL> topology_path = expand_path ( configs . topology_path ) <EOL> topology_dir = expand_path ( os . path . dirname ( topology_path ) ) <EOL> base_jar = expand_path ( configs . base_jar ) <EOL> original_topology_spec = parse_original_topology ( topology_path ) <EOL> output_jar = _build_output_path ( configs . output_jar , <EOL> original_topology_spec . name ) <EOL> include_packages = None <EOL> if configs . include_packages is not None : <EOL> include_packages = configs . include_packages . split ( "<STR_LIT:U+0020>" ) <EOL> zip_file = _open_jar ( base_jar ) <EOL> try : <EOL> tmp_dir = tempfile . mkdtemp ( ) <EOL> try : <EOL> _create_pyleus_jar ( <EOL> original_topology_spec = original_topology_spec , <EOL> topology_dir = topology_dir , <EOL> base_jar = base_jar , <EOL> output_jar = output_jar , <EOL> zip_file = zip_file , <EOL> tmp_dir = tmp_dir , <EOL> include_packages = include_packages , <EOL> system_site_packages = configs . system_site_packages , <EOL> pypi_index_url = configs . pypi_index_url , <EOL> verbose = configs . verbose , <EOL> ) <EOL> finally : <EOL> shutil . rmtree ( tmp_dir ) <EOL> finally : <EOL> zip_file . close ( ) </s>
