<s> '''<STR_LIT>''' <EOL> from __future__ import absolute_import <EOL> import contextlib <EOL> import logging <EOL> import hashlib <EOL> import os <EOL> import shutil <EOL> import ftplib <EOL> from tornado . httputil import parse_response_start_line , HTTPInputError <EOL> from salt . exceptions import ( <EOL> CommandExecutionError , MinionError <EOL> ) <EOL> import salt . client <EOL> import salt . crypt <EOL> import salt . loader <EOL> import salt . payload <EOL> import salt . transport <EOL> import salt . fileserver <EOL> import salt . utils <EOL> import salt . utils . files <EOL> import salt . utils . templates <EOL> import salt . utils . url <EOL> import salt . utils . gzip_util <EOL> import salt . utils . http <EOL> import salt . utils . s3 <EOL> from salt . utils . locales import sdecode <EOL> from salt . utils . openstack . swift import SaltSwift <EOL> import salt . ext . six . moves . BaseHTTPServer as BaseHTTPServer <EOL> from salt . ext . six . moves . urllib . error import HTTPError , URLError <EOL> from salt . ext . six . moves . urllib . parse import urlparse , urlunparse <EOL> log = logging . getLogger ( __name__ ) <EOL> def get_file_client ( opts , pillar = False ) : <EOL> '''<STR_LIT>''' <EOL> client = opts . get ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> if pillar and client == '<STR_LIT>' : <EOL> client = '<STR_LIT>' <EOL> return { <EOL> '<STR_LIT>' : RemoteClient , <EOL> '<STR_LIT>' : FSClient , <EOL> '<STR_LIT>' : LocalClient , <EOL> } . get ( client , RemoteClient ) ( opts ) <EOL> class Client ( object ) : <EOL> '''<STR_LIT>''' <EOL> def __init__ ( self , opts ) : <EOL> self . opts = opts <EOL> self . serial = salt . payload . Serial ( self . opts ) <EOL> def __setstate__ ( self , state ) : <EOL> self . __init__ ( state [ '<STR_LIT>' ] ) <EOL> def __getstate__ ( self ) : <EOL> return { '<STR_LIT>' : self . opts } <EOL> def _check_proto ( self , path ) : <EOL> '''<STR_LIT>''' <EOL> if not path . startswith ( '<STR_LIT>' ) : <EOL> raise MinionError ( u'<STR_LIT>' . format ( path ) ) <EOL> file_path , saltenv = salt . utils . url . parse ( path ) <EOL> return file_path <EOL> def _file_local_list ( self , dest ) : <EOL> '''<STR_LIT>''' <EOL> if os . path . isdir ( dest ) : <EOL> destdir = dest <EOL> else : <EOL> destdir = os . path . dirname ( dest ) <EOL> filelist = set ( ) <EOL> for root , dirs , files in os . walk ( destdir , followlinks = True ) : <EOL> for name in files : <EOL> path = os . path . join ( root , name ) <EOL> filelist . add ( path ) <EOL> return filelist <EOL> @ contextlib . contextmanager <EOL> def _cache_loc ( self , path , saltenv = '<STR_LIT>' , cachedir = None ) : <EOL> '''<STR_LIT>''' <EOL> if cachedir is None : <EOL> cachedir = self . opts [ '<STR_LIT>' ] <EOL> elif not os . path . isabs ( cachedir ) : <EOL> cachedir = os . path . join ( self . opts [ '<STR_LIT>' ] , cachedir ) <EOL> dest = salt . utils . path_join ( cachedir , <EOL> '<STR_LIT>' , <EOL> saltenv , <EOL> path ) <EOL> destdir = os . path . dirname ( dest ) <EOL> cumask = os . umask ( <NUM_LIT> ) <EOL> if not os . path . isdir ( destdir ) : <EOL> if os . path . isfile ( destdir ) : <EOL> os . remove ( destdir ) <EOL> os . makedirs ( destdir ) <EOL> yield dest <EOL> os . umask ( cumask ) <EOL> def get_file ( self , <EOL> path , <EOL> dest = '<STR_LIT>' , <EOL> makedirs = False , <EOL> saltenv = '<STR_LIT>' , <EOL> gzip = None , <EOL> cachedir = None ) : <EOL> '''<STR_LIT>''' <EOL> raise NotImplementedError <EOL> def file_list_emptydirs ( self , saltenv = '<STR_LIT>' , prefix = '<STR_LIT>' ) : <EOL> '''<STR_LIT>''' <EOL> raise NotImplementedError <EOL> def cache_file ( self , path , saltenv = '<STR_LIT>' , cachedir = None ) : <EOL> '''<STR_LIT>''' <EOL> return self . get_url ( path , '<STR_LIT>' , True , saltenv , cachedir = cachedir ) <EOL> def cache_files ( self , paths , saltenv = '<STR_LIT>' , cachedir = None ) : <EOL> '''<STR_LIT>''' <EOL> ret = [ ] <EOL> if isinstance ( paths , str ) : <EOL> paths = paths . split ( '<STR_LIT:U+002C>' ) <EOL> for path in paths : <EOL> ret . append ( self . cache_file ( path , saltenv , cachedir = cachedir ) ) <EOL> return ret <EOL> def cache_master ( self , saltenv = '<STR_LIT>' , cachedir = None ) : <EOL> '''<STR_LIT>''' <EOL> ret = [ ] <EOL> for path in self . file_list ( saltenv ) : <EOL> ret . append ( <EOL> self . cache_file ( <EOL> salt . utils . url . create ( path ) , saltenv , cachedir = cachedir ) <EOL> ) <EOL> return ret <EOL> def cache_dir ( self , path , saltenv = '<STR_LIT>' , include_empty = False , <EOL> include_pat = None , exclude_pat = None , cachedir = None ) : <EOL> '''<STR_LIT>''' <EOL> ret = [ ] <EOL> path = self . _check_proto ( sdecode ( path ) ) <EOL> if not path . endswith ( '<STR_LIT:/>' ) : <EOL> path = path + '<STR_LIT:/>' <EOL> log . info ( <EOL> '<STR_LIT>' . format ( <EOL> path , saltenv <EOL> ) <EOL> ) <EOL> for fn_ in self . file_list ( saltenv ) : <EOL> fn_ = sdecode ( fn_ ) <EOL> if fn_ . strip ( ) and fn_ . startswith ( path ) : <EOL> if salt . utils . check_include_exclude ( <EOL> fn_ , include_pat , exclude_pat ) : <EOL> fn_ = self . cache_file ( <EOL> salt . utils . url . create ( fn_ ) , saltenv , cachedir = cachedir ) <EOL> if fn_ : <EOL> ret . append ( fn_ ) <EOL> if include_empty : <EOL> if cachedir is None : <EOL> cachedir = self . opts [ '<STR_LIT>' ] <EOL> elif not os . path . isabs ( cachedir ) : <EOL> cachedir = os . path . join ( self . opts [ '<STR_LIT>' ] , cachedir ) <EOL> dest = salt . utils . path_join ( cachedir , '<STR_LIT>' , saltenv ) <EOL> for fn_ in self . file_list_emptydirs ( saltenv ) : <EOL> fn_ = sdecode ( fn_ ) <EOL> if fn_ . startswith ( path ) : <EOL> minion_dir = '<STR_LIT>' . format ( dest , fn_ ) <EOL> if not os . path . isdir ( minion_dir ) : <EOL> os . makedirs ( minion_dir ) <EOL> ret . append ( minion_dir ) <EOL> return ret <EOL> def cache_local_file ( self , path , ** kwargs ) : <EOL> '''<STR_LIT>''' <EOL> dest = os . path . join ( self . opts [ '<STR_LIT>' ] , '<STR_LIT>' , <EOL> path . lstrip ( '<STR_LIT:/>' ) ) <EOL> destdir = os . path . dirname ( dest ) <EOL> if not os . path . isdir ( destdir ) : <EOL> os . makedirs ( destdir ) <EOL> shutil . copyfile ( path , dest ) <EOL> return dest <EOL> def file_local_list ( self , saltenv = '<STR_LIT>' ) : <EOL> '''<STR_LIT>''' <EOL> filesdest = os . path . join ( self . opts [ '<STR_LIT>' ] , '<STR_LIT>' , saltenv ) <EOL> localfilesdest = os . path . join ( self . opts [ '<STR_LIT>' ] , '<STR_LIT>' ) <EOL> fdest = self . _file_local_list ( filesdest ) <EOL> ldest = self . _file_local_list ( localfilesdest ) <EOL> return sorted ( fdest . union ( ldest ) ) <EOL> def file_list ( self , saltenv = '<STR_LIT>' , prefix = '<STR_LIT>' ) : <EOL> '''<STR_LIT>''' <EOL> return [ ] <EOL> def dir_list ( self , saltenv = '<STR_LIT>' , prefix = '<STR_LIT>' ) : <EOL> '''<STR_LIT>''' <EOL> return [ ] <EOL> def symlink_list ( self , saltenv = '<STR_LIT>' , prefix = '<STR_LIT>' ) : <EOL> '''<STR_LIT>''' <EOL> return { } <EOL> def is_cached ( self , path , saltenv = '<STR_LIT>' , cachedir = None ) : <EOL> '''<STR_LIT>''' <EOL> if path . startswith ( '<STR_LIT>' ) : <EOL> path , senv = salt . utils . url . parse ( path ) <EOL> if senv : <EOL> saltenv = senv <EOL> escaped = True if salt . utils . url . is_escaped ( path ) else False <EOL> localsfilesdest = os . path . join ( <EOL> self . opts [ '<STR_LIT>' ] , '<STR_LIT>' , path . lstrip ( '<STR_LIT>' ) ) <EOL> filesdest = os . path . join ( <EOL> self . opts [ '<STR_LIT>' ] , '<STR_LIT>' , saltenv , path . lstrip ( '<STR_LIT>' ) ) <EOL> extrndest = self . _extrn_path ( path , saltenv , cachedir = cachedir ) <EOL> if os . path . exists ( filesdest ) : <EOL> return salt . utils . url . escape ( filesdest ) if escaped else filesdest <EOL> elif os . path . exists ( localsfilesdest ) : <EOL> return salt . utils . url . escape ( localsfilesdest ) if escaped else localsfilesdest <EOL> elif os . path . exists ( extrndest ) : <EOL> return extrndest <EOL> return '<STR_LIT>' <EOL> def list_states ( self , saltenv ) : <EOL> '''<STR_LIT>''' <EOL> limit_traversal = self . opts . get ( '<STR_LIT>' , False ) <EOL> states = [ ] <EOL> if limit_traversal : <EOL> if saltenv not in self . opts [ '<STR_LIT>' ] : <EOL> log . warning ( <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' . format ( saltenv ) <EOL> ) <EOL> return states <EOL> for path in self . opts [ '<STR_LIT>' ] [ saltenv ] : <EOL> for root , dirs , files in os . walk ( path , topdown = True ) : <EOL> log . debug ( '<STR_LIT>' <EOL> '<STR_LIT>' . format ( dirs , files ) ) <EOL> if not [ filename . endswith ( '<STR_LIT>' ) for filename in files ] : <EOL> del dirs [ : ] <EOL> else : <EOL> for found_file in files : <EOL> stripped_root = os . path . relpath ( root , path ) . replace ( '<STR_LIT:/>' , '<STR_LIT:.>' ) <EOL> if salt . utils . is_windows ( ) : <EOL> stripped_root = stripped_root . replace ( '<STR_LIT:\\>' , '<STR_LIT:/>' ) <EOL> if found_file . endswith ( ( '<STR_LIT>' ) ) : <EOL> if found_file . endswith ( '<STR_LIT>' ) : <EOL> if stripped_root . endswith ( '<STR_LIT:.>' ) : <EOL> stripped_root = stripped_root . rstrip ( '<STR_LIT:.>' ) <EOL> states . append ( stripped_root ) <EOL> else : <EOL> if not stripped_root . endswith ( '<STR_LIT:.>' ) : <EOL> stripped_root += '<STR_LIT:.>' <EOL> if stripped_root . startswith ( '<STR_LIT:.>' ) : <EOL> stripped_root = stripped_root . lstrip ( '<STR_LIT:.>' ) <EOL> states . append ( stripped_root + found_file [ : - <NUM_LIT:4> ] ) <EOL> else : <EOL> for path in self . file_list ( saltenv ) : <EOL> if salt . utils . is_windows ( ) : <EOL> path = path . replace ( '<STR_LIT:\\>' , '<STR_LIT:/>' ) <EOL> if path . endswith ( '<STR_LIT>' ) : <EOL> if path . endswith ( '<STR_LIT>' . format ( '<STR_LIT:/>' ) ) : <EOL> states . append ( path . replace ( '<STR_LIT:/>' , '<STR_LIT:.>' ) [ : - <NUM_LIT:9> ] ) <EOL> else : <EOL> states . append ( path . replace ( '<STR_LIT:/>' , '<STR_LIT:.>' ) [ : - <NUM_LIT:4> ] ) <EOL> return states <EOL> def get_state ( self , sls , saltenv , cachedir = None ) : <EOL> '''<STR_LIT>''' <EOL> if '<STR_LIT:.>' in sls : <EOL> sls = sls . replace ( '<STR_LIT:.>' , '<STR_LIT:/>' ) <EOL> sls_url = salt . utils . url . create ( sls + '<STR_LIT>' ) <EOL> init_url = salt . utils . url . create ( sls + '<STR_LIT>' ) <EOL> for path in [ sls_url , init_url ] : <EOL> dest = self . cache_file ( path , saltenv , cachedir = cachedir ) <EOL> if dest : <EOL> return { '<STR_LIT:source>' : path , '<STR_LIT>' : dest } <EOL> return { } <EOL> def get_dir ( self , path , dest = '<STR_LIT>' , saltenv = '<STR_LIT>' , gzip = None , <EOL> cachedir = None ) : <EOL> '''<STR_LIT>''' <EOL> ret = [ ] <EOL> path = self . _check_proto ( path ) . rstrip ( '<STR_LIT:/>' ) <EOL> separated = path . rsplit ( '<STR_LIT:/>' , <NUM_LIT:1> ) <EOL> if len ( separated ) != <NUM_LIT:2> : <EOL> prefix = '<STR_LIT>' <EOL> else : <EOL> prefix = separated [ <NUM_LIT:0> ] <EOL> for fn_ in self . file_list ( saltenv , prefix = path ) : <EOL> try : <EOL> if fn_ [ len ( path ) ] != '<STR_LIT:/>' : <EOL> continue <EOL> except IndexError : <EOL> continue <EOL> minion_relpath = fn_ [ len ( prefix ) : ] . lstrip ( '<STR_LIT:/>' ) <EOL> ret . append ( <EOL> self . get_file ( <EOL> salt . utils . url . create ( fn_ ) , <EOL> '<STR_LIT>' . format ( dest , minion_relpath ) , <EOL> True , saltenv , gzip <EOL> ) <EOL> ) <EOL> try : <EOL> for fn_ in self . file_list_emptydirs ( saltenv , prefix = path ) : <EOL> try : <EOL> if fn_ [ len ( path ) ] != '<STR_LIT:/>' : <EOL> continue <EOL> except IndexError : <EOL> continue <EOL> minion_relpath = fn_ [ len ( prefix ) : ] . lstrip ( '<STR_LIT:/>' ) <EOL> minion_mkdir = '<STR_LIT>' . format ( dest , minion_relpath ) <EOL> if not os . path . isdir ( minion_mkdir ) : <EOL> os . makedirs ( minion_mkdir ) <EOL> ret . append ( minion_mkdir ) <EOL> except TypeError : <EOL> pass <EOL> ret . sort ( ) <EOL> return ret <EOL> def get_url ( self , url , dest , makedirs = False , saltenv = '<STR_LIT>' , <EOL> no_cache = False , cachedir = None ) : <EOL> '''<STR_LIT>''' <EOL> url_data = urlparse ( url ) <EOL> if url_data . scheme in ( '<STR_LIT:file>' , '<STR_LIT>' ) : <EOL> if not os . path . isabs ( url_data . path ) : <EOL> raise CommandExecutionError ( <EOL> '<STR_LIT>' . format ( url_data . path ) <EOL> ) <EOL> return url_data . path <EOL> if url_data . scheme == '<STR_LIT>' : <EOL> return self . get_file ( <EOL> url , dest , makedirs , saltenv , cachedir = cachedir ) <EOL> if dest : <EOL> destdir = os . path . dirname ( dest ) <EOL> if not os . path . isdir ( destdir ) : <EOL> if makedirs : <EOL> os . makedirs ( destdir ) <EOL> else : <EOL> return '<STR_LIT>' <EOL> elif not no_cache : <EOL> dest = self . _extrn_path ( url , saltenv , cachedir = cachedir ) <EOL> destdir = os . path . dirname ( dest ) <EOL> if not os . path . isdir ( destdir ) : <EOL> os . makedirs ( destdir ) <EOL> if url_data . scheme == '<STR_LIT>' : <EOL> try : <EOL> def s3_opt ( key , default = None ) : <EOL> '''<STR_LIT>''' <EOL> if '<STR_LIT>' + key in self . opts : <EOL> return self . opts [ '<STR_LIT>' + key ] <EOL> try : <EOL> return self . opts [ '<STR_LIT>' ] [ '<STR_LIT>' ] [ key ] <EOL> except ( KeyError , TypeError ) : <EOL> return default <EOL> salt . utils . s3 . query ( method = '<STR_LIT:GET>' , <EOL> bucket = url_data . netloc , <EOL> path = url_data . path [ <NUM_LIT:1> : ] , <EOL> return_bin = False , <EOL> local_file = dest , <EOL> action = None , <EOL> key = s3_opt ( '<STR_LIT:key>' ) , <EOL> keyid = s3_opt ( '<STR_LIT>' ) , <EOL> service_url = s3_opt ( '<STR_LIT>' ) , <EOL> verify_ssl = s3_opt ( '<STR_LIT>' , True ) , <EOL> location = s3_opt ( '<STR_LIT:location>' ) ) <EOL> return dest <EOL> except Exception as exc : <EOL> raise MinionError ( <EOL> '<STR_LIT>' . format ( url , exc ) <EOL> ) <EOL> if url_data . scheme == '<STR_LIT>' : <EOL> try : <EOL> ftp = ftplib . FTP ( url_data . hostname ) <EOL> ftp . login ( ) <EOL> with salt . utils . fopen ( dest , '<STR_LIT:wb>' ) as fp_ : <EOL> ftp . retrbinary ( '<STR_LIT>' . format ( url_data . path ) , fp_ . write ) <EOL> return dest <EOL> except Exception as exc : <EOL> raise MinionError ( '<STR_LIT>' . format ( url , exc ) ) <EOL> if url_data . scheme == '<STR_LIT>' : <EOL> try : <EOL> def swift_opt ( key , default ) : <EOL> '''<STR_LIT>''' <EOL> if key in self . opts : <EOL> return self . opts [ key ] <EOL> try : <EOL> return self . opts [ '<STR_LIT>' ] [ key ] <EOL> except ( KeyError , TypeError ) : <EOL> return default <EOL> swift_conn = SaltSwift ( swift_opt ( '<STR_LIT>' , None ) , <EOL> swift_opt ( '<STR_LIT>' , None ) , <EOL> swift_opt ( '<STR_LIT>' , None ) , <EOL> swift_opt ( '<STR_LIT>' , None ) ) <EOL> swift_conn . get_object ( url_data . netloc , <EOL> url_data . path [ <NUM_LIT:1> : ] , <EOL> dest ) <EOL> return dest <EOL> except Exception : <EOL> raise MinionError ( '<STR_LIT>' . format ( url ) ) <EOL> get_kwargs = { } <EOL> if url_data . username is not None and url_data . scheme in ( '<STR_LIT:http>' , '<STR_LIT>' ) : <EOL> netloc = url_data . netloc <EOL> at_sign_pos = netloc . rfind ( '<STR_LIT:@>' ) <EOL> if at_sign_pos != - <NUM_LIT:1> : <EOL> netloc = netloc [ at_sign_pos + <NUM_LIT:1> : ] <EOL> fixed_url = urlunparse ( <EOL> ( url_data . scheme , netloc , url_data . path , <EOL> url_data . params , url_data . query , url_data . fragment ) ) <EOL> get_kwargs [ '<STR_LIT>' ] = ( url_data . username , url_data . password ) <EOL> else : <EOL> fixed_url = url <EOL> destfp = None <EOL> try : <EOL> write_body = [ False ] <EOL> def on_header ( hdr ) : <EOL> try : <EOL> hdr = parse_response_start_line ( hdr ) <EOL> except HTTPInputError : <EOL> return <EOL> write_body [ <NUM_LIT:0> ] = hdr . code not in [ <NUM_LIT> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT> ] <EOL> if no_cache : <EOL> result = [ ] <EOL> def on_chunk ( chunk ) : <EOL> if write_body [ <NUM_LIT:0> ] : <EOL> result . append ( chunk ) <EOL> else : <EOL> dest_tmp = "<STR_LIT>" . format ( dest ) <EOL> destfp = salt . utils . fopen ( dest_tmp , '<STR_LIT:wb>' ) <EOL> def on_chunk ( chunk ) : <EOL> if write_body [ <NUM_LIT:0> ] : <EOL> destfp . write ( chunk ) <EOL> query = salt . utils . http . query ( <EOL> fixed_url , <EOL> stream = True , <EOL> streaming_callback = on_chunk , <EOL> header_callback = on_header , <EOL> username = url_data . username , <EOL> password = url_data . password , <EOL> ** get_kwargs <EOL> ) <EOL> if '<STR_LIT>' not in query : <EOL> raise MinionError ( '<STR_LIT>' . format ( query [ '<STR_LIT:error>' ] , url ) ) <EOL> if no_cache : <EOL> return '<STR_LIT>' . join ( result ) <EOL> else : <EOL> destfp . close ( ) <EOL> destfp = None <EOL> salt . utils . files . rename ( dest_tmp , dest ) <EOL> return dest <EOL> except HTTPError as exc : <EOL> raise MinionError ( '<STR_LIT>' . format ( <EOL> exc . code , <EOL> url , <EOL> * BaseHTTPServer . BaseHTTPRequestHandler . responses [ exc . code ] ) ) <EOL> except URLError as exc : <EOL> raise MinionError ( '<STR_LIT>' . format ( url , exc . reason ) ) <EOL> finally : <EOL> if destfp is not None : <EOL> destfp . close ( ) <EOL> def get_template ( <EOL> self , <EOL> url , <EOL> dest , <EOL> template = '<STR_LIT>' , <EOL> makedirs = False , <EOL> saltenv = '<STR_LIT>' , <EOL> cachedir = None , <EOL> ** kwargs ) : <EOL> '''<STR_LIT>''' <EOL> if '<STR_LIT>' in kwargs : <EOL> salt . utils . warn_until ( <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> ) <EOL> kwargs . pop ( '<STR_LIT>' ) <EOL> kwargs [ '<STR_LIT>' ] = saltenv <EOL> url_data = urlparse ( url ) <EOL> sfn = self . cache_file ( url , saltenv , cachedir = cachedir ) <EOL> if not os . path . exists ( sfn ) : <EOL> return '<STR_LIT>' <EOL> if template in salt . utils . templates . TEMPLATE_REGISTRY : <EOL> data = salt . utils . templates . TEMPLATE_REGISTRY [ template ] ( <EOL> sfn , <EOL> ** kwargs <EOL> ) <EOL> else : <EOL> log . error ( '<STR_LIT>' <EOL> '<STR_LIT>' . format ( template ) ) <EOL> return '<STR_LIT>' <EOL> if not data [ '<STR_LIT:result>' ] : <EOL> log . error ( <EOL> '<STR_LIT>' . format ( <EOL> data [ '<STR_LIT:data>' ] <EOL> ) <EOL> ) <EOL> return '<STR_LIT>' <EOL> if not dest : <EOL> dest = self . _extrn_path ( url , saltenv , cachedir = cachedir ) <EOL> makedirs = True <EOL> destdir = os . path . dirname ( dest ) <EOL> if not os . path . isdir ( destdir ) : <EOL> if makedirs : <EOL> os . makedirs ( destdir ) <EOL> else : <EOL> salt . utils . safe_rm ( data [ '<STR_LIT:data>' ] ) <EOL> return '<STR_LIT>' <EOL> shutil . move ( data [ '<STR_LIT:data>' ] , dest ) <EOL> return dest <EOL> def _extrn_path ( self , url , saltenv , cachedir = None ) : <EOL> '''<STR_LIT>''' <EOL> url_data = urlparse ( url ) <EOL> if salt . utils . is_windows ( ) : <EOL> netloc = salt . utils . sanitize_win_path_string ( url_data . netloc ) <EOL> else : <EOL> netloc = url_data . netloc <EOL> if cachedir is None : <EOL> cachedir = self . opts [ '<STR_LIT>' ] <EOL> elif not os . path . isabs ( cachedir ) : <EOL> cachedir = os . path . join ( self . opts [ '<STR_LIT>' ] , cachedir ) <EOL> return salt . utils . path_join ( <EOL> cachedir , <EOL> '<STR_LIT>' , <EOL> saltenv , <EOL> netloc , <EOL> url_data . path <EOL> ) <EOL> class LocalClient ( Client ) : <EOL> '''<STR_LIT>''' <EOL> def __init__ ( self , opts ) : <EOL> Client . __init__ ( self , opts ) <EOL> def _find_file ( self , path , saltenv = '<STR_LIT>' ) : <EOL> '''<STR_LIT>''' <EOL> fnd = { '<STR_LIT:path>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' } <EOL> if saltenv not in self . opts [ '<STR_LIT>' ] : <EOL> return fnd <EOL> if salt . utils . url . is_escaped ( path ) : <EOL> path = salt . utils . url . unescape ( path ) <EOL> for root in self . opts [ '<STR_LIT>' ] [ saltenv ] : <EOL> full = os . path . join ( root , path ) <EOL> if os . path . isfile ( full ) : <EOL> fnd [ '<STR_LIT:path>' ] = full <EOL> fnd [ '<STR_LIT>' ] = path <EOL> return fnd <EOL> return fnd <EOL> def get_file ( self , <EOL> path , <EOL> dest = '<STR_LIT>' , <EOL> makedirs = False , <EOL> saltenv = '<STR_LIT>' , <EOL> gzip = None , <EOL> cachedir = None ) : <EOL> '''<STR_LIT>''' <EOL> path = self . _check_proto ( path ) <EOL> fnd = self . _find_file ( path , saltenv ) <EOL> if not fnd [ '<STR_LIT:path>' ] : <EOL> return '<STR_LIT>' <EOL> return fnd [ '<STR_LIT:path>' ] <EOL> def file_list ( self , saltenv = '<STR_LIT>' , prefix = '<STR_LIT>' ) : <EOL> '''<STR_LIT>''' <EOL> ret = [ ] <EOL> if saltenv not in self . opts [ '<STR_LIT>' ] : <EOL> return ret <EOL> prefix = prefix . strip ( '<STR_LIT:/>' ) <EOL> for path in self . opts [ '<STR_LIT>' ] [ saltenv ] : <EOL> for root , dirs , files in os . walk ( <EOL> os . path . join ( path , prefix ) , followlinks = True <EOL> ) : <EOL> for fname in files : <EOL> relpath = os . path . relpath ( os . path . join ( root , fname ) , path ) <EOL> ret . append ( sdecode ( relpath ) ) <EOL> return ret <EOL> def file_list_emptydirs ( self , saltenv = '<STR_LIT>' , prefix = '<STR_LIT>' ) : <EOL> '''<STR_LIT>''' <EOL> ret = [ ] <EOL> prefix = prefix . strip ( '<STR_LIT:/>' ) <EOL> if saltenv not in self . opts [ '<STR_LIT>' ] : <EOL> return ret <EOL> for path in self . opts [ '<STR_LIT>' ] [ saltenv ] : <EOL> for root , dirs , files in os . walk ( <EOL> os . path . join ( path , prefix ) , followlinks = True <EOL> ) : <EOL> if len ( dirs ) == <NUM_LIT:0> and len ( files ) == <NUM_LIT:0> : <EOL> ret . append ( sdecode ( os . path . relpath ( root , path ) ) ) <EOL> return ret <EOL> def dir_list ( self , saltenv = '<STR_LIT>' , prefix = '<STR_LIT>' ) : <EOL> '''<STR_LIT>''' <EOL> ret = [ ] <EOL> if saltenv not in self . opts [ '<STR_LIT>' ] : <EOL> return ret <EOL> prefix = prefix . strip ( '<STR_LIT:/>' ) <EOL> for path in self . opts [ '<STR_LIT>' ] [ saltenv ] : <EOL> for root , dirs , files in os . walk ( <EOL> os . path . join ( path , prefix ) , followlinks = True <EOL> ) : <EOL> ret . append ( sdecode ( os . path . relpath ( root , path ) ) ) <EOL> return ret <EOL> def hash_file ( self , path , saltenv = '<STR_LIT>' ) : <EOL> '''<STR_LIT>''' <EOL> ret = { } <EOL> try : <EOL> path = self . _check_proto ( path ) <EOL> except MinionError : <EOL> if not os . path . isfile ( path ) : <EOL> err = '<STR_LIT>' <EOL> log . warning ( err . format ( path ) ) <EOL> return ret <EOL> else : <EOL> opts_hash_type = self . opts . get ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> hash_type = getattr ( hashlib , opts_hash_type ) <EOL> ret [ '<STR_LIT>' ] = salt . utils . get_hash ( <EOL> path , form = hash_type ) <EOL> ret [ '<STR_LIT>' ] = opts_hash_type <EOL> return ret <EOL> path = self . _find_file ( path , saltenv ) [ '<STR_LIT:path>' ] <EOL> if not path : <EOL> return { } <EOL> ret = { } <EOL> ret [ '<STR_LIT>' ] = salt . utils . get_hash ( path , self . opts [ '<STR_LIT>' ] ) <EOL> ret [ '<STR_LIT>' ] = self . opts [ '<STR_LIT>' ] <EOL> return ret <EOL> def list_env ( self , saltenv = '<STR_LIT>' ) : <EOL> '''<STR_LIT>''' <EOL> return self . file_list ( saltenv ) <EOL> def master_opts ( self ) : <EOL> '''<STR_LIT>''' <EOL> return self . opts <EOL> def envs ( self ) : <EOL> '''<STR_LIT>''' <EOL> ret = [ ] <EOL> for saltenv in self . opts [ '<STR_LIT>' ] : <EOL> ret . append ( saltenv ) <EOL> return ret <EOL> def ext_nodes ( self ) : <EOL> '''<STR_LIT>''' <EOL> return { } <EOL> class RemoteClient ( Client ) : <EOL> '''<STR_LIT>''' <EOL> def __init__ ( self , opts ) : <EOL> Client . __init__ ( self , opts ) <EOL> self . channel = salt . transport . Channel . factory ( self . opts ) <EOL> if hasattr ( self . channel , '<STR_LIT>' ) : <EOL> self . auth = self . channel . auth <EOL> else : <EOL> self . auth = '<STR_LIT>' <EOL> def _refresh_channel ( self ) : <EOL> '''<STR_LIT>''' <EOL> self . channel = salt . transport . Channel . factory ( self . opts ) <EOL> return self . channel <EOL> def get_file ( self , <EOL> path , <EOL> dest = '<STR_LIT>' , <EOL> makedirs = False , <EOL> saltenv = '<STR_LIT>' , <EOL> gzip = None , <EOL> cachedir = None ) : <EOL> '''<STR_LIT>''' <EOL> path , senv = salt . utils . url . split_env ( path ) <EOL> if senv : <EOL> saltenv = senv <EOL> hash_server = self . hash_file ( path , saltenv ) <EOL> if hash_server == '<STR_LIT>' : <EOL> log . debug ( <EOL> '<STR_LIT>' . format ( <EOL> saltenv , path <EOL> ) <EOL> ) <EOL> return False <EOL> dest2check = dest <EOL> if not dest2check : <EOL> rel_path = self . _check_proto ( path ) <EOL> log . debug ( <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' . format ( saltenv , rel_path , path ) <EOL> ) <EOL> with self . _cache_loc ( <EOL> rel_path , saltenv , cachedir = cachedir ) as cache_dest : <EOL> dest2check = cache_dest <EOL> log . debug ( <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' . format ( saltenv , dest2check , path ) <EOL> ) <EOL> if dest2check and os . path . isfile ( dest2check ) : <EOL> hash_local = self . hash_file ( dest2check , saltenv ) <EOL> if hash_local == hash_server : <EOL> log . info ( <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' . format ( <EOL> saltenv , path <EOL> ) <EOL> ) <EOL> return dest2check <EOL> log . debug ( <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' . format ( saltenv , path ) <EOL> ) <EOL> d_tries = <NUM_LIT:0> <EOL> transport_tries = <NUM_LIT:0> <EOL> path = self . _check_proto ( path ) <EOL> load = { '<STR_LIT:path>' : path , <EOL> '<STR_LIT>' : saltenv , <EOL> '<STR_LIT>' : '<STR_LIT>' } <EOL> if gzip : <EOL> gzip = int ( gzip ) <EOL> load [ '<STR_LIT>' ] = gzip <EOL> fn_ = None <EOL> if dest : <EOL> destdir = os . path . dirname ( dest ) <EOL> if not os . path . isdir ( destdir ) : <EOL> if makedirs : <EOL> os . makedirs ( destdir ) <EOL> else : <EOL> return False <EOL> fn_ = salt . utils . fopen ( dest , '<STR_LIT>' ) <EOL> else : <EOL> log . debug ( '<STR_LIT>' . format ( dest ) ) <EOL> while True : <EOL> if not fn_ : <EOL> load [ '<STR_LIT>' ] = <NUM_LIT:0> <EOL> else : <EOL> load [ '<STR_LIT>' ] = fn_ . tell ( ) <EOL> data = self . channel . send ( load ) <EOL> try : <EOL> if not data [ '<STR_LIT:data>' ] : <EOL> if not fn_ and data [ '<STR_LIT>' ] : <EOL> with self . _cache_loc ( <EOL> data [ '<STR_LIT>' ] , <EOL> saltenv , <EOL> cachedir = cachedir ) as cache_dest : <EOL> dest = cache_dest <EOL> with salt . utils . fopen ( cache_dest , '<STR_LIT>' ) as ofile : <EOL> ofile . write ( data [ '<STR_LIT:data>' ] ) <EOL> if '<STR_LIT>' in data and d_tries < <NUM_LIT:3> : <EOL> d_tries += <NUM_LIT:1> <EOL> hsum = salt . utils . get_hash ( dest , data . get ( '<STR_LIT>' , '<STR_LIT>' ) ) <EOL> if hsum != data [ '<STR_LIT>' ] : <EOL> log . warning ( '<STR_LIT>' <EOL> '<STR_LIT>' . format ( path , d_tries ) ) <EOL> continue <EOL> break <EOL> if not fn_ : <EOL> with self . _cache_loc ( <EOL> data [ '<STR_LIT>' ] , <EOL> saltenv , <EOL> cachedir = cachedir ) as cache_dest : <EOL> dest = cache_dest <EOL> if os . path . isdir ( dest ) : <EOL> salt . utils . rm_rf ( dest ) <EOL> fn_ = salt . utils . fopen ( dest , '<STR_LIT>' ) <EOL> if data . get ( '<STR_LIT>' , None ) : <EOL> data = salt . utils . gzip_util . uncompress ( data [ '<STR_LIT:data>' ] ) <EOL> else : <EOL> data = data [ '<STR_LIT:data>' ] <EOL> fn_ . write ( data ) <EOL> except ( TypeError , KeyError ) as e : <EOL> transport_tries += <NUM_LIT:1> <EOL> log . warning ( '<STR_LIT>' <EOL> '<STR_LIT>' . format ( <EOL> data , type ( data ) , e , transport_tries ) <EOL> ) <EOL> self . _refresh_channel ( ) <EOL> if transport_tries > <NUM_LIT:3> : <EOL> log . error ( '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' . format ( <EOL> data , type ( data ) , e ) <EOL> ) <EOL> break <EOL> if fn_ : <EOL> fn_ . close ( ) <EOL> log . info ( <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' . format ( saltenv , path ) <EOL> ) <EOL> else : <EOL> log . debug ( <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' . format ( saltenv , path ) <EOL> ) <EOL> return dest <EOL> def file_list ( self , saltenv = '<STR_LIT>' , prefix = '<STR_LIT>' ) : <EOL> '''<STR_LIT>''' <EOL> load = { '<STR_LIT>' : saltenv , <EOL> '<STR_LIT>' : prefix , <EOL> '<STR_LIT>' : '<STR_LIT>' } <EOL> return [ sdecode ( fn_ ) for fn_ in self . channel . send ( load ) ] <EOL> def file_list_emptydirs ( self , saltenv = '<STR_LIT>' , prefix = '<STR_LIT>' ) : <EOL> '''<STR_LIT>''' <EOL> load = { '<STR_LIT>' : saltenv , <EOL> '<STR_LIT>' : prefix , <EOL> '<STR_LIT>' : '<STR_LIT>' } <EOL> self . channel . send ( load ) <EOL> def dir_list ( self , saltenv = '<STR_LIT>' , prefix = '<STR_LIT>' ) : <EOL> '''<STR_LIT>''' <EOL> load = { '<STR_LIT>' : saltenv , <EOL> '<STR_LIT>' : prefix , <EOL> '<STR_LIT>' : '<STR_LIT>' } <EOL> return self . channel . send ( load ) <EOL> def symlink_list ( self , saltenv = '<STR_LIT>' , prefix = '<STR_LIT>' ) : <EOL> '''<STR_LIT>''' <EOL> load = { '<STR_LIT>' : saltenv , <EOL> '<STR_LIT>' : prefix , <EOL> '<STR_LIT>' : '<STR_LIT>' } <EOL> return self . channel . send ( load ) <EOL> def hash_file ( self , path , saltenv = '<STR_LIT>' ) : <EOL> '''<STR_LIT>''' <EOL> try : <EOL> path = self . _check_proto ( path ) <EOL> except MinionError : <EOL> if not os . path . isfile ( path ) : <EOL> err = '<STR_LIT>' <EOL> log . warning ( err . format ( path ) ) <EOL> return { } <EOL> else : <EOL> ret = { } <EOL> hash_type = self . opts . get ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> ret [ '<STR_LIT>' ] = salt . utils . get_hash ( <EOL> path , form = hash_type ) <EOL> ret [ '<STR_LIT>' ] = hash_type <EOL> return ret <EOL> load = { '<STR_LIT:path>' : path , <EOL> '<STR_LIT>' : saltenv , <EOL> '<STR_LIT>' : '<STR_LIT>' } <EOL> return self . channel . send ( load ) <EOL> def list_env ( self , saltenv = '<STR_LIT>' ) : <EOL> '''<STR_LIT>''' <EOL> load = { '<STR_LIT>' : saltenv , <EOL> '<STR_LIT>' : '<STR_LIT>' } <EOL> return self . channel . send ( load ) <EOL> def envs ( self ) : <EOL> '''<STR_LIT>''' <EOL> load = { '<STR_LIT>' : '<STR_LIT>' } <EOL> return self . channel . send ( load ) <EOL> def master_opts ( self ) : <EOL> '''<STR_LIT>''' <EOL> load = { '<STR_LIT>' : '<STR_LIT>' } <EOL> return self . channel . send ( load ) <EOL> def ext_nodes ( self ) : <EOL> '''<STR_LIT>''' <EOL> load = { '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT:id>' : self . opts [ '<STR_LIT:id>' ] , <EOL> '<STR_LIT>' : self . opts } <EOL> if self . auth : <EOL> load [ '<STR_LIT>' ] = self . auth . gen_token ( '<STR_LIT>' ) <EOL> return self . channel . send ( load ) <EOL> class FSClient ( RemoteClient ) : <EOL> '''<STR_LIT>''' <EOL> def __init__ ( self , opts ) : <EOL> self . opts = opts <EOL> self . channel = salt . fileserver . FSChan ( opts ) <EOL> self . auth = DumbAuth ( ) <EOL> class DumbAuth ( object ) : <EOL> '''<STR_LIT>''' <EOL> def gen_token ( self , clear_tok ) : <EOL> return clear_tok </s>
