def sync(opts,
         form,
         saltenv=None,
         extmod_whitelist=None,
         extmod_blacklist=None):
    '''
    Sync custom modules into the extension_modules directory
    '''
    if saltenv is None:
        saltenv = ['base']

    if extmod_whitelist is None:
        extmod_whitelist = opts['extmod_whitelist']
    elif isinstance(extmod_whitelist, six.string_types):
        extmod_whitelist = {form: extmod_whitelist.split(',')}
    elif not isinstance(extmod_whitelist, dict):
        log.error('extmod_whitelist must be a string or dictionary: %s',
                  extmod_whitelist)

    if extmod_blacklist is None:
        extmod_blacklist = opts['extmod_blacklist']
    elif isinstance(extmod_blacklist, six.string_types):
        extmod_blacklist = {form: extmod_blacklist.split(',')}
    elif not isinstance(extmod_blacklist, dict):
        log.error('extmod_blacklist must be a string or dictionary: %s',
                  extmod_blacklist)

    if isinstance(saltenv, six.string_types):
        saltenv = saltenv.split(',')
    ret = []
    remote = set()
    source = salt.utils.url.create('_' + form)
    mod_dir = os.path.join(opts['extension_modules'], '{0}'.format(form))
    touched = False
    with salt.utils.files.set_umask(0o077):
        try:
            if not os.path.isdir(mod_dir):
                log.info('Creating module dir \'%s\'', mod_dir)
                try:
                    os.makedirs(mod_dir)
                except (IOError, OSError):
                    log.error(
                        'Cannot create cache module directory %s. Check '
                        'permissions.', mod_dir
                    )
            fileclient = salt.fileclient.get_file_client(opts)
            for sub_env in saltenv:
                log.info(
                    'Syncing %s for environment \'%s\'', form, sub_env
                )
                cache = []
                log.info('Loading cache from %s, for %s', source, sub_env)
                # Grab only the desired files (.py, .pyx, .so)
                cache.extend(
                    fileclient.cache_dir(
                        source, sub_env, include_empty=False,
                        include_pat=r'E@\.(pyx?|so|zip)$', exclude_pat=None
                    )
                )
                local_cache_dir = os.path.join(
                        opts['cachedir'],
                        'files',
                        sub_env,
                        '_{0}'.format(form)
                        )
                log.debug('Local cache dir: \'%s\'', local_cache_dir)
                for fn_ in cache:
                    relpath = os.path.relpath(fn_, local_cache_dir)
                    relname = os.path.splitext(relpath)[0].replace(os.sep, '.')
                    if extmod_whitelist and form in extmod_whitelist and relname not in extmod_whitelist[form]:
                        continue
                    if extmod_blacklist and form in extmod_blacklist and relname in extmod_blacklist[form]:
                        continue
                    remote.add(relpath)
                    dest = os.path.join(mod_dir, relpath)
                    log.info('Copying \'%s\' to \'%s\'', fn_, dest)
                    if os.path.isfile(dest):
                        # The file is present, if the sum differs replace it
                        hash_type = opts.get('hash_type', 'md5')
                        src_digest = salt.utils.hashutils.get_hash(fn_, hash_type)
                        dst_digest = salt.utils.hashutils.get_hash(dest, hash_type)
                        if src_digest != dst_digest:
                            # The downloaded file differs, replace!
                            shutil.copyfile(fn_, dest)
                            ret.append('{0}.{1}'.format(form, relname))
                    else:
                        dest_dir = os.path.dirname(dest)
                        if not os.path.isdir(dest_dir):
                            os.makedirs(dest_dir)
                        shutil.copyfile(fn_, dest)
                        ret.append('{0}.{1}'.format(form, relname))

                    # If the synchronized module is an utils
                    # directory, we add it to sys.path
                    for util_dir in opts['utils_dirs']:
                        if mod_dir.endswith(util_dir) and mod_dir not in sys.path:
                            sys.path.append(mod_dir)

            touched = bool(ret)
            if opts['clean_dynamic_modules'] is True:
                current = set(_listdir_recursively(mod_dir))
                for fn_ in current - remote:
                    full = os.path.join(mod_dir, fn_)
                    if os.path.isfile(full):
                        touched = True
                        os.remove(full)
                # Cleanup empty dirs
                while True:
                    emptydirs = _list_emptydirs(mod_dir)
                    if not emptydirs:
                        break
                    for emptydir in emptydirs:
                        touched = True
                        shutil.rmtree(emptydir, ignore_errors=True)
        except Exception as exc:
            log.error('Failed to sync %s module: %s', form, exc)
    return ret, touched
def sync ( opts , form , saltenv = None , extmod_whitelist = None , extmod_blacklist = None ) : if saltenv is None : saltenv = [ 'base' ] if extmod_whitelist is None : extmod_whitelist = opts [ 'extmod_whitelist' ] elif isinstance ( extmod_whitelist , six . string_types ) : extmod_whitelist = { form : extmod_whitelist . split ( ',' ) } elif not isinstance ( extmod_whitelist , dict ) : log . error ( 'extmod_whitelist must be a string or dictionary: %s' , extmod_whitelist ) if extmod_blacklist is None : extmod_blacklist = opts [ 'extmod_blacklist' ] elif isinstance ( extmod_blacklist , six . string_types ) : extmod_blacklist = { form : extmod_blacklist . split ( ',' ) } elif not isinstance ( extmod_blacklist , dict ) : log . error ( 'extmod_blacklist must be a string or dictionary: %s' , extmod_blacklist ) if isinstance ( saltenv , six . string_types ) : saltenv = saltenv . split ( ',' ) ret = [ ] remote = set ( ) source = salt . utils . url . create ( '_' + form ) mod_dir = os . path . join ( opts [ 'extension_modules' ] , '{0}' . format ( form ) ) touched = False with salt . utils . files . set_umask ( 0o077 ) : try : if not os . path . isdir ( mod_dir ) : log . info ( 'Creating module dir \'%s\'' , mod_dir ) try : os . makedirs ( mod_dir ) except ( IOError , OSError ) : log . error ( 'Cannot create cache module directory %s. Check ' 'permissions.' , mod_dir ) fileclient = salt . fileclient . get_file_client ( opts ) for sub_env in saltenv : log . info ( 'Syncing %s for environment \'%s\'' , form , sub_env ) cache = [ ] log . info ( 'Loading cache from %s, for %s' , source , sub_env ) cache . extend ( fileclient . cache_dir ( source , sub_env , include_empty = False , include_pat = r'E@\.(pyx?|so|zip)$' , exclude_pat = None ) ) local_cache_dir = os . path . join ( opts [ 'cachedir' ] , 'files' , sub_env , '_{0}' . format ( form ) ) log . debug ( 'Local cache dir: \'%s\'' , local_cache_dir ) for fn_ in cache : relpath = os . path . relpath ( fn_ , local_cache_dir ) relname = os . path . splitext ( relpath ) [ 0 ] . replace ( os . sep , '.' ) if extmod_whitelist and form in extmod_whitelist and relname not in extmod_whitelist [ form ] : continue if extmod_blacklist and form in extmod_blacklist and relname in extmod_blacklist [ form ] : continue remote . add ( relpath ) dest = os . path . join ( mod_dir , relpath ) log . info ( 'Copying \'%s\' to \'%s\'' , fn_ , dest ) if os . path . isfile ( dest ) : hash_type = opts . get ( 'hash_type' , 'md5' ) src_digest = salt . utils . hashutils . get_hash ( fn_ , hash_type ) dst_digest = salt . utils . hashutils . get_hash ( dest , hash_type ) if src_digest != dst_digest : shutil . copyfile ( fn_ , dest ) ret . append ( '{0}.{1}' . format ( form , relname ) ) else : dest_dir = os . path . dirname ( dest ) if not os . path . isdir ( dest_dir ) : os . makedirs ( dest_dir ) shutil . copyfile ( fn_ , dest ) ret . append ( '{0}.{1}' . format ( form , relname ) ) for util_dir in opts [ 'utils_dirs' ] : if mod_dir . endswith ( util_dir ) and mod_dir not in sys . path : sys . path . append ( mod_dir ) touched = bool ( ret ) if opts [ 'clean_dynamic_modules' ] is True : current = set ( _listdir_recursively ( mod_dir ) ) for fn_ in current - remote : full = os . path . join ( mod_dir , fn_ ) if os . path . isfile ( full ) : touched = True os . remove ( full ) while True : emptydirs = _list_emptydirs ( mod_dir ) if not emptydirs : break for emptydir in emptydirs : touched = True shutil . rmtree ( emptydir , ignore_errors = True ) except Exception as exc : log . error ( 'Failed to sync %s module: %s' , form , exc ) return ret , touched
Sync custom modules into the extension_modules directory