streamcorpus_pipeline/run.py
def make_absolute_paths(config):
    '''given a config dict with streamcorpus_pipeline as a key, find all
    keys under streamcorpus_pipeline that end with "_path" and if the
    value of that key is a relative path, convert it to an absolute
    path using the value provided by root_path
    '''
    if not 'streamcorpus_pipeline' in config:
        logger.critical('bad config: %r', config)
        raise ConfigurationError('missing "streamcorpus_pipeline" from config')
    ## remove the root_path, so it does not get extended itself
    root_path = config['streamcorpus_pipeline'].pop('root_path', None)
    if not root_path:
        root_path = os.getcwd()

    if not root_path.startswith('/'):
        root_path = os.path.join( os.getcwd(), root_path )

    def recursive_abs_path( sub_config, root_path ):
        for key, val in sub_config.items():
            if isinstance(val, basestring):
                if key.endswith('path'):
                    ## ignore URLs in *_path parameters
                    if re.match('^http.?://', val): continue
                    ## we have a path... is it already absolute?
                    if not val.startswith('/'):
                        ## make the path absolute
                        sub_config[key] = os.path.join(root_path, val)

            elif isinstance(val, dict):
                recursive_abs_path( val, root_path )

    recursive_abs_path( config, root_path )

    ## put the root_path back
    config['root_path'] = root_path
def make_absolute_paths(config):
    '''given a config dict with streamcorpus_pipeline as a key, find all
    keys under streamcorpus_pipeline that end with "_path" and if the
    value of that key is a relative path, convert it to an absolute
    path using the value provided by root_path
    '''
    if not 'streamcorpus_pipeline' in config:
        logger.critical('bad config: %r', config)
        raise ConfigurationError('missing "streamcorpus_pipeline" from config')
    ## remove the root_path, so it does not get extended itself
    root_path = config['streamcorpus_pipeline'].pop('root_path', None)
    if not root_path:
        root_path = os.getcwd()

    if not root_path.startswith('/'):
        root_path = os.path.join( os.getcwd(), root_path )

    def recursive_abs_path( sub_config, root_path ):
        for key, val in sub_config.items():
            if isinstance(val, basestring):
                if key.endswith('path'):
                    ## ignore URLs in *_path parameters
                    if re.match('^http.?://', val): continue
                    ## we have a path... is it already absolute?
                    if not val.startswith('/'):
                        ## make the path absolute
                        sub_config[key] = os.path.join(root_path, val)

            elif isinstance(val, dict):
                recursive_abs_path( val, root_path )

    recursive_abs_path( config, root_path )

    ## put the root_path back
    config['root_path'] = root_path
def make_absolute_paths ( config ) : if not 'streamcorpus_pipeline' in config : logger . critical ( 'bad config: %r' , config ) raise ConfigurationError ( 'missing "streamcorpus_pipeline" from config' ) ## remove the root_path, so it does not get extended itself root_path = config [ 'streamcorpus_pipeline' ] . pop ( 'root_path' , None ) if not root_path : root_path = os . getcwd ( ) if not root_path . startswith ( '/' ) : root_path = os . path . join ( os . getcwd ( ) , root_path ) def recursive_abs_path ( sub_config , root_path ) : for key , val in sub_config . items ( ) : if isinstance ( val , basestring ) : if key . endswith ( 'path' ) : ## ignore URLs in *_path parameters if re . match ( '^http.?://' , val ) : continue ## we have a path... is it already absolute? if not val . startswith ( '/' ) : ## make the path absolute sub_config [ key ] = os . path . join ( root_path , val ) elif isinstance ( val , dict ) : recursive_abs_path ( val , root_path ) recursive_abs_path ( config , root_path ) ## put the root_path back config [ 'root_path' ] = root_path
given a config dict with streamcorpus_pipeline as a key, find all
    keys under streamcorpus_pipeline that end with "_path" and if the
    value of that key is a relative path, convert it to an absolute
    path using the value provided by root_path
given a config dict with streamcorpus_pipeline as a key find all keys under streamcorpus_pipeline that end with _path and if the value of that key is a relative path convert it to an absolute path using the value provided by root_path