airflow/sensors/hdfs_sensor.py
def filter_for_ignored_ext(result, ignored_ext, ignore_copying):
        """
        Will filter if instructed to do so the result to remove matching criteria

        :param result: list of dicts returned by Snakebite ls
        :type result: list[dict]
        :param ignored_ext: list of ignored extensions
        :type ignored_ext: list
        :param ignore_copying: shall we ignore ?
        :type ignore_copying: bool
        :return: list of dicts which were not removed
        :rtype: list[dict]
        """
        if ignore_copying:
            log = LoggingMixin().log
            regex_builder = r"^.*\.(%s$)$" % '$|'.join(ignored_ext)
            ignored_extensions_regex = re.compile(regex_builder)
            log.debug(
                'Filtering result for ignored extensions: %s in files %s',
                ignored_extensions_regex.pattern, map(lambda x: x['path'], result)
            )
            result = [x for x in result if not ignored_extensions_regex.match(x['path'])]
            log.debug('HdfsSensor.poke: after ext filter result is %s', result)
        return result
def filter_for_ignored_ext(result, ignored_ext, ignore_copying):
        if ignore_copying:
            log = LoggingMixin().log
            regex_builder = r"^.*\.(%s$)$" % '$|'.join(ignored_ext)
            ignored_extensions_regex = re.compile(regex_builder)
            log.debug(
                'Filtering result for ignored extensions: %s in files %s',
                ignored_extensions_regex.pattern, map(lambda x: x['path'], result)
            )
            result = [x for x in result if not ignored_extensions_regex.match(x['path'])]
            log.debug('HdfsSensor.poke: after ext filter result is %s', result)
        return result
def filter_for_ignored_ext ( result , ignored_ext , ignore_copying ) : if ignore_copying : log = LoggingMixin ( ) . log regex_builder = r"^.*\.(%s$)$" % '$|' . join ( ignored_ext ) ignored_extensions_regex = re . compile ( regex_builder ) log . debug ( 'Filtering result for ignored extensions: %s in files %s' , ignored_extensions_regex . pattern , map ( lambda x : x [ 'path' ] , result ) ) result = [ x for x in result if not ignored_extensions_regex . match ( x [ 'path' ] ) ] log . debug ( 'HdfsSensor.poke: after ext filter result is %s' , result ) return result
Will filter if instructed to do so the result to remove matching criteria

        :param result: list of dicts returned by Snakebite ls
        :type result: list[dict]
        :param ignored_ext: list of ignored extensions
        :type ignored_ext: list
        :param ignore_copying: shall we ignore ?
        :type ignore_copying: bool
        :return: list of dicts which were not removed
        :rtype: list[dict]
Will filter if instructed to do so the result to remove matching criteria