def strsize_to_bytes(strsize):
    """
    This function returns the byte size equivalent (as an integer) that best
    represents the string passed in.  the string passed in is intended to be
    in the format as:
        value <unit>

        example: 4TB, 3GB, 10MB, 25KB, etc

    """
    if isinstance(strsize, int):
        # Nothing further to do
        return strsize

    try:
        strsize_re = re.match(
            r'\s*(?P<size>(0|[1-9][0-9]*)(\.(0+|[1-9][0-9]*))?)\s*'
            r'((?P<unit>.)(?P<type>[bB])?)?\s*',
            strsize,
        )
    except TypeError:
        return None

    if not strsize_re:
        # Not good
        return None

    size = int(strsize_re.group('size'))
    unit = strsize_re.group('unit')
    use_bytes = strsize_re.group('type') != 'b'

    if not unit:
        return size

    # convert unit to uppercase
    unit = unit.upper()

    if use_bytes:
        if unit == 'B':
            # We're going to assume bytes and return now
            return size

        elif unit == 'K':
            # We're dealing with Kilobytes
            return size * 1024

        elif unit == 'M':
            # We're dealing with Megabytes
            return size * 1048576

        elif unit == 'G':
            # We're dealing with Gigabytes
            return size * 1073741824

        elif unit == 'T':
            # We're dealing with Terabytes
            return size * 1073741824 * 1024

    # In Bits
    if unit == 'K':
        # We're dealing with Kilobytes
        return size * 1e3

    elif unit == 'M':
        # We're dealing with Megabytes
        return size * 1e6

    elif unit == 'G':
        # We're dealing with Gigabytes
        return size * 1e9

    elif unit == 'T':
        # We're dealing with Terabytes
        return size * 1e12

    # Unsupported Type
    return None def bytes_to_strsize(byteval):
    """
    This function returns the string size equivalent (as a string) that best
    represents the integer (in bytes) passed in.

    """
    unit = 'B'
    try:
        byteval = float(byteval)

    except(ValueError, TypeError):
        return None

    if byteval >= 1024.0:
        byteval = byteval / 1024.0
        unit = 'KB'
        if byteval >= 1024.0:
            byteval = byteval / 1024.0
            unit = 'MB'
            if byteval >= 1024.0:
                byteval = byteval / 1024.0
                unit = 'GB'
                if byteval >= 1024.0:
                    byteval = byteval / 1024.0
                    unit = 'TB'

    return '%.2f%s' % (round(byteval, 2), unit) def stat(path, fsinfo=True, mime=True):
    """
    A wrapper to the stat() python class that handles exceptions and
    converts all file stats into datetime objects.

    A simple response would look like this:
        {
            'basename': 'filename.rar',
            'dirname': '/path/to/location/',
            'extension': 'rar',
            'filename': 'filename',

            # the below appears if you specified fsinfo=True
            'created': datetime(),      # file creation time
            'modified': datetime(),     # file last modified time
            'accessed': datetime(),     # file last access time
            'size': 3002423,            # size is in bytes

            # Mime Type (if boolean set)
            'mime': 'application/rar'
        }
    """
    # If we reach here, we store the file found
    _abspath = abspath(path)
    _basename = basename(path)

    try:
        stat_obj = os_stat(_abspath)

    except OSError:
        # File was not found or recently removed
        return None

    nfo = {
        'basename': _basename,
        'dirname': dirname(_abspath),
        'extension': splitext(_basename)[1].lower(),
        'filename': splitext(_basename)[0],
    }

    if fsinfo:
        # Extend file information
        try:
            nfo['modified'] = \
                datetime.fromtimestamp(stat_obj[ST_MTIME])

        except ValueError:
            nfo['modified'] = \
                datetime(1980, 1, 1, 0, 0, 0, 0)

        try:
            nfo['accessed'] = \
                datetime.fromtimestamp(stat_obj[ST_ATIME])

        except ValueError:
            nfo['accessed'] = \
                datetime(1980, 1, 1, 0, 0, 0, 0)

        try:
            nfo['created'] = \
                datetime.fromtimestamp(stat_obj[ST_CTIME])

        except ValueError:
            nfo['created'] = \
                datetime(1980, 1, 1, 0, 0, 0, 0)

        nfo['size'] = stat_obj[ST_SIZE]

    if mime:
        # Create ourselves a mime object
        m = Mime()
        mr = m.from_file(_abspath)

        if mr is None or mr.type() == DEFAULT_MIME_TYPE:
            mr = m.from_filename(_basename)

        # Store our type
        nfo['mime'] = mr.type()

    return nfo def mkdir(name, perm=0775):
    """
    A more contained wrapper to directory management
    """
    attempt = 3
    if isdir(name):
        return True

    while attempt > 0:
        try:
            makedirs(name, perm)
            logger.debug('Created directory: %s' % name)
            return True

        except OSError, e:
            if e[0] == errno.EEXIST:
                # directory exists; this is okay
                return isdir(name)

            logger.debug('Created directory %s exception: %s' % (
                name, e,
            ))

        # racing condition; just try again
        attempt -= 1

    # To many attempts... fail
    # ... fall through...
    return False @contextmanager
def pushd(newdir, create_if_missing=False, perm=0775):
    """
    # A pushd/popd implimentation
    # Based on : http://stackoverflow.com/questions/6194499/\
                    pushd-through-os-system

    # It's use is pretty straight forward:
    # with pushd('somewhere'):
    #     # somewhere
    #     print os.getcwd()
    #
    # # wherever you started
    # print os.getcwd()

    """
    prevdir = getcwd()
    if not isdir(newdir) and create_if_missing:
        # Don't bother checking the success or not
        # we'll find out soon enough with chdir()
        mkdir(newdir, perm)

    chdir(newdir)
    try:
        yield

    finally:
        # Fall back to previous directory popd()
        chdir(prevdir) def find(search_dir, regex_filter=None, prefix_filter=None,
         suffix_filter=None, fsinfo=False, mime=False, followlinks=False,
         min_depth=None, max_depth=None, case_sensitive=False,
         *args, **kwargs):
    """Returns a dict object of the files found in the download
       directory. You can additionally pass in filters as a list or
       string) to filter the results returned.

          ex:
          {
             '/full/path/to/file.mkv': {
                 'basename': 'file.mkv',
                 'dirname': '/full/path/to',
                 # identify the filename (without applied extension)
                 'filename': 'file',
                 # always tolower() applied to:
                 'extension': 'mkv',

                 # If fullstatus == True then the following additional
                 # content is provided.

                 # file size is in bytes
                 'size': 10000,
                 # accessed date
                 'accessed': datetime(),
                 # created date
                 'created': datetime(),
                 # created date
                 'modified': datetime(),
             }
          }

          the function returns None if something bad happens

    """

    # Internal Tracking of Directory Depth
    current_depth = kwargs.get('__current_depth', 1)
    if current_depth == 1:
        # General Error Checking for first iteration through
        if min_depth and max_depth and min_depth > max_depth:
            return None

        # Translate to full absolute path
        search_dir = abspath(expanduser(search_dir))

    root_dir = kwargs.get('__root_dir', search_dir)

    # Build file list
    files = {}
    if isinstance(search_dir, (sortedset, set, list, tuple)):
        for _dir in search_dir:
            # use recursion to build a master (unique) list
            files = dict(files.items() + find(
                search_dir=_dir,
                regex_filter=regex_filter,
                prefix_filter=prefix_filter,
                suffix_filter=suffix_filter,
                fsinfo=fsinfo,
                mime=mime,
                followlinks=followlinks,
                min_depth=min_depth,
                max_depth=max_depth,
                case_sensitive=case_sensitive,
                # Internal Current Directory Depth tracking
                __current_depth=current_depth,
                __root_dir=root_dir,
            ).items())
        return files

    elif not isinstance(search_dir, basestring):
        # Unsupported
        return {}

    # Change all filters strings lists (if they aren't already)
    if regex_filter is None:
        regex_filter = tuple()
    if isinstance(regex_filter, basestring):
        regex_filter = (regex_filter,)
    elif isinstance(regex_filter, re._pattern_type):
        regex_filter = (regex_filter,)
    if suffix_filter is None:
        suffix_filter = tuple()
    if isinstance(suffix_filter, basestring):
        suffix_filter = (suffix_filter, )
    if prefix_filter is None:
        prefix_filter = tuple()
    if isinstance(prefix_filter, basestring):
        prefix_filter = (prefix_filter, )

    # clean prefix list
    if prefix_filter:
        prefix_filter = parse_list(prefix_filter)

    # clean up suffix list
    if suffix_filter:
        suffix_filter = parse_list(suffix_filter)

    # Precompile any defined regex definitions
    if regex_filter:
        _filters = []
        for f in regex_filter:
            if not isinstance(f, re._pattern_type):
                flags = 0x0
                if not case_sensitive:
                    flags = re.IGNORECASE

                try:
                    _filters.append(re.compile(f, flags=flags))

                except:
                    logger.error(
                        'Invalid regular expression: "%s"' % f,
                    )
                    return None
            else:
                # precompiled already
                _filters.append(f)
        # apply
        regex_filter = _filters

    if current_depth == 1:
        # noise reduction; only display this notice once (but not on
        # each recursive call)
        logger.debug("get_files('%s') with %d filter(s)" % (
            search_dir,
            len(prefix_filter) + len(suffix_filter) + len(regex_filter),
        ))

    if isfile(search_dir):
        fname = basename(search_dir)
        filtered = False
        if regex_filter:
            filtered = True
            for regex in regex_filter:
                if regex.search(fname):
                    logger.debug('Allowed %s (regex)' % fname)
                    filtered = False
                    break

        if not filtered and prefix_filter:
            filtered = True
            for prefix in prefix_filter:
                if case_sensitive:
                    # We use slicing and not startswith() because slicing is
                    # faster
                    if fname[0:len(prefix)] == prefix:
                        logger.debug('Allowed %s (prefix)' % fname)
                        filtered = False
                        break
                else:
                    # Not Case Sensitive
                    if fname[0:len(prefix)].lower() == prefix.lower():
                        logger.debug('Allowed %s (prefix)' % fname)
                        filtered = False
                        break

        if not filtered and suffix_filter:
            filtered = True
            for suffix in suffix_filter:
                if case_sensitive:
                    # We use slicing and not endswith() because slicing is
                    # faster
                    if fname[-len(suffix):] == suffix:
                        # Allowed
                        filtered = False
                        break
                else:
                    # Not Case Sensitive
                    if fname[-len(suffix):].lower() == suffix.lower():
                        logger.debug('Allowed %s (suffix)' % fname)
                        filtered = False
                        break

        if filtered:
            # File does not meet implied filters
            return {}

        # If we reach here, we can prepare a file using the data
        # we fetch
        _file = {
            search_dir: stat(search_dir, fsinfo=fsinfo, mime=mime),
        }
        if _file[search_dir] is None:
            del files[search_dir]
            logger.warning(
                'The file %s became inaccessible' % fname,
            )
            return {}

        return _file

    elif not isdir(search_dir):
        return {}

    # Get Directory entries
    dirents = [d for d in listdir(search_dir)
               if d not in ('..', '.')]

    for dirent in dirents:
        # Store Path
        fullpath = join(search_dir, dirent)

        if isdir(fullpath):
            # Max Depth Handling
            if max_depth and max_depth <= current_depth:
                continue

            if not followlinks and islink(fullpath):
                # honor followlinks
                continue

            # use recursion to build a master (unique) list
            files = dict(files.items() + find(
                search_dir=fullpath,
                regex_filter=regex_filter,
                prefix_filter=prefix_filter,
                suffix_filter=suffix_filter,
                fsinfo=fsinfo,
                mime=mime,
                followlinks=followlinks,
                min_depth=min_depth,
                max_depth=max_depth,
                case_sensitive=case_sensitive,
                # Internal Current Directory Depth tracking
                __current_depth=current_depth + 1,
                __root_dir=root_dir,
            ).items())
            continue

        elif not isfile(fullpath):
            # Unknown Type
            logger.debug('Skipping %s (unknown)' % dirent)
            continue

        # Min depth handling
        if min_depth and min_depth > current_depth:
            continue

        # Apply filters to match filed
        if regex_filter:
            filtered = True
            for regex in regex_filter:
                if regex.search(dirent):
                    logger.debug('Allowed %s (regex)' % dirent)
                    filtered = False
                    break

            if filtered:
                # Denied
                continue

        if prefix_filter:
            filtered = True
            for prefix in prefix_filter:
                if case_sensitive:
                    # We use slicing and not startswith() because slicing is
                    # faster
                    if dirent[0:len(prefix)] == prefix:
                        logger.debug('Allowed %s (prefix)' % dirent)
                        filtered = False
                        break
                else:
                    # Not Case Sensitive
                    if dirent[0:len(prefix)].lower() == prefix.lower():
                        logger.debug('Allowed %s (prefix)' % dirent)
                        filtered = False
                        break

            if filtered:
                # Denied
                continue

        if suffix_filter:
            filtered = True
            for suffix in suffix_filter:
                if case_sensitive:
                    # We use slicing and not endswith() because slicing is
                    # faster
                    if dirent[-len(suffix):] == suffix:
                        logger.debug('Allowed %s (suffix)' % dirent)
                        filtered = False
                        break
                else:
                    # Not Case Sensitive
                    if dirent[-len(suffix):].lower() == suffix.lower():
                        logger.debug('Allowed %s (suffix)' % dirent)
                        filtered = False
                        break

            if filtered:
                # Denied
                continue

        # If we reach here, we store the file found
        files[fullpath] = stat(fullpath, fsinfo=fsinfo, mime=mime)
        if files[fullpath] is None:
            # File was not found or recently removed
            del files[fullpath]
            logger.warning(
                'The file %s became inaccessible' % dirent,
            )
            continue

    # Return all files
    return files def rm(path, *args, **kwargs):
    """
    A  rmtree reimplimentation. More importantly, Microsoft Windows
    can't recursively delete a directory if it contains read-only files
    with rmtree().  This rm() attempts to handle this. Symbolic links
    are not supported

    See http://stackoverflow.com/a/2656405 for a partial basis of
    why this rewrite exists.  The on_error refernced in this function didn't
    really satisfy my needs, so a rewrite of the rmtree() function was
    required.

    """
    if not exists(path):
        # Nothing more to do
        return True

    # We track our recursion level so we don't go in some unexitable loop
    recursion_count = kwargs.get('__recursion_count', 0)
    if recursion_count == 0:
        # We're on our first iteration
        logger.debug('Attempting to remove %s' % path)

    elif recursion_count >= RM_RECURSION_LIMIT:
        # Recursive Limit reached
        return False

    if not access(path, W_OK):
        # Is the error an access error ?
        chmod(path, S_IRWXU)

    if islink(path):
        # symlinks to directories are forbidden (taken from shutils.rmtree()
        # see: https://bugs.python.org/issue1669

        # We'll attempt to treat the link as a file and NOT a directory. If we
        # fail then we're done
        try:
            unlink(path)
            logger.debug('Removed link %s' % path)
            return True

        except:
            logger.warning(
                'Failed to remove link %s' % path)
            return False

    if isdir(path):
        names = []
        try:
            names = listdir(path)

        except _OSError:
            # Failed
            return False

        for name in names:
            fullname = join(path, name)
            try:
                mode = lstat(fullname).st_mode

            except _OSError:
                mode = 0

            if S_ISDIR(mode):
                if not access(fullname, W_OK):
                    # Make sure we can access 'this' path
                    chmod(fullname, S_IRWXU)

                try:
                    # Attempt to remove directory
                    rmdir(fullname)

                except _OSError:
                    # It's not empty, so recursively enter it
                    if not rm(
                            fullname, __recursion_level=recursion_count + 1):
                        return False
            else:
                # We're dealing with file/link
                try:
                    unlink(fullname)
                    logger.debug('Removed file %s' % fullname)

                except:
                    logger.warning('Failed to remove file %s' % fullname)
                    return False

        try:
            rmdir(path)
            logger.debug('Removed file %s' % path)

        except _OSError:
            logger.warning('Failed to remove file %s' % path)
            return False

    else:
        # We're dealing with file/link
        try:
            unlink(path)
            logger.debug('Removed file %s' % path)

        except:
            logger.warning('Failed to remove file %s' % path)
            return False

    return True def parse_list(*args):
    """
    Take a string list and break it into a delimited
    list of arguments. This funciton also supports
    the processing of a list of delmited strings and will
    always return a unique set of arguments. Duplicates are
    always combined in the final results.

    You can append as many items to the argument listing for
    parsing.

    Hence: parse_list('.mkv, .iso, .avi') becomes:
        ['.mkv', '.iso', '.avi']

    Hence: parse_list('.mkv, .iso, .avi', ['.avi', '.mp4']) becomes:
        ['.mkv', '.iso', '.avi', '.mp4']

    The parsing is very forgiving and accepts spaces, slashes, commas
    semicolons, and pipes as delimiters
    """

    result = []
    for arg in args:
        if isinstance(arg, basestring):
            result += re.split(STRING_DELIMITERS, arg)

        elif isinstance(arg, (list, tuple, set, sortedset)):
            for _arg in arg:
                if isinstance(arg, basestring):
                    result += re.split(STRING_DELIMITERS, arg)

                # A list inside a list? - use recursion
                elif isinstance(_arg, (list, tuple, set, sortedset)):
                    result += parse_list(_arg)

                else:
                    # Convert whatever it is to a string and work with it
                    result += parse_list(str(_arg))
        else:
            # Convert whatever it is to a string and work with it
            result += parse_list(str(arg))

    # apply as well as make the list unique by converting it
    # to a set() first. filter() eliminates any empty entries
    return filter(bool, list(set(result))) def parse_bool(arg, default=False):
    """
    Parses strings such as 'yes' and 'no' as well as other strings such as
    'on' or 'off' , 'enable' or 'disable', etc.

    This method can just simplify checks to these variables.

    If the content could not be parsed, then the default is
    returned.
    """

    if isinstance(arg, basestring):
        # no = no - False
        # of = short for off - False
        # 0  = int for False
        # fa = short for False - False
        # f  = short for False - False
        # n  = short for No or Never - False
        # ne  = short for Never - False
        # di  = short for Disable(d) - False
        # de  = short for Deny - False
        if arg.lower()[0:2] in ('de', 'di', 'ne', 'f', 'n', 'no', 'of',
                                '0', 'fa'):
            return False
        # ye = yes - True
        # on = short for off - True
        # 1  = int for True
        # tr = short for True - True
        # t  = short for True - True
        # al = short for Always (and Allow) - True
        # en  = short for Enable(d) - True
        elif arg.lower()[0:2] in ('en', 'al', 't', 'y', 'ye', 'on', '1',
                                  'tr'):
            return True
        # otherwise
        return default

    # Handle other types
    return bool(arg) def parse_paths(*args):
    """
    Very similar to the parse_list() however this parses a listing of
    provided directories.  The difference is that white space is
    treated a bit more strictly since directory paths can contain
    spaces in them. Trailing (back)slashes are always removed from
    results. Duplicates are always combined in final results.

    Hence: parse_paths('C:\\test dir\\, D:\\test2') becomes:
        [ 'C:\\test dir', D:\\test2' ]

    Hence: parse_paths('C:\\test dir\\, D:\\test2',
        [ 'H:\\test 4', 'C:\\test dir', 'D:\\test2' ]
    becomes:
        [ 'C:\\test dir', D:\\test2', 'H:\\test 4' ]
    """

    result = []
    for arg in args:
        if isinstance(arg, basestring):
            cleaned = PATH_DELIMITERS_RE.sub('|/', tidy_path(arg))
            cleaned = WIN_PATH_DELIMITERS_RE.sub('|\\1', cleaned)
            cleaned = WIN_NETWORK_DRIVE_DELIMITERS_RE.sub('|\\1', cleaned)
            cleaned = WIN_LOCAL_DRIVE_DELIMITERS_RE.sub('|\\1:\\2', cleaned)
            result += re.split('[,|]+', cleaned)

        elif isinstance(arg, (list, tuple)):
            for _arg in arg:
                if isinstance(_arg, basestring):
                    cleaned = PATH_DELIMITERS_RE.sub('|', tidy_path(_arg))
                    cleaned = WIN_PATH_DELIMITERS_RE.sub('|\\1', cleaned)
                    cleaned = WIN_NETWORK_DRIVE_DELIMITERS_RE.sub(
                        '|\\1', cleaned)
                    cleaned = WIN_LOCAL_DRIVE_DELIMITERS_RE.sub(
                        '|\\1:\\2', cleaned)
                    result += re.split('[,|]+', cleaned)

                # A list inside a list? - use recursion
                elif isinstance(_arg, (list, tuple)):
                    result += parse_paths(_arg)

                else:

                    # unsupported content
                    continue
        else:
            # unsupported content (None, bool's, int's, floats, etc)
            continue

    # apply as well as make the list unique by converting it
    # to a set() first. filter() eliminates any empty entries
    return filter(bool, list(set([tidy_path(p) for p in result]))) def scan_pylib(paths, ignore_re=DEFAULT_PYLIB_IGNORE_LIST):
    """
    A simple function that scans specified paths for .pyc files
    it returns a dictionary of files it scanned using the paths
    specified.

    You can optionally specify a list of items you wish to
    ignore from the matched results.  This allows you to filter
    content. By default, anything starting with an underscore
    is skipped.

    The list of module names found is returned:
        {
            'foo' : (
               '/absolute/path/to/foo.py',
               '/another_path/to/another/foo.py',
            ),
            'bob' : (
                '/absolute/path/to/bob.py',
            ),
        }
    """
    # Module paths
    rpaths = {}

    # Prepare our list of paths
    paths = parse_paths(paths)

    def store(path, filename):
        """
        Attempts to store module
        """
        result = PYTHON_MODULE_RE.match(filename)
        if not result:
            # Not our type of file
            return False

        if next((True for r in ignore_re
                 if r.match(result.group('fname')) is not None), False):
            # We already loaded an alike file
            # we don't wnat to over-ride it or we matched an element
            # from our ignore list
            return False

        if result.group('fname') not in rpaths:
            rpaths[result.group('fname')] = set()

        # Store unique entry
        rpaths[result.group('fname')].add(join(path, filename))

        return True

    for _path in paths:
        path = abspath(expanduser(_path))
        try:
            if isfile(path):
                # we're being passed a module directly
                store(dirname(path), basename(path))

            else:
                # We're dealing with a directory
                for filename in listdir(path):
                    store(path, filename)

        except (OSError, IOError) as e:
            # We failed, do not return an empty set; just abort
            # out right
            logger.debug('scan_pylib(%s) exception: %s' % (path, e))
            return None

    return rpaths def load_pylib(module_name, filepath=None):
    """
    Loads a python module presumable retreived by the
    scan_pylib() function call.

    module_name should be the name of the module path
    you want to import from within the file.

    ie:
        load_pylib('plugin.test', '/path/to/plugin.py')

    If the filepath is None, then the module_name is
    expected to be the full path to the module in question
    to be loaded.

    """

    if filepath is None:
        if not isfile(module_name):
            return None

        # this is an allowed setting
        filepath = module_name

        module_name = basename(filepath)
        match = PYTHON_MODULE_RE.match(module_name)
        if not match:
            return None

        module_name = match.group('fname')

        # fall through for loading

    try:
        if PYTHON_3:
            return SourceFileLoader(module_name, filepath).load_module()
        return load_source(module_name, filepath)

    except ImportError as e:
        # Could not load module
        logger.warning(
            'Failed to import from dynamic module: %s (reason: %s)' % (
                filepath,
                str(e),
            )
        )
    except IOError as e:
        # Could not load module
        logger.warning(
            'Failed to load dynamic module: %s (reason: %s)' % (
                filepath,
                e[1],
            )
        )
        return None def hexdump(src, length=16, sep='.'):
    """
    Displays a hex output of the content it is passed.

    This was based on https://gist.github.com/7h3rAm/5603718 with some
    minor modifications
    """
    allowed = digits + ascii_letters + punctuation + ' '

    print_map = ''.join(((x if x in allowed else '.')
                        for x in map(chr, range(256))))
    lines = []

    for c in xrange(0, len(src), length):
        chars = src[c:c + length]
        hex = ' '.join(["%02x" % ord(x) for x in chars])
        if len(hex) > 24:
            hex = "%s %s" % (hex[:24], hex[24:])
        printable = ''.join(["%s" % (
            (ord(x) <= 127 and print_map[ord(x)]) or sep) for x in chars])
        lines.append("%08x:  %-*s  |%s|" % (c, length * 3, hex, printable))
    return '\n'.join(lines) def dirsize(src):
    """
    Takes a source directory and returns the entire size of all of it's
    content(s) in bytes.

    The function returns None if the size can't be properly calculated.
    """
    if not isdir(src):
        # Nothing to return
        return 0

    try:
        with pushd(src, create_if_missing=False):
            size = sum(getsize(f) for f in listdir('.') if isfile(f))

    except (OSError, IOError):
        return None

    # Return our total size
    return size NEWSREAP_ENGINE = '%s.engine' % NEWSREAP_LOGGER
import sys
import gevent.monkey
import errno
import re
import logging
from blist import sortedset
from os.path import abspath
from os.path import dirname
from os.path import basename
from os.path import isdir
from os.path import isfile
from os.path import join
from os import chmod
from os import getcwd
from itertools import chain
    from tests.TestBase import TestBase
    from tests.TestBase import TestBase
from newsreap.Utils import strsize_to_bytes
from newsreap.Utils import bytes_to_strsize
from newsreap.Utils import stat
from newsreap.Utils import mkdir
from newsreap.Utils import pushd
from newsreap.Utils import find
from newsreap.Utils import rm
from newsreap.Utils import parse_list
from newsreap.Utils import parse_bool
from newsreap.Utils import parse_paths
from newsreap.Utils import scan_pylib
from newsreap.Utils import load_pylib
from newsreap.Utils import hexdump
from newsreap.Utils import dirsize
from newsreap.Logging import NEWSREAP_ENGINE
        )
        assert isinstance(results, dict)
        # Not 20 (because no extra file was created on depth level 1)
        assert len(results) == 19

        # Search only the second level
        results = find(
            work_dir,
            suffix_filter='.jpeg',
            min_depth=2,
            max_depth=2,
            case_sensitive=True,
        )
        assert isinstance(results, dict)
        # there should be 2 now
        assert len(results) == 2

        for k in results.keys():
            # 2 directories now each with the same filename
            assert 'depth02.jpeg' == basename(k)

        # Create a 12th and 13th level; but store nothing in the 12th
        work_dir_12 = join(work_dir_depth, 'level%.2d' % 12)

        assert mkdir(work_dir_12) is True
        work_dir_13 = join(work_dir_12, 'level%.2d' % 13)
        assert self.touch(
            join(work_dir_13, 'depth%.2d.jpeg' % 13),
        ) is True

        # Search the 12th level which contains no files
        # (the 13th does but we're explicity not looking there)
        results = find(
            work_dir_12,
            min_depth=1,
            max_depth=1,
        )
        # even with no results we should get a dictionary response
        assert isinstance(results, dict)
        # there should be 0 now
        assert len(results) == 0

    def test_rm(self):
        """
        rm is just a simple wrapper for unlink and rmtree.
        it returns True if it was successful and false if it failed.

        it's equivalent to rm -rf <path>

        """

        work_dir = join(self.tmp_dir, 'Utils_Test.rm')
        # The directory should not exist
        assert isdir(work_dir) is False

        # mkdir() should be successful
        assert mkdir(work_dir) is True

        # Remove the directory
        assert rm(work_dir) is True

        # The directory should not exist
        assert isdir(work_dir) is False

        # Temporary directory
        tmp_dir = join(work_dir, 'testdir', 'test01')
        tmp_file = join(tmp_dir, 'test.file.ogg')

        # create a file in it
        assert self.touch(tmp_file, perm=0000) is True

        # The directory should exist
        assert isdir(tmp_dir) is True
        assert isfile(tmp_file) is True

        # Remove the directory
        assert rm(tmp_dir) is True

        # The directory nor the file should no longer exist
        assert isdir(tmp_dir) is False
        assert isfile(tmp_file) is False

        # Create the file again
        assert self.touch(tmp_file, perm=0000) is True
        assert isfile(tmp_file) is True
        # Set the directory it resides in with bad permissions
        chmod(tmp_dir, 0000)

        # The directory should exist
        assert isdir(tmp_dir) is True

        # Remove the directory; just using rmtree() in this circumstance would
        # cause an exception to be thrown, however rm() should handle this
        # gracefully
        assert rm(tmp_dir) is True

        # The directory nor the file should no longer exist
        assert isdir(tmp_dir) is False
        assert isfile(tmp_file) is False

        # Now just to repeat this step with a directory without permissions
        # within a directory without permissions

        tmp_dir_level2 = join(tmp_dir, 'level02')
        tmp_file = join(tmp_dir_level2, 'test.file.ogg')
        # create a file in it
        assert self.touch(tmp_file, perm=0000) is True

        # The directories and file should exist now
        assert isdir(tmp_dir) is True
        assert isdir(tmp_dir_level2) is True
        assert isfile(tmp_file) is True

        # Set the directory it resides in with bad permissions
        chmod(tmp_dir_level2, 0000)
        chmod(tmp_dir, 0000)

        # Remove the directory; just using rmtree() in this circumstance would
        # cause an exception to be thrown, however rm() should handle this
        # gracefully

        assert rm(tmp_dir) is True