Recursively index files under a root directory.

    Each regular file is added *in-place* to the files_by_size dictionary,
    according to the file size. This is a (possibly empty) dictionary of
    lists of filenames, indexed by file size.

    exclude_dirs is a list of glob patterns to exclude directories.
    exclude_files is a list of glob patterns to exclude files.

    follow_dirlinks controls whether to follow symbolic links to
    subdirectories while crawling.

    Returns True if there were any I/O errors while listing directories.

    Returns a list of error messages that occurred. If empty, there were no
    errors.

def index_files_by_size(root, files_by_size, exclude_dirs, exclude_files,
        follow_dirlinks):
    """Recursively index files under a root directory.

    Each regular file is added *in-place* to the files_by_size dictionary,
    according to the file size. This is a (possibly empty) dictionary of
    lists of filenames, indexed by file size.

    exclude_dirs is a list of glob patterns to exclude directories.
    exclude_files is a list of glob patterns to exclude files.

    follow_dirlinks controls whether to follow symbolic links to
    subdirectories while crawling.

    Returns True if there were any I/O errors while listing directories.

    Returns a list of error messages that occurred. If empty, there were no
    errors.

    """
    # encapsulate the value in a list, so we can modify it by reference
    # inside the auxiliary function
    errors = []
    already_visited = set()

    def _print_error(error):
        """Print a listing error to stderr.

        error should be an os.OSError instance.

        """
        # modify the outside errors value; must be encapsulated in a list,
        # because if we assign to a variable here we just create an
        # independent local copy
        msg = "error listing '%s': %s" % (error.filename, error.strerror)
        sys.stderr.write("%s\n" % msg)
        errors.append(msg)


    # XXX: The actual root may be matched by the exclude pattern. Should we
    # prune it as well?

    for curr_dir, subdirs, filenames in os.walk(root, topdown=True,
            onerror=_print_error, followlinks=follow_dirlinks):

        # modify subdirs in-place to influence os.walk
        subdirs[:] = prune_names(subdirs, exclude_dirs)
        filenames = prune_names(filenames, exclude_files)

        # remove subdirs that have already been visited; loops can happen
        # if there's a symlink loop and follow_dirlinks==True, or if
        # there's a hardlink loop (which is usually a corrupted filesystem)
        subdirs[:], already_visited = filter_visited(curr_dir, subdirs,
                already_visited, follow_dirlinks, _print_error)

        for base_filename in filenames:
            full_path = os.path.join(curr_dir, base_filename)

            # avoid race condition: file can be deleted between os.walk()
            # seeing it and us calling os.lstat()
            try:
                file_info = os.lstat(full_path)
            except OSError as e:
                _print_error(e)
                continue

            # only want regular files, not symlinks
            if stat.S_ISREG(file_info.st_mode):
                size = file_info.st_size

                if size in files_by_size:
                    # append to the list of files with the same size
                    files_by_size[size].append(full_path)
                else:
                    # start a new list for this file size
                    files_by_size[size] = [full_path]

    return errors