def web_listing_all_files(url, count=0, max_count=None):
    urls = []
    dirs, files = parse_web_listing(url)
    for f in files:
        urls.append(url + f)
        count += 1
        if max_count is not None and count >= max_count:
            logger.warning('Reached maximum limit for this category')
            return urls, count
    for d in dirs:
        new_urls, count = web_listing_all_files(url + d, count, max_count)
        urls += new_urls
        if max_count is not None and count >= max_count:
            break
    return urls, count
Utility for parse_folder()
dataset/ETHPy150Open NVIDIA/DIGITS/tools/parse_folder.py/web_listing_all_files