def web_listing_all_files(url, count=0, max_count=None):
 urls = []
 (dirs, files) = parse_web_listing(url)
 for f in files:
  urls.append((url + f))
  count += 1
  if ((max_count is not None) and (count >= max_count)):
   logger.warning('Reached maximum limit for this category')
   return (urls, count)
 for d in dirs:
  (new_urls, count) = web_listing_all_files((url + d), count, max_count)
  urls += new_urls
  if ((max_count is not None) and (count >= max_count)):
   break
 return (urls, count)

def web_listing_all_files ( url , count = 0 , max_count = None ) : urls = [ ] ( dirs , files ) = parse_web_listing ( url ) for f in files : urls . append ( ( url + f ) ) count += 1 if ( ( max_count is not None ) and ( count >= max_count ) ) : logger . warning ( 'Reached maximum limit for this category' ) return ( urls , count ) for d in dirs : ( new_urls , count ) = web_listing_all_files ( ( url + d ) , count , max_count ) urls += new_urls if ( ( max_count is not None ) and ( count >= max_count ) ) : break return ( urls , count )
utility for parse_folder() gets all files from a url by parsing the directory and all subdirectories looking for image files returns .