def _fill_load_queue(filename, queue, shuffle):
 total_lines = 0
 valid_lines = 0
 distribution = Counter()
 with open(filename) as infile:
  if shuffle:
   lines = infile.readlines()
   random.shuffle(lines)
   for line in lines:
    total_lines += 1
    try:
     result = _parse_line(line, distribution)
     valid_lines += 1
     queue.put(result)
    except ParseLineError:
     pass
  else:
   for line in infile:
    total_lines += 1
    try:
     result = _parse_line(line, distribution)
     valid_lines += 1
     queue.put(result)
    except ParseLineError:
     pass
 logger.debug(('%s total lines in file' % total_lines))
 if (valid_lines == 0):
  raise BadInputFileError('No valid lines in input file')
 logger.info(('%s valid lines in file' % valid_lines))
 for key in sorted(distribution):
  logger.debug(('Category %s has %d images.' % (key, distribution[key])))
 return valid_lines

def _fill_load_queue ( filename , queue , shuffle ) : total_lines = 0 valid_lines = 0 distribution = Counter ( ) with open ( filename ) as infile : if shuffle : lines = infile . readlines ( ) random . shuffle ( lines ) for line in lines : total_lines += 1 try : result = _parse_line ( line , distribution ) valid_lines += 1 queue . put ( result ) except ParseLineError : pass else : for line in infile : total_lines += 1 try : result = _parse_line ( line , distribution ) valid_lines += 1 queue . put ( result ) except ParseLineError : pass logger . debug ( ( '%s total lines in file' % total_lines ) ) if ( valid_lines == 0 ) : raise BadInputFileError ( 'No valid lines in input file' ) logger . info ( ( '%s valid lines in file' % valid_lines ) ) for key in sorted ( distribution ) : logger . debug ( ( 'Category %s has %d images.' % ( key , distribution [ key ] ) ) ) return valid_lines
fill the queue with data from the input file print the category distribution returns the number of lines added to the queue note: this can be slow on a large input file .