def _process_dataset(name, images, vocab, num_shards):
 images = [ImageMetadata(image.image_id, image.filename, [caption]) for image in images for caption in image.captions]
 random.seed(12345)
 random.shuffle(images)
 num_threads = min(num_shards, FLAGS.num_threads)
 spacing = np.linspace(0, len(images), (num_threads + 1)).astype(np.int)
 ranges = []
 threads = []
 for i in xrange((len(spacing) - 1)):
  ranges.append([spacing[i], spacing[(i + 1)]])
 coord = tf.train.Coordinator()
 decoder = ImageDecoder()
 print(('Launching %d threads for spacings: %s' % (num_threads, ranges)))
 for thread_index in xrange(len(ranges)):
  args = (thread_index, ranges, name, images, decoder, vocab, num_shards)
  t = threading.Thread(target=_process_image_files, args=args)
  t.start()
  threads.append(t)
 coord.join(threads)
 print(("%s: Finished processing all %d image-caption pairs in data set '%s'." % (datetime.now(), len(images), name)))

def _process_dataset ( name , images , vocab , num_shards ) : images = [ ImageMetadata ( image . image_id , image . filename , [ caption ] ) for image in images for caption in image . captions ] random . seed ( 12345 ) random . shuffle ( images ) num_threads = min ( num_shards , FLAGS . num_threads ) spacing = np . linspace ( 0 , len ( images ) , ( num_threads + 1 ) ) . astype ( np . int ) ranges = [ ] threads = [ ] for i in xrange ( ( len ( spacing ) - 1 ) ) : ranges . append ( [ spacing [ i ] , spacing [ ( i + 1 ) ] ] ) coord = tf . train . Coordinator ( ) decoder = ImageDecoder ( ) print ( ( 'Launching %d threads for spacings: %s' % ( num_threads , ranges ) ) ) for thread_index in xrange ( len ( ranges ) ) : args = ( thread_index , ranges , name , images , decoder , vocab , num_shards ) t = threading . Thread ( target = _process_image_files , args = args ) t . start ( ) threads . append ( t ) coord . join ( threads ) print ( ( "%s: Finished processing all %d image-caption pairs in data set '%s'." % ( datetime . now ( ) , len ( images ) , name ) ) )
process a complete data set and save it as a tfrecord .