original_docstring: Benchmark online learning.
  
      Sequentially optimize a set of tasks, and compute
      the predictions for each task over time.
  
      Args:
          datasets: list of (inputs, labels) tuples
          model: Keras model
          loss: string or function
          optimizer: string or Keras Optimizer object
          epochs_per_dataset: number of passes through an individual dataset
          ages: number of passes over datasets
          batch_size: batch size
          callbacks: list of functions to call with the model at each iteration
  
      Returns:
          labels:
          predictions:
      
docstring: Benchmark online learning.
  Sequentially optimize a set of tasks, and compute
  the predictions for each task over time.
docstring_tokens: Benchmark online learning . Sequentially optimize a set of tasks and compute the predictions for each task over time .
code: def online_benchmark(datasets, model, loss, optimizer, epochs_per_dataset=1,
          ages=1, batch_size=256, callbacks=None, **kwargs):
      """Benchmark online learning.
  
      Sequentially optimize a set of tasks, and compute
      the predictions for each task over time.
  
      Args:
          datasets: list of (inputs, labels) tuples
          model: Keras model
          loss: string or function
          optimizer: string or Keras Optimizer object
          epochs_per_dataset: number of passes through an individual dataset
          ages: number of passes over datasets
          batch_size: batch size
          callbacks: list of functions to call with the model at each iteration
  
      Returns:
          labels:
          predictions:
      """
  
      # Build the model
      model.compile(loss=loss, optimizer=optimizer, metrics=['accuracy'])
  
      ndataset = len(datasets)
      predictions = [[] for i in range(ndataset)]
      labels = [[] for i in range(ndataset)]
      if callbacks is not None:
          callback_outputs = [[] for i in range(len(callbacks))]
          for cidx, callback in enumerate(callbacks):
              callback_outputs[cidx].append(callback(model))
  
      optimization_data = [[] for i in range(len(model.get_weights())) ]
      for age in range(ages):
          for didx, dataset in enumerate(datasets):
  
              model.fit(*dataset,  batch_size=batch_size, nb_epoch=epochs_per_dataset, verbose=1)
              # Log w, g, g2, ...
              # For all variables, ... ,
              if isinstance(optimizer, Adam):
                  weights = model.get_weights()
                  opt_vars = optimizer.weights[1:]
                  ms = opt_vars[:len(opt_vars)//2]
                  vs = opt_vars[len(opt_vars)//2:]
                  sess = K.get_session()
                  stuff = sess.run([ms, vs])
                  for i in range(len(model.get_weights())):
                      optimization_data[i].append([stuff[0][i], stuff[1][i]])
                  #optimization_data.append(stuff)
  
              # Evaluate on all datasets
              for eval_didx, eval_dataset in enumerate(datasets):
                  # Evaluate model on dataset
                  preds = model.predict(eval_dataset[0])
                  predictions[eval_didx].append(preds)
                  # Convert from 1-hot back to categorical
                  labels[eval_didx].append(np.argmax(eval_dataset[1], 1))
                  print(model.evaluate(*eval_dataset))
              print("")
              if callbacks is not None:
                  for cidx, callback in enumerate(callbacks):
                      callback_outputs[cidx].append(callback(model))
  
      if callbacks is None:
          callback_outputs = None
      # TODO(ben): might break some shit
  
  
      return dict(labels=labels, predictions=predictions,
                  callback_outputs=callback_outputs,
                  optimization_data=optimization_data)
code_tokens: def online_benchmark ( datasets , model , loss , optimizer , epochs_per_dataset = 1 , ages = 1 , batch_size = 256 , callbacks = None , ** kwargs ) : model . compile ( loss = loss , optimizer = optimizer , metrics = [ 'accuracy' ] ) ndataset = len ( datasets ) predictions = [ [ ] for i in range ( ndataset ) ] labels = [ [ ] for i in range ( ndataset ) ] if callbacks is not None : callback_outputs = [ [ ] for i in range ( len ( callbacks ) ) ] for cidx , callback in enumerate ( callbacks ) : callback_outputs [ cidx ] . append ( callback ( model ) ) optimization_data = [ [ ] for i in range ( len ( model . get_weights ( ) ) ) ] for age in range ( ages ) : for didx , dataset in enumerate ( datasets ) : model . fit ( * dataset , batch_size = batch_size , nb_epoch = epochs_per_dataset , verbose = 1 ) if isinstance ( optimizer , Adam ) : weights = model . get_weights ( ) opt_vars = optimizer . weights [ 1 : ] ms = opt_vars [ : len ( opt_vars ) // 2 ] vs = opt_vars [ len ( opt_vars ) // 2 : ] sess = K . get_session ( ) stuff = sess . run ( [ ms , vs ] ) for i in range ( len ( model . get_weights ( ) ) ) : optimization_data [ i ] . append ( [ stuff [ 0 ] [ i ] , stuff [ 1 ] [ i ] ] ) for eval_didx , eval_dataset in enumerate ( datasets ) : preds = model . predict ( eval_dataset [ 0 ] ) predictions [ eval_didx ] . append ( preds ) labels [ eval_didx ] . append ( np . argmax ( eval_dataset [ 1 ] , 1 ) ) print ( model . evaluate ( * eval_dataset ) ) print ( "" ) if callbacks is not None : for cidx , callback in enumerate ( callbacks ) : callback_outputs [ cidx ] . append ( callback ( model ) ) if callbacks is None : callback_outputs = None return dict ( labels = labels , predictions = predictions , callback_outputs = callback_outputs , optimization_data = optimization_data )
short_docstring: Benchmark online learning.
short_docstring_tokens: Benchmark online learning .
comment: """Benchmark online learning.
  
      Sequentially optimize a set of tasks, and compute
      the predictions for each task over time.
  
      Args:
          datasets: list of (inputs, labels) tuples
          model: Keras model
          loss: string or function
          optimizer: string or Keras Optimizer object
          epochs_per_dataset: number of passes through an individual dataset
          ages: number of passes over datasets
          batch_size: batch size
          callbacks: list of functions to call with the model at each iteration
  
      Returns:
          labels:
          predictions:
      """$SEP$# Build the model$SEP$# Log w, g, g2, ...$SEP$# For all variables, ... ,$SEP$#optimization_data.append(stuff)$SEP$# Evaluate on all datasets$SEP$# Evaluate model on dataset$SEP$# Convert from 1-hot back to categorical$SEP$# TODO(ben): might break some shit