original_docstring:  Train the Hybrid_Recsys_Model and export model to bucket.
  
      Args:
          args (dict): dict of arguments from task.py
  
      Returns:
          None
      
docstring: Train the Hybrid_Recsys_Model and export model to bucket.
docstring_tokens: Train the Hybrid_Recsys_Model and export model to bucket .
code: def train_and_export_model(args):
      """ Train the Hybrid_Recsys_Model and export model to bucket.
  
      Args:
          args (dict): dict of arguments from task.py
  
      Returns:
          None
      """
      # create dataset
      feature_col = ['user_id', 'item_id', 'title', 'author', 'category', 'device_brand', 'article_year', 'article_month', 'next_item_id']
      u_latent_col = ["user_latent_{}".format(i) for i in range(2 * args["latent_num"])]
      i_latent_col = ["item_latent_{}".format(i) for i in range(2 * args["latent_num"])]
      column_name = feature_col + u_latent_col + i_latent_col
      
      label_name = 'next_item_id'
      defaults = ['unknown'] * len(feature_col) + [0.0] * (len(u_latent_col) + len(i_latent_col))
      batch_size = args["batch_size"]
      train_path = args["train_data_path"]
      test_path = args["test_data_path"]
      
      train_dataset = create_dataset(train_path, column_name, label_name, defaults, batch_size, True)
      test_dataset = create_dataset(test_path, column_name, label_name, defaults, batch_size, False)
      
      # create model
      model = Hybrid_Recsys_Model(args["item_id_path"], args["author_path"], args["category_path"], args["device_brand_path"], \
                                  args["article_year_path"], args["article_month_path"], args["latent_num"])
      
      # loopup table (convvert id to index)
      item_index_initializer = tf.lookup.TextFileInitializer(args["item_id_path"], key_dtype=tf.string, key_index=tf.lookup.TextFileIndex.WHOLE_LINE, \
                              value_dtype=tf.int64, value_index=tf.lookup.TextFileIndex.LINE_NUMBER, delimiter="\n")
      item_index_table = tf.lookup.StaticVocabularyTable(item_index_initializer, num_oov_buckets=1)
      
      
      # loss function and optimizers
      loss_object = tf.keras.losses.SparseCategoricalCrossentropy()
      optimizer = tf.keras.optimizers.Adam(learning_rate = 0.001)
      
      # loss metrics
      train_loss = tf.keras.metrics.Mean(name='train_loss')
      train_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(name='train_accuracy')
      train_top_10_accuracy = tf.keras.metrics.SparseTopKCategoricalAccuracy(k=10, name='train_top_10_accuracy')
  
      test_loss = tf.keras.metrics.Mean(name='test_loss')
      test_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(name='test_accuracy')
      test_top_10_accuracy = tf.keras.metrics.SparseTopKCategoricalAccuracy(k=10, name='test_top_10_accuracy')
      
      # tensorboard
      current_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
      train_log_dir = './logs/gradient_tape/' + current_time + '/train'
      test_log_dir = './logs/gradient_tape/' + current_time + '/test'
      train_summary_writer = tf.summary.create_file_writer(train_log_dir)
      test_summary_writer = tf.summary.create_file_writer(test_log_dir)
      
      @tf.function
      def train_step(features, labels):
          """ Concrete function for train setp and update train metircs
  
          Args:
              features (OrderedDict:tf.Tensor): OrderedDict of tensor containing input  features.
              labels (tf.Tensor): labels indicating the next_item id
  
          Returns:
              None
          """
          with tf.GradientTape() as tape:
              predictions = model(features, training=True)
              label_indicies = item_index_table.lookup(labels)
              loss = loss_object(label_indicies, predictions)
          gradients = tape.gradient(loss, model.trainable_variables)
          optimizer.apply_gradients(zip(gradients, model.trainable_variables))
  
          train_loss(loss)
          train_accuracy(label_indicies, predictions)
          train_top_10_accuracy(label_indicies, predictions)
      
      @tf.function
      def test_step(features, labels):
          """ Concrete function for test setp and update test metircs
  
          Args:
              features (OrderedDict:tf.Tensor): OrderedDict of tensor containing input features.
              labels (tf.Tensor): labels indicating the next_item id
  
          Returns:
              None
          """
          predictions = model(features, training=False)
          label_indicies = item_index_table.lookup(labels)
          loss = loss_object(label_indicies, predictions)
  
          test_loss(loss)
          test_accuracy(label_indicies, predictions)
          test_top_10_accuracy(label_indicies, predictions)
      
      # custom train loop
      EPOCHS = args["epochs"]
  
      for epoch in range(EPOCHS):
          train_loss.reset_states()
          train_accuracy.reset_states()
          train_top_10_accuracy.reset_states()
  
          test_loss.reset_states()
          test_accuracy.reset_states()
          test_top_10_accuracy.reset_states()
  
          for features, labels in train_dataset:
              train_step(features, labels)
          with train_summary_writer.as_default():
              tf.summary.scalar('loss', train_loss.result(), step=epoch)
              tf.summary.scalar('accuracy', train_accuracy.result(), step=epoch)
              tf.summary.scalar('top_10_accuracy', train_top_10_accuracy.result(), step=epoch)
  
          for features, labels in test_dataset:
              test_step(features, labels)
          with test_summary_writer.as_default():
              tf.summary.scalar('loss', test_loss.result(), step=epoch)
              tf.summary.scalar('accuracy', test_accuracy.result(), step=epoch)
              tf.summary.scalar('top_10_accuracy', test_top_10_accuracy.result(), step=epoch)
  
          template = 'Epoch {:d}, train[loss: {:.6f}, acc: {:.6f}, top_10_acc: {:.6f}], Test[loss: {:.6f}, acc: {:.6f}, top_10_acc: {:.6f}]'
  
          print(template.format(epoch + 1,
                                train_loss.result(),
                                train_accuracy.result() * 100,
                                train_top_10_accuracy.result() * 100,
  
                                test_loss.result(),
                                test_accuracy.result() * 100,
                                test_top_10_accuracy.result() * 100,
                                ))
      
      # exprot tensorboard log
      if args["save_tb_log_to_bucket"]:
          script = "gsutil cp -r ./logs {}".format(args["bucket_tb_log_path"])
          os.system(script)
      
      # export model
      EXPORT_PATH = os.path.join(args["output_dir"], datetime.datetime.now().strftime("%Y%m%d%H%M%S"))
      tf.saved_model.save(obj=model, export_dir=EXPORT_PATH, signatures={'serving_default': model.my_serve})
code_tokens: def train_and_export_model ( args ) : feature_col = [ 'user_id' , 'item_id' , 'title' , 'author' , 'category' , 'device_brand' , 'article_year' , 'article_month' , 'next_item_id' ] u_latent_col = [ "user_latent_{}" . format ( i ) for i in range ( 2 * args [ "latent_num" ] ) ] i_latent_col = [ "item_latent_{}" . format ( i ) for i in range ( 2 * args [ "latent_num" ] ) ] column_name = feature_col + u_latent_col + i_latent_col label_name = 'next_item_id' defaults = [ 'unknown' ] * len ( feature_col ) + [ 0.0 ] * ( len ( u_latent_col ) + len ( i_latent_col ) ) batch_size = args [ "batch_size" ] train_path = args [ "train_data_path" ] test_path = args [ "test_data_path" ] train_dataset = create_dataset ( train_path , column_name , label_name , defaults , batch_size , True ) test_dataset = create_dataset ( test_path , column_name , label_name , defaults , batch_size , False ) model = Hybrid_Recsys_Model ( args [ "item_id_path" ] , args [ "author_path" ] , args [ "category_path" ] , args [ "device_brand_path" ] , args [ "article_year_path" ] , args [ "article_month_path" ] , args [ "latent_num" ] ) item_index_initializer = tf . lookup . TextFileInitializer ( args [ "item_id_path" ] , key_dtype = tf . string , key_index = tf . lookup . TextFileIndex . WHOLE_LINE , value_dtype = tf . int64 , value_index = tf . lookup . TextFileIndex . LINE_NUMBER , delimiter = "\n" ) item_index_table = tf . lookup . StaticVocabularyTable ( item_index_initializer , num_oov_buckets = 1 ) loss_object = tf . keras . losses . SparseCategoricalCrossentropy ( ) optimizer = tf . keras . optimizers . Adam ( learning_rate = 0.001 ) train_loss = tf . keras . metrics . Mean ( name = 'train_loss' ) train_accuracy = tf . keras . metrics . SparseCategoricalAccuracy ( name = 'train_accuracy' ) train_top_10_accuracy = tf . keras . metrics . SparseTopKCategoricalAccuracy ( k = 10 , name = 'train_top_10_accuracy' ) test_loss = tf . keras . metrics . Mean ( name = 'test_loss' ) test_accuracy = tf . keras . metrics . SparseCategoricalAccuracy ( name = 'test_accuracy' ) test_top_10_accuracy = tf . keras . metrics . SparseTopKCategoricalAccuracy ( k = 10 , name = 'test_top_10_accuracy' ) current_time = datetime . datetime . now ( ) . strftime ( "%Y%m%d-%H%M%S" ) train_log_dir = './logs/gradient_tape/' + current_time + '/train' test_log_dir = './logs/gradient_tape/' + current_time + '/test' train_summary_writer = tf . summary . create_file_writer ( train_log_dir ) test_summary_writer = tf . summary . create_file_writer ( test_log_dir ) @ tf . function def train_step ( features , labels ) : """ Concrete function for train setp and update train metircs
  
          Args:
              features (OrderedDict:tf.Tensor): OrderedDict of tensor containing input  features.
              labels (tf.Tensor): labels indicating the next_item id
  
          Returns:
              None
          """ with tf . GradientTape ( ) as tape : predictions = model ( features , training = True ) label_indicies = item_index_table . lookup ( labels ) loss = loss_object ( label_indicies , predictions ) gradients = tape . gradient ( loss , model . trainable_variables ) optimizer . apply_gradients ( zip ( gradients , model . trainable_variables ) ) train_loss ( loss ) train_accuracy ( label_indicies , predictions ) train_top_10_accuracy ( label_indicies , predictions ) @ tf . function def test_step ( features , labels ) : """ Concrete function for test setp and update test metircs
  
          Args:
              features (OrderedDict:tf.Tensor): OrderedDict of tensor containing input features.
              labels (tf.Tensor): labels indicating the next_item id
  
          Returns:
              None
          """ predictions = model ( features , training = False ) label_indicies = item_index_table . lookup ( labels ) loss = loss_object ( label_indicies , predictions ) test_loss ( loss ) test_accuracy ( label_indicies , predictions ) test_top_10_accuracy ( label_indicies , predictions ) EPOCHS = args [ "epochs" ] for epoch in range ( EPOCHS ) : train_loss . reset_states ( ) train_accuracy . reset_states ( ) train_top_10_accuracy . reset_states ( ) test_loss . reset_states ( ) test_accuracy . reset_states ( ) test_top_10_accuracy . reset_states ( ) for features , labels in train_dataset : train_step ( features , labels ) with train_summary_writer . as_default ( ) : tf . summary . scalar ( 'loss' , train_loss . result ( ) , step = epoch ) tf . summary . scalar ( 'accuracy' , train_accuracy . result ( ) , step = epoch ) tf . summary . scalar ( 'top_10_accuracy' , train_top_10_accuracy . result ( ) , step = epoch ) for features , labels in test_dataset : test_step ( features , labels ) with test_summary_writer . as_default ( ) : tf . summary . scalar ( 'loss' , test_loss . result ( ) , step = epoch ) tf . summary . scalar ( 'accuracy' , test_accuracy . result ( ) , step = epoch ) tf . summary . scalar ( 'top_10_accuracy' , test_top_10_accuracy . result ( ) , step = epoch ) template = 'Epoch {:d}, train[loss: {:.6f}, acc: {:.6f}, top_10_acc: {:.6f}], Test[loss: {:.6f}, acc: {:.6f}, top_10_acc: {:.6f}]' print ( template . format ( epoch + 1 , train_loss . result ( ) , train_accuracy . result ( ) * 100 , train_top_10_accuracy . result ( ) * 100 , test_loss . result ( ) , test_accuracy . result ( ) * 100 , test_top_10_accuracy . result ( ) * 100 , ) ) if args [ "save_tb_log_to_bucket" ] : script = "gsutil cp -r ./logs {}" . format ( args [ "bucket_tb_log_path" ] ) os . system ( script ) EXPORT_PATH = os . path . join ( args [ "output_dir" ] , datetime . datetime . now ( ) . strftime ( "%Y%m%d%H%M%S" ) ) tf . saved_model . save ( obj = model , export_dir = EXPORT_PATH , signatures = { 'serving_default' : model . my_serve } )
short_docstring: Train the Hybrid_Recsys_Model and export model to bucket.
short_docstring_tokens: Train the Hybrid_Recsys_Model and export model to bucket .
comment: """ Train the Hybrid_Recsys_Model and export model to bucket.
  
      Args:
          args (dict): dict of arguments from task.py
  
      Returns:
          None
      """$SEP$# create dataset$SEP$# create model$SEP$# loopup table (convvert id to index)$SEP$# loss function and optimizers$SEP$# loss metrics$SEP$# tensorboard$SEP$""" Concrete function for train setp and update train metircs
  
          Args:
              features (OrderedDict:tf.Tensor): OrderedDict of tensor containing input  features.
              labels (tf.Tensor): labels indicating the next_item id
  
          Returns:
              None
          """$SEP$""" Concrete function for test setp and update test metircs
  
          Args:
              features (OrderedDict:tf.Tensor): OrderedDict of tensor containing input features.
              labels (tf.Tensor): labels indicating the next_item id
  
          Returns:
              None
          """$SEP$# custom train loop$SEP$# exprot tensorboard log$SEP$# export model