original_docstring: calculate the matric scores for the fine tuned model
         given the validation set
  
      Args:
          data (dict): validation set
          metric (Metric): supported metric
          model (AbstractiveSummarizer): fine tuned model
          base_model (Union[None, AbstractiveSummarizer]): baseline model
          out_dir (str): store output
      
docstring: calculate the matric scores for the fine tuned model
  given the validation set
docstring_tokens: calculate the matric scores for the fine tuned model given the validation set
code: def calculate_scores(
          data: dict,
          metric: Metric,
          model: AbstractiveSummarizer,
          base_model: Union[None, AbstractiveSummarizer],
          out_dir: str):
      """calculate the matric scores for the fine tuned model
         given the validation set
  
      Args:
          data (dict): validation set
          metric (Metric): supported metric
          model (AbstractiveSummarizer): fine tuned model
          base_model (Union[None, AbstractiveSummarizer]): baseline model
          out_dir (str): store output
      """
      # calculate predictions
      device = torch.device(
          'cuda') if torch.cuda.is_available() else torch.device('cpu')
      source_texts = model.tokenizer.batch_decode(
          data['source']['input_ids'].to(device))
      predicted_summaries = model.predict(data['source'])
      target_summaries = model.tokenizer.batch_decode(
          data['target']['input_ids'].to(device))
  
      # only if base model given
      if base_model:
          base_predictions = base_model.predict(data['source'])
          base_scores = list()
  
      gold_scores = list()
      fine_tuned_scores = list()
      similarity_scores = list()
  
      # calculate metrics
      for i, (source_text, predicted_summary, target_summary) in \
              enumerate(zip(source_texts, predicted_summaries, target_summaries)):
  
          gold_prediction_score = metric.get_score(target_summary, source_text)
          fine_tuned_prediction_score = metric.get_score(
              predicted_summary, source_text)
          summary_similarity_score = metric.get_score(
              predicted_summary, target_summary)
  
          gold_scores.append(gold_prediction_score)
          fine_tuned_scores.append(fine_tuned_prediction_score)
          similarity_scores.append(summary_similarity_score)
  
          if base_model:
              base_prediction_score = metric.get_score(
                  base_predictions[i], source_text)
              base_scores.append(base_prediction_score)
  
      # create dict to prepare
      # data for excel/csv output
      overview_dict = {
          "text": source_texts,
          "target_summary": target_summaries,
          "fine_tuned_summary": predicted_summaries
      }
      if base_model:
          overview_dict["base_model_summary"] = base_predictions
  
      overview_dict.update(
          {
              "gold_score": gold_scores,
              "fine_tuned_score": fine_tuned_scores,
              "summary_similarity_score": similarity_scores
          }
      )
  
      if base_model:
          overview_dict["base_model_score"] = base_scores
  
      # produce overview as excel or csv
      write_table(overview_dict, out_dir, "Overview", "excel")
code_tokens: def calculate_scores ( data : dict , metric : Metric , model : AbstractiveSummarizer , base_model : Union [ None , AbstractiveSummarizer ] , out_dir : str ) : device = torch . device ( 'cuda' ) if torch . cuda . is_available ( ) else torch . device ( 'cpu' ) source_texts = model . tokenizer . batch_decode ( data [ 'source' ] [ 'input_ids' ] . to ( device ) ) predicted_summaries = model . predict ( data [ 'source' ] ) target_summaries = model . tokenizer . batch_decode ( data [ 'target' ] [ 'input_ids' ] . to ( device ) ) if base_model : base_predictions = base_model . predict ( data [ 'source' ] ) base_scores = list ( ) gold_scores = list ( ) fine_tuned_scores = list ( ) similarity_scores = list ( ) for i , ( source_text , predicted_summary , target_summary ) in enumerate ( zip ( source_texts , predicted_summaries , target_summaries ) ) : gold_prediction_score = metric . get_score ( target_summary , source_text ) fine_tuned_prediction_score = metric . get_score ( predicted_summary , source_text ) summary_similarity_score = metric . get_score ( predicted_summary , target_summary ) gold_scores . append ( gold_prediction_score ) fine_tuned_scores . append ( fine_tuned_prediction_score ) similarity_scores . append ( summary_similarity_score ) if base_model : base_prediction_score = metric . get_score ( base_predictions [ i ] , source_text ) base_scores . append ( base_prediction_score ) overview_dict = { "text" : source_texts , "target_summary" : target_summaries , "fine_tuned_summary" : predicted_summaries } if base_model : overview_dict [ "base_model_summary" ] = base_predictions overview_dict . update ( { "gold_score" : gold_scores , "fine_tuned_score" : fine_tuned_scores , "summary_similarity_score" : similarity_scores } ) if base_model : overview_dict [ "base_model_score" ] = base_scores write_table ( overview_dict , out_dir , "Overview" , "excel" )
short_docstring: calculate the matric scores for the fine tuned model
  given the validation set
short_docstring_tokens: calculate the matric scores for the fine tuned model given the validation set
comment: """calculate the matric scores for the fine tuned model
         given the validation set
  
      Args:
          data (dict): validation set
          metric (Metric): supported metric
          model (AbstractiveSummarizer): fine tuned model
          base_model (Union[None, AbstractiveSummarizer]): baseline model
          out_dir (str): store output
      """$SEP$# calculate predictions$SEP$# only if base model given$SEP$# calculate metrics$SEP$# create dict to prepare$SEP$# data for excel/csv output$SEP$# produce overview as excel or csv