src/dataconvert/ConvertASR.cc
// Compute the BLEU loss given the candidate.
Compute the BLEU loss given the candidate.
Compute the BLEU loss given the candidate .
bool ConvertASR::ComputeBLEU(const string& lstring,
                             confusion_learning::CandidateMessage* cand) {
  CDEBUG(4, "Processing loss: " << lstring);
  double overlap[4];
  double ref_len = numeric_limits<double>::min();
  stringstream loss_stream(lstring);
  loss_stream >> ws >> overlap[0];
  loss_stream >> ws >> overlap[1];
  loss_stream >> ws >> overlap[2];
  loss_stream >> ws >> overlap[3];
  loss_stream >> ws >> ref_len >> ws;
  if (ref_len == numeric_limits<double>::min()) {
    cerr << "BLEU loss requires 7 fields in input file" << endl;
    return false;
  }
  const string& candidate = cand->raw_data();
  size_t cur_pos = 0;
  int cand_len = 0;
  while (cur_pos != string::npos) {
    cur_pos = candidate.find_first_of(SPACE_DELIM, cur_pos + 1);
    if (cur_pos != string::npos) {
      cur_pos = candidate.find_first_not_of(SPACE_DELIM, cur_pos + 1);
    }
    cand_len++;
  }
  CDEBUG(5, "Reference length: " << ref_len);
  CDEBUG(5, "Candidate length: " << cand_len);
  double smooth = 1.0;
  double loss = 0.0;
  for (int order = 0; order < 4; order++) {
    double ngorder = overlap[order];
    if (ngorder == 0) {  
      smooth *= 0.5;
      ngorder = smooth;
    }
    if (cand_len > order) {
      loss += log(ngorder / static_cast<double>(cand_len - order)) ;
    }
  }
  loss /= 4.0;
  CDEBUG(5, "  BLEU loss with smoothing: " << loss);
  if (cand_len < ref_len && cand_len > 0) {
    loss += 1.0 - (ref_len / static_cast<double>(cand_len));
    CDEBUG(5, "  After adding brefity penalty: " << loss);
  }
  loss = -exp(loss);
  confusion_learning::ScoreMessage* score_msg = cand->add_score();
  score_msg->set_type(confusion_learning::ScoreMessage::LOSS);
  score_msg->set_score(loss);
  CDEBUG(4, "     Candidate loss: " << loss);
  return true;
}
bool ConvertASR :: ComputeBLEU ( const string & lstring , confusion_learning :: CandidateMessage * cand ) { CDEBUG ( 4 , " " << lstring ) ; double overlap [ 4 ] ; double ref_len = numeric_limits < double > :: min ( ) ; stringstream loss_stream ( lstring ) ; loss_stream >> ws >> overlap [ 0 ] ; loss_stream >> ws >> overlap [ 1 ] ; loss_stream >> ws >> overlap [ 2 ] ; loss_stream >> ws >> overlap [ 3 ] ; loss_stream >> ws >> ref_len >> ws ; if ( ref_len == numeric_limits < double > :: min ( ) ) { cerr << " " << endl ; return false ; } const string & candidate = cand -> raw_data ( ) ; size_t cur_pos = 0 ; int cand_len = 0 ; while ( cur_pos != string :: npos ) { cur_pos = candidate . find_first_of ( SPACE_DELIM , cur_pos + 1 ) ; if ( cur_pos != string :: npos ) { cur_pos = candidate . find_first_not_of ( SPACE_DELIM , cur_pos + 1 ) ; } cand_len ++ ; } CDEBUG ( 5 , " " << ref_len ) ; CDEBUG ( 5 , " " << cand_len ) ; double smooth = 1.0 ; double loss = 0.0 ; for ( int order = 0 ; order < 4 ; order ++ ) { double ngorder = overlap [ order ] ; if ( ngorder == 0 ) { smooth *= 0.5 ; ngorder = smooth ; } if ( cand_len > order ) { loss += log ( ngorder / static_cast < double > ( cand_len - order ) ) ; } } loss /= 4.0 ; CDEBUG ( 5 , " " << loss ) ; if ( cand_len < ref_len && cand_len > 0 ) { loss += 1.0 - ( ref_len / static_cast < double > ( cand_len ) ) ; CDEBUG ( 5 , " " << loss ) ; } loss = - exp ( loss ) ; confusion_learning :: ScoreMessage * score_msg = cand -> add_score ( ) ; score_msg -> set_type ( confusion_learning :: ScoreMessage :: LOSS ) ; score_msg -> set_score ( loss ) ; CDEBUG ( 4 , " " << loss ) ; return true ; }
Compute the BLEU loss given the candidate.
Compute the BLEU loss given the candidate .
// Compute the length of the candidate string. // Compute the sentence-level BLEU loss // smooth sentence level bleu // brevity penalty