research/seq_flow_lite/tf_ops/text_distorter.cc
// Distorts the words in the text by inserting, deleting and swapping
// unicodes randomly with probability one third of distortion_probability.
Distorts the words in the text by inserting, deleting and swapping
unicodes randomly with probability one third of distortion_probability.
Distorts the words in the text by inserting deleting and swapping unicodes randomly with probability one third of distortion_probability .
std::string TextDistorter::DistortText(icu::UnicodeString* uword) {
  if (distortion_probability_ > 0.0 &&
      generator_.RandFloat() < distortion_probability_ && uword->length()) {
    float distortion_type = generator_.RandFloat();
    uint32 rindex = generator_.Rand32() % uword->length();
    if (distortion_type < 0.33f) {
      random_char_ = (*uword)[rindex];
      uword->remove(rindex, 1);
    } else if (distortion_type < 0.66f) {
      if (uword->length() > 2) {
        random_char_ = (*uword)[rindex];
        uword->remove(rindex, 1);
        uword->insert(generator_.Rand32() % uword->length(), random_char_);
      }
    } else if (random_char_) {
      uword->insert(rindex, random_char_);
    }
  }
  std::string word;
  icu::StringByteSink<std::string> sink(&word);
  uword->toUTF8(sink);
  return word;
}
std :: string TextDistorter :: DistortText ( icu :: UnicodeString * uword ) { if ( distortion_probability_ > 0.0 && generator_ . RandFloat ( ) < distortion_probability_ && uword -> length ( ) ) { float distortion_type = generator_ . RandFloat ( ) ; uint32 rindex = generator_ . Rand32 ( ) % uword -> length ( ) ; if ( distortion_type < 0.33f ) { random_char_ = ( * uword ) [ rindex ] ; uword -> remove ( rindex , 1 ) ; } else if ( distortion_type < 0.66f ) { if ( uword -> length ( ) > 2 ) { random_char_ = ( * uword ) [ rindex ] ; uword -> remove ( rindex , 1 ) ; uword -> insert ( generator_ . Rand32 ( ) % uword -> length ( ) , random_char_ ) ; } } else if ( random_char_ ) { uword -> insert ( rindex , random_char_ ) ; } } std :: string word ; icu :: StringByteSink < std :: string > sink ( & word ) ; uword -> toUTF8 ( sink ) ; return word ; }
Distorts the words in the text by inserting, deleting and swapping
unicodes randomly with probability one third of distortion_probability.
Distorts the words in the text by inserting deleting and swapping unicodes randomly with probability one third of distortion_probability .
// Distort text with non zero length with distortion_probability_. // Remove character with one third probability. // Swap character with one third probability if there are more than 2 // characters. // Insert character with one third probability. // Convert unicode sequence back to string.