hphp/runtime/ext/ext_icu.cpp
/* Returns a list of tokens, but with various normalizations performed
 * based on the token type.
 *
 * Default behavior:
 * Whitespace: dropped (removed from output)
 * Words: converted to lower case
 * Numbers: replaced with #XXX, where the number of X's is based on the
 *          format of the number; any punctuation is maintained
 * Japanese/Chinese scripts: converted to lower case
 * Email: Converted to TOKEN_EMAIL
 * URL: Converted to TOKEN_URL
 * Emoticon: Left as-is
 * Heart: Converted to TOKEN_HEART
 * Exclamation: Replaced with an empty string
 * Date: Replaced with TOKEN_DATE
 * Money: Replaced with TOKEN_MONEY
 * Time: Replaced with TOKEN_TIME
 * Acronym: converted to lower case
 * Other: replaced with empty string
 *
 */
Returns a list of tokens, but with various normalizations performed
based on the token type.


Returns a list of tokens but with various normalizations performed based on the token type .
Array f_icu_tokenize(const String& text) {
  const String BEGIN_MARKER("_B_");
  const String END_MARKER("_E_");
  Array ret;
  std::vector<Token> tokens;
  tokenizeString(tokens, getMaster(), UnicodeString::fromUTF8(text.data()));
  int i = 0;
  ret.set(i++, BEGIN_MARKER);
  for(std::vector<Token>::iterator iter = tokens.begin();
      iter != tokens.end();
      iter++) {
    normalizeToken(*iter);
    const UnicodeString& word = iter->value;
    if(!s_spaceMatcher->matches(word) && word.length() > 0) {
      ret.set(i++, String(icuStringToUTF8(word)));
    }
  }
  ret.set(i++, END_MARKER);
  return ret;
}
Array f_icu_tokenize ( const String & text ) { const String BEGIN_MARKER ( " " ) ; const String END_MARKER ( " " ) ; Array ret ; std :: vector < Token > tokens ; tokenizeString ( tokens , getMaster ( ) , UnicodeString :: fromUTF8 ( text . data ( ) ) ) ; int i = 0 ; ret . set ( i ++ , BEGIN_MARKER ) ; for ( std :: vector < Token > :: iterator iter = tokens . begin ( ) ; iter != tokens . end ( ) ; iter ++ ) { normalizeToken ( * iter ) ; const UnicodeString & word = iter -> value ; if ( ! s_spaceMatcher -> matches ( word ) && word . length ( ) > 0 ) { ret . set ( i ++ , String ( icuStringToUTF8 ( word ) ) ) ; } } ret . set ( i ++ , END_MARKER ) ; return ret ; }
Returns a list of tokens, but with various normalizations performed
 based on the token type.
Returns a list of tokens but with various normalizations performed based on the token type .
// Boundary markers that indicate the beginning and end of a token stream. // Ignore spaces and empty strings.