src/English/tokens/en_Tokenizer.cpp
//Replace common emoticons and symbols with -EMOT- and -SYM- respectively
//Information about the specific emoticon is lost
//This means that tokenization will not split emoticon into multiple tokens, which is good

Replace common emoticons and symbols with -EMOT- and -SYM- respectively
Information about the specific emoticon is lost
This means that tokenization will not split emoticon into multiple tokens, which is good
Replace common emoticons and symbols with - EMOT - and - SYM - respectively Information about the specific emoticon is lost This means that tokenization will not split emoticon into multiple tokens which is good
void EnglishTokenizer::replaceEmoticons(LocatedString *string){
	static const boost::wregex emoticons(
		L"[:;=]" 
		L"-?" 
		L"[dDpPoOvVsS\\)\\]\\}\\(\\[\\{]" 
		);
	static const boost::wregex symbols(
		L"(&lt;/?3)" 
		);
	std::wstring as_wstring = string->substringAsWString(0, string->length());
	boost::wsregex_iterator next_emo(as_wstring.begin(), as_wstring.end(), emoticons);
	boost::wsregex_iterator end_emo;
	while (next_emo != end_emo) {
		boost::wsmatch match = *next_emo;
		string->replace(match.str(), L" -EMOT- ");
		next_emo++;
	}
	boost::wsregex_iterator next_sym(as_wstring.begin(), as_wstring.end(), symbols);
	boost::wsregex_iterator end_sym;
	while (next_sym != end_sym) {
		boost::wsmatch match = *next_sym;
		string->replace(match.str(), L" -SYMB- ");
		next_sym++;
	}
}
void EnglishTokenizer :: replaceEmoticons ( LocatedString * string ) { static const boost :: wregex emoticons ( L" " L" " L" \\ \\ \\ \\ \\ \\ " ) ; static const boost :: wregex symbols ( L" " ) ; std :: wstring as_wstring = string -> substringAsWString ( 0 , string -> length ( ) ) ; boost :: wsregex_iterator next_emo ( as_wstring . begin ( ) , as_wstring . end ( ) , emoticons ) ; boost :: wsregex_iterator end_emo ; while ( next_emo != end_emo ) { boost :: wsmatch match = * next_emo ; string -> replace ( match . str ( ) , L" " ) ; next_emo ++ ; } boost :: wsregex_iterator next_sym ( as_wstring . begin ( ) , as_wstring . end ( ) , symbols ) ; boost :: wsregex_iterator end_sym ; while ( next_sym != end_sym ) { boost :: wsmatch match = * next_sym ; string -> replace ( match . str ( ) , L" " ) ; next_sym ++ ; } }
Replace common emoticons and symbols with -EMOT- and -SYM- respectively
Information about the specific emoticon is lost
This means that tokenization will not split emoticon into multiple tokens, which is good
Replace common emoticons and symbols with - EMOT - and - SYM - respectively Information about the specific emoticon is lost This means that tokenization will not split emoticon into multiple tokens which is good
//eyes
 //optional nose
 //mouth
 //heart (i.e. <3)
