src/Generic/common/UnicodeUtil.cpp
//Boost supports unicode, and so chinese hieroglyphs and arabic letters will 
// be correctly recognized as [:alnum:].  Note that we pass the argument by
// value -- i.e., we start the function with a *copy* of the string that was
// passed in.

Boost supports unicode, and so chinese hieroglyphs and arabic letters will
be correctly recognized as [:alnum:].  Note that we pass the argument by
value -- i.e., we start the function with a *copy* of the string that was
passed in.
Boost supports unicode and so chinese hieroglyphs and arabic letters will be correctly recognized as [ : alnum : ] . Note that we pass the argument by value -- i . e . we start the function with a * copy * of the string that was passed in .
std::wstring UnicodeUtil::normalizeTextString(std::wstring str) {
	static const boost::wregex escapedPunct(L"(-[LR][RC]B-)|(&(amp;)+)");
	static const boost::wregex punct(L"[^[:alnum:]]+");
	static const boost::wregex whitespace(L"[\\t ]+");
	str = boost::regex_replace(str, escapedPunct, L" ", boost::match_default | boost::format_sed);
	str = boost::regex_replace(str, punct, L" ", boost::match_default | boost::format_sed);
	str = boost::regex_replace(str, whitespace, L" ", boost::match_default | boost::format_sed);
	boost::trim(str);
	std::transform(str.begin(), str.end(), str.begin(), std::towlower);
	return str;
}
std :: wstring UnicodeUtil :: normalizeTextString ( std :: wstring str ) { static const boost :: wregex escapedPunct ( L" " ) ; static const boost :: wregex punct ( L" " ) ; static const boost :: wregex whitespace ( L" \\ " ) ; str = boost :: regex_replace ( str , escapedPunct , L" " , boost :: match_default | boost :: format_sed ) ; str = boost :: regex_replace ( str , punct , L" " , boost :: match_default | boost :: format_sed ) ; str = boost :: regex_replace ( str , whitespace , L" " , boost :: match_default | boost :: format_sed ) ; boost :: trim ( str ) ; std :: transform ( str . begin ( ) , str . end ( ) , str . begin ( ) , std :: towlower ) ; return str ; }
Boost supports unicode, and so chinese hieroglyphs and arabic letters will
be correctly recognized as [:alnum:].
Boost supports unicode and so chinese hieroglyphs and arabic letters will be correctly recognized as [ : alnum : ] .
// regexps used by this method.  These are static so we don't have to
 // re-generate them each time this method is called.
 //replace special tokens by spaces
 //replace non-alphanumerics by spaces
 //multiple spaces ==> one ' '
 //trim leading and trailing spaces
 //remove capitalization
