@Override public void tokenize(String s){
  m_N=m_NMax;
  m_SplitString=s.split("[" + getDelimiters() + "]");
  filterOutEmptyStrings();
  m_CurrentPosition=0;
  m_MaxPosition=m_SplitString.length;
  if (m_SplitString.length < m_NMax) {
    m_N=m_SplitString.length;
  }
}

@ Override public void tokenize ( String s ) { m_N = m_NMax ; m_SplitString = s . split ( "[" + getDelimiters ( ) + "]" ) ; filterOutEmptyStrings ( ) ; m_CurrentPosition = 0 ; m_MaxPosition = m_SplitString . length ; if ( m_SplitString . length < m_NMax ) { m_N = m_SplitString . length ; } }
sets the string to tokenize . tokenization happens immediately .
