protected void initTokenizer(){
  m_Tokenizer.resetSyntax();
  m_Tokenizer.whitespaceChars(0,' ');
  m_Tokenizer.wordChars(' ' + 1,'\u00FF');
  if (m_fieldSeparator != null) {
    m_Tokenizer.whitespaceChars(m_fieldSeparator.charAt(0),m_fieldSeparator.charAt(0));
  }
 else {
    m_Tokenizer.whitespaceChars(',',',');
  }
  m_Tokenizer.commentChar('%');
  if (m_enclosures != null && m_enclosures.size() > 0) {
    for (    String e : m_enclosures) {
      m_Tokenizer.quoteChar(e.charAt(0));
    }
  }
 else {
    m_Tokenizer.quoteChar('"');
    m_Tokenizer.quoteChar('\'');
  }
  m_Tokenizer.ordinaryChar('{');
  m_Tokenizer.ordinaryChar('}');
  m_Tokenizer.eolIsSignificant(true);
}

protected void initTokenizer ( ) { m_Tokenizer . resetSyntax ( ) ; m_Tokenizer . whitespaceChars ( 0 , ' ' ) ; m_Tokenizer . wordChars ( ' ' + 1 , 'ÿ' ) ; if ( m_fieldSeparator != null ) { m_Tokenizer . whitespaceChars ( m_fieldSeparator . charAt ( 0 ) , m_fieldSeparator . charAt ( 0 ) ) ; } else { m_Tokenizer . whitespaceChars ( ',' , ',' ) ; } m_Tokenizer . commentChar ( '%' ) ; if ( m_enclosures != null && m_enclosures . size ( ) > 0 ) { for ( String e : m_enclosures ) { m_Tokenizer . quoteChar ( e . charAt ( 0 ) ) ; } } else { m_Tokenizer . quoteChar ( '"' ) ; m_Tokenizer . quoteChar ( '\'' ) ; } m_Tokenizer . ordinaryChar ( '{' ) ; m_Tokenizer . ordinaryChar ( '}' ) ; m_Tokenizer . eolIsSignificant ( true ) ; }
initializes the streamtokenizer used for reading the arff file .
