public void tokenizeToKAF(final KAFDocument kaf) throws IOException {

    int noSents = 0;
    int noParas = 1;
    
    List<List<Token>> tokens;

    if (isNoSeg) {
      String[] sentences = text.toArray(new String[text.size()]);
      tokens = tokenizer.tokenize(sentences);
    } else {
      final String[] sentences = segmenter.segmentSentence();
      tokens = tokenizer.tokenize(sentences);
    }
    for (final List<Token> tokenizedSentence : tokens) {
      noSents = noSents + 1;
      for (final Token token : tokenizedSentence) {
        if (token.getTokenValue().equals(RuleBasedSegmenter.PARAGRAPH)) {
          ++noParas;
          // TODO debug this
          if (noSents < noParas) {
            ++noSents;
          }
        } else {
          final WF wf = kaf.newWF(token.startOffset(), token.getTokenValue(),
              noSents);
          wf.setLength(token.tokenLength());
          wf.setPara(noParas);
        }
      }
    }
  }
public void tokenizeToKAF ( final KAFDocument kaf ) throws IOException { int noSents = 0 ; int noParas = 1 ; List < List < Token > > tokens ; if ( isNoSeg ) { String [ ] sentences = text . toArray ( new String [ text . size ( ) ] ) ; tokens = tokenizer . tokenize ( sentences ) ; } else { final String [ ] sentences = segmenter . segmentSentence ( ) ; tokens = tokenizer . tokenize ( sentences ) ; } for ( final List < Token > tokenizedSentence : tokens ) { noSents = noSents + 1 ; for ( final Token token : tokenizedSentence ) { if ( token . getTokenValue ( ) . equals ( RuleBasedSegmenter . PARAGRAPH ) ) { ++ noParas ; if ( noSents < noParas ) { ++ noSents ; } } else { final WF wf = kaf . newWF ( token . startOffset ( ) , token . getTokenValue ( ) , noSents ) ; wf . setLength ( token . tokenLength ( ) ) ; wf . setPara ( noParas ) ; } } } }
Tokenize document to NAF.

@param kaf
the incoming naf document
@throws IOException
if io problems