public static Span[] split(String d) {
    int tokStart = -1;
    List tokens = new ArrayList();
    boolean inTok = false;

    //gather up potential tokens
    int end = d.length();
    for (int i = 0; i < end; i++) {
      if (Character.isWhitespace(d.charAt(i))) {
        if (inTok) {
          tokens.add(new Span(tokStart, i));
          inTok = false;
          tokStart = -1;
        }
      }
      else {
        if (!inTok) {
          tokStart = i;
          inTok = true;
        }
      }
    }
    if (inTok) {
      tokens.add(new Span(tokStart, end));
    }
    return (Span[]) tokens.toArray(new Span[tokens.size()]);
  }
public static Span [ ] split ( String d ) { int tokStart = - 1 ; List tokens = new ArrayList ( ) ; boolean inTok = false ; int end = d . length ( ) ; for ( int i = 0 ; i < end ; i ++ ) { if ( Character . isWhitespace ( d . charAt ( i ) ) ) { if ( inTok ) { tokens . add ( new Span ( tokStart , i ) ) ; inTok = false ; tokStart = - 1 ; } } else { if ( ! inTok ) { tokStart = i ; inTok = true ; } } } if ( inTok ) { tokens . add ( new Span ( tokStart , end ) ) ; } return ( Span [ ] ) tokens . toArray ( new Span [ tokens . size ( ) ] ) ; }
Constructs a list of Span objects, one for each whitespace
delimited token. Token strings can be constructed form these
spans as follows: d.substring(span.getStart(),span.getEnd());
@param d string to tokenize.
@return List is spans.