public List<List<Token>> process(final String text) {
        final List<List<Token>> paragraph = new ArrayList<>();
        currentSentence = new ArrayList<>();
        final Tokens tokens = splitText(text);

        while (tokens.hasNext()) {
            final Token t = tokens.next();
            final String trimmedWord = t.text.trim();

            // skip spaces
            if (trimmedWord.isEmpty()) continue;

            if (((mode == WITH_PUNCTUATION) || (mode == WITHOUT_PUNCTUATION && isLetterOrDigit(initChar(t.text))))) {
                boolean canBreakSentence = true;
                if (t.text.contains("'")) {
                    wordContainsApostrophe(t);
                }
                else if (".".equals(trimmedWord)) {
                    canBreakSentence = wordIsFullStop(t);
                }
                else if (":".equals(trimmedWord)) {
                    wordIsColon(tokens, t);
                }
                else currentSentence.add(t);

                // handling the end of a sentence
                if (canBreakSentence && equalss(trimmedWord, ".", ";", "?", "!")) {
                    paragraph.add(currentSentence);
                    currentSentence = new ArrayList<>();
                }
            }
        }

        if (!currentSentence.isEmpty()) paragraph.add(currentSentence);
        return paragraph;
    }
public List < List < Token > > process ( final String text ) { final List < List < Token > > paragraph = new ArrayList < > ( ) ; currentSentence = new ArrayList < > ( ) ; final Tokens tokens = splitText ( text ) ; while ( tokens . hasNext ( ) ) { final Token t = tokens . next ( ) ; final String trimmedWord = t . text . trim ( ) ; if ( trimmedWord . isEmpty ( ) ) continue ; if ( ( ( mode == WITH_PUNCTUATION ) || ( mode == WITHOUT_PUNCTUATION && isLetterOrDigit ( initChar ( t . text ) ) ) ) ) { boolean canBreakSentence = true ; if ( t . text . contains ( "'" ) ) { wordContainsApostrophe ( t ) ; } else if ( "." . equals ( trimmedWord ) ) { canBreakSentence = wordIsFullStop ( t ) ; } else if ( ":" . equals ( trimmedWord ) ) { wordIsColon ( tokens , t ) ; } else currentSentence . add ( t ) ; if ( canBreakSentence && equalss ( trimmedWord , "." , ";" , "?" , "!" ) ) { paragraph . add ( currentSentence ) ; currentSentence = new ArrayList < > ( ) ; } } } if ( ! currentSentence . isEmpty ( ) ) paragraph . add ( currentSentence ) ; return paragraph ; }
Tokenize some text - not thread safe.
@param text tokenize this
@return tokenized text