public static List<String> tokenize(String arguments,boolean stringify){
  LinkedList<String> argList=new LinkedList<String>();
  StringBuilder currArg=new StringBuilder();
  boolean escaped=false;
  int state=NO_TOKEN_STATE;
  int len=arguments.length();
  for (int i=0; i < len; i++) {
    char c=arguments.charAt(i);
    if (escaped) {
      escaped=false;
      currArg.append(c);
    }
 else {
switch (state) {
case SINGLE_QUOTE_STATE:
        if (c == '\'') {
          state=NORMAL_TOKEN_STATE;
        }
 else {
          currArg.append(c);
        }
      break;
case DOUBLE_QUOTE_STATE:
    if (c == '"') {
      state=NORMAL_TOKEN_STATE;
    }
 else     if (c == '\\') {
      i++;
      char next=arguments.charAt(i);
      if (next == '"' || next == '\\') {
        currArg.append(next);
      }
 else {
        currArg.append(c);
        currArg.append(next);
      }
    }
 else {
      currArg.append(c);
    }
  break;
case NO_TOKEN_STATE:
case NORMAL_TOKEN_STATE:
switch (c) {
case '\\':
  escaped=true;
state=NORMAL_TOKEN_STATE;
break;
case '\'':
state=SINGLE_QUOTE_STATE;
break;
case '"':
state=DOUBLE_QUOTE_STATE;
break;
default :
if (!Character.isWhitespace(c)) {
currArg.append(c);
state=NORMAL_TOKEN_STATE;
}
 else if (state == NORMAL_TOKEN_STATE) {
argList.add(currArg.toString());
currArg=new StringBuilder();
state=NO_TOKEN_STATE;
}
}
break;
default :
throw new IllegalStateException("ArgumentTokenizer state " + state + " is invalid!");
}
}
}
if (escaped) {
currArg.append('\\');
argList.add(currArg.toString());
}
 else if (state != NO_TOKEN_STATE) {
argList.add(currArg.toString());
}
if (stringify) {
for (int i=0; i < argList.size(); i++) {
argList.set(i,"\"" + _escapeQuotesAndBackslashes(argList.get(i)) + "\"");
}
}
return argList;
}

public static List < String > tokenize ( String arguments , boolean stringify ) { LinkedList < String > argList = new LinkedList < String > ( ) ; StringBuilder currArg = new StringBuilder ( ) ; boolean escaped = false ; int state = NO_TOKEN_STATE ; int len = arguments . length ( ) ; for ( int i = 0 ; i < len ; i ++ ) { char c = arguments . charAt ( i ) ; if ( escaped ) { escaped = false ; currArg . append ( c ) ; } else { switch ( state ) { case SINGLE_QUOTE_STATE : if ( c == '\'' ) { state = NORMAL_TOKEN_STATE ; } else { currArg . append ( c ) ; } break ; case DOUBLE_QUOTE_STATE : if ( c == '"' ) { state = NORMAL_TOKEN_STATE ; } else if ( c == '\\' ) { i ++ ; char next = arguments . charAt ( i ) ; if ( next == '"' || next == '\\' ) { currArg . append ( next ) ; } else { currArg . append ( c ) ; currArg . append ( next ) ; } } else { currArg . append ( c ) ; } break ; case NO_TOKEN_STATE : case NORMAL_TOKEN_STATE : switch ( c ) { case '\\' : escaped = true ; state = NORMAL_TOKEN_STATE ; break ; case '\'' : state = SINGLE_QUOTE_STATE ; break ; case '"' : state = DOUBLE_QUOTE_STATE ; break ; default : if ( ! Character . isWhitespace ( c ) ) { currArg . append ( c ) ; state = NORMAL_TOKEN_STATE ; } else if ( state == NORMAL_TOKEN_STATE ) { argList . add ( currArg . toString ( ) ) ; currArg = new StringBuilder ( ) ; state = NO_TOKEN_STATE ; } } break ; default : throw new IllegalStateException ( "ArgumentTokenizer state " + state + " is invalid!" ) ; } } } if ( escaped ) { currArg . append ( '\\' ) ; argList . add ( currArg . toString ( ) ) ; } else if ( state != NO_TOKEN_STATE ) { argList . add ( currArg . toString ( ) ) ; } if ( stringify ) { for ( int i = 0 ; i < argList . size ( ) ; i ++ ) { argList . set ( i , "\"" + _escapeQuotesAndBackslashes ( argList . get ( i ) ) + "\"" ) ; } } return argList ; }
tokenizes the given string into string tokens .
