private List<TokenType> extractStringLiteralDelimitingTokens(String[] tokens){
  List<TokenType> delimitingTokens=new ArrayList<TokenType>();
  for (  String token : tokens) {
    String cleanToken=cleanToken(token);
    boolean handled=false;
    if (alternateQuote == null) {
      String alternateQuoteFromToken=extractAlternateOpenQuote(cleanToken);
      if (alternateQuoteFromToken != null) {
        String closeQuote=computeAlternateCloseQuote(alternateQuoteFromToken);
        if (cleanToken.length() >= (alternateQuoteFromToken.length() + closeQuote.length()) && cleanToken.startsWith(alternateQuoteFromToken) && cleanToken.endsWith(closeQuote)) {
          continue;
        }
        alternateQuote=closeQuote;
        delimitingTokens.add(TokenType.ALTERNATE_QUOTE);
        continue;
      }
    }
    if ((alternateQuote != null) && cleanToken.endsWith(alternateQuote)) {
      alternateQuote=null;
      delimitingTokens.add(TokenType.ALTERNATE_QUOTE);
      continue;
    }
    if ((cleanToken.length() >= 2) && cleanToken.startsWith("'") && cleanToken.endsWith("'")) {
      continue;
    }
    if ((cleanToken.length() >= 4) && cleanToken.startsWith("/*") && cleanToken.endsWith("*/")) {
      continue;
    }
    if (isSingleLineComment(cleanToken)) {
      delimitingTokens.add(TokenType.SINGLE_LINE_COMMENT);
      handled=true;
    }
    if (cleanToken.startsWith("/*")) {
      delimitingTokens.add(TokenType.MULTI_LINE_COMMENT);
      handled=true;
    }
 else     if (cleanToken.startsWith("'")) {
      delimitingTokens.add(TokenType.QUOTE);
      handled=true;
    }
    if (!cleanToken.startsWith("/*") && cleanToken.endsWith("*/")) {
      delimitingTokens.add(TokenType.MULTI_LINE_COMMENT);
      handled=true;
    }
 else     if (!cleanToken.startsWith("'") && cleanToken.endsWith("'")) {
      delimitingTokens.add(TokenType.QUOTE);
      handled=true;
    }
    if (!handled) {
      delimitingTokens.add(TokenType.OTHER);
    }
  }
  return delimitingTokens;
}

private List < TokenType > extractStringLiteralDelimitingTokens ( String [ ] tokens ) { List < TokenType > delimitingTokens = new ArrayList < TokenType > ( ) ; for ( String token : tokens ) { String cleanToken = cleanToken ( token ) ; boolean handled = false ; if ( alternateQuote == null ) { String alternateQuoteFromToken = extractAlternateOpenQuote ( cleanToken ) ; if ( alternateQuoteFromToken != null ) { String closeQuote = computeAlternateCloseQuote ( alternateQuoteFromToken ) ; if ( cleanToken . length ( ) >= ( alternateQuoteFromToken . length ( ) + closeQuote . length ( ) ) && cleanToken . startsWith ( alternateQuoteFromToken ) && cleanToken . endsWith ( closeQuote ) ) { continue ; } alternateQuote = closeQuote ; delimitingTokens . add ( TokenType . ALTERNATE_QUOTE ) ; continue ; } } if ( ( alternateQuote != null ) && cleanToken . endsWith ( alternateQuote ) ) { alternateQuote = null ; delimitingTokens . add ( TokenType . ALTERNATE_QUOTE ) ; continue ; } if ( ( cleanToken . length ( ) >= 2 ) && cleanToken . startsWith ( "'" ) && cleanToken . endsWith ( "'" ) ) { continue ; } if ( ( cleanToken . length ( ) >= 4 ) && cleanToken . startsWith ( "/*" ) && cleanToken . endsWith ( "*/" ) ) { continue ; } if ( isSingleLineComment ( cleanToken ) ) { delimitingTokens . add ( TokenType . SINGLE_LINE_COMMENT ) ; handled = true ; } if ( cleanToken . startsWith ( "/*" ) ) { delimitingTokens . add ( TokenType . MULTI_LINE_COMMENT ) ; handled = true ; } else if ( cleanToken . startsWith ( "'" ) ) { delimitingTokens . add ( TokenType . QUOTE ) ; handled = true ; } if ( ! cleanToken . startsWith ( "/*" ) && cleanToken . endsWith ( "*/" ) ) { delimitingTokens . add ( TokenType . MULTI_LINE_COMMENT ) ; handled = true ; } else if ( ! cleanToken . startsWith ( "'" ) && cleanToken . endsWith ( "'" ) ) { delimitingTokens . add ( TokenType . QUOTE ) ; handled = true ; } if ( ! handled ) { delimitingTokens . add ( TokenType . OTHER ) ; } } return delimitingTokens ; }
extract the type of all tokens that potentially delimit string literals .
