src-gu/mir-reasoner/src/common_classes/SpecialTokenizer.java
/** Tokenizes and adds a given line of input to the token stream as tokens
	 * @param line The line of the input file to tokenize and add
	 */
Tokenizes and adds a given line of input to the token stream as tokens
@param line The line of the input file to tokenize and add
Tokenizes and adds a given line of input to the token stream as tokens @param line The line of the input file to tokenize and add
private void addLineToTokenStream(String[] line) throws InvalidTokenException{
		boolean foundSubjectCandidate = false;
		boolean foundSubjectCandidate2 = false;
		boolean foundSubjectCandidate3 = false;
		boolean foundPredicateCandidate = false;
		boolean foundObjectCandidate = false;
		boolean foundObjectCandidate2 = false;
		boolean foundObjectCandidate3 = false;
		String predToken = "";
		for (int i=0;  i < line.length;  i++){
			String curToken = line[i];
			String nextToken = new String();
			String thirdToken = new String();
			if(i < line.length - 2)
				nextToken = line[i+1];
			if(i < line.length - 3)
				thirdToken = line[i+2];
			curToken = curToken.replace("\t","");
			nextToken = nextToken.replace("\t", "");
			if ( curToken.isEmpty() )
				continue;
			else if (!foundSubjectCandidate){
				foundSubjectCandidate = true;
				if ( isClassToken(curToken) )
					tokenStream.add(new ClassToken(lineNumber, curToken));
				else
					throw new InvalidTokenException("Invalid Subject", lineNumber, curToken);
			}
			else if (!foundSubjectCandidate2 && (isLeftSum(nextToken) || isLeftSum3(thirdToken))){
				foundSubjectCandidate2 = true;
				if( isClassToken(curToken) )
					tokenStream.add(new ClassToken(lineNumber, curToken));
				else
					throw new InvalidTokenException("Invalid Subject 2", lineNumber, curToken);
			}
			else if (!foundSubjectCandidate3 && isLeftSum3(nextToken)){
				foundSubjectCandidate3 = true;
				if(isClassToken(curToken))
					tokenStream.add(new ClassToken(lineNumber, curToken));
				else
					throw new InvalidTokenException("Invalid Subject 3", lineNumber, curToken);
			}
			else if (!foundPredicateCandidate){
				foundPredicateCandidate = true;
				if ( isRelationToken(curToken) || isSum3Token(curToken)){
					tokenStream.add(new RelationToken(lineNumber, curToken));
					predToken = curToken;
				}
				else
					throw new InvalidTokenException("Invalid Predicate", lineNumber, curToken);
			}
			else if (!foundObjectCandidate){
				foundObjectCandidate = true;
				if ( isClassToken(curToken) )
					tokenStream.add(new ClassToken(lineNumber, curToken));
				else
					throw new InvalidTokenException("Invalid Object", lineNumber, curToken);
			}
			else if (!foundObjectCandidate2 && (isRightSum(predToken) || isRightSum3(predToken))){
				foundObjectCandidate2 = true;
				if ( isClassToken(curToken) )
					tokenStream.add(new ClassToken(lineNumber, curToken));
				else
					throw new InvalidTokenException("Invalid Object 2", lineNumber, curToken);
			}
			else if (!foundObjectCandidate3 && isRightSum3(predToken)){
				foundObjectCandidate3 = true;
				if(isClassToken(curToken))
					tokenStream.add(new ClassToken(lineNumber, curToken));
				else
					throw new InvalidTokenException("Invalid Object 3", lineNumber, curToken);
			}
			else
				throw new InvalidTokenException("Too many tokens: ", lineNumber, curToken);
		}
		if (foundSubjectCandidate && !foundObjectCandidate)
			throw new InvalidTokenException("Too few tokens: ", lineNumber, " beginning with " + line[0]);
		lineNumber++;
	}
private void addLineToTokenStream ( String [ ] line ) throws InvalidTokenException { boolean foundSubjectCandidate = false ; boolean foundSubjectCandidate2 = false ; boolean foundSubjectCandidate3 = false ; boolean foundPredicateCandidate = false ; boolean foundObjectCandidate = false ; boolean foundObjectCandidate2 = false ; boolean foundObjectCandidate3 = false ; String predToken = "" ; for ( int i = 0 ; i < line . length ; i ++ ) { String curToken = line [ i ] ; String nextToken = new String ( ) ; String thirdToken = new String ( ) ; if ( i < line . length - 2 ) nextToken = line [ i + 1 ] ; if ( i < line . length - 3 ) thirdToken = line [ i + 2 ] ; curToken = curToken . replace ( "\t" , "" ) ; nextToken = nextToken . replace ( "\t" , "" ) ; if ( curToken . isEmpty ( ) ) continue ; else if ( ! foundSubjectCandidate ) { foundSubjectCandidate = true ; if ( isClassToken ( curToken ) ) tokenStream . add ( new ClassToken ( lineNumber , curToken ) ) ; else throw new InvalidTokenException ( "Invalid Subject" , lineNumber , curToken ) ; } else if ( ! foundSubjectCandidate2 && ( isLeftSum ( nextToken ) || isLeftSum3 ( thirdToken ) ) ) { foundSubjectCandidate2 = true ; if ( isClassToken ( curToken ) ) tokenStream . add ( new ClassToken ( lineNumber , curToken ) ) ; else throw new InvalidTokenException ( "Invalid Subject 2" , lineNumber , curToken ) ; } else if ( ! foundSubjectCandidate3 && isLeftSum3 ( nextToken ) ) { foundSubjectCandidate3 = true ; if ( isClassToken ( curToken ) ) tokenStream . add ( new ClassToken ( lineNumber , curToken ) ) ; else throw new InvalidTokenException ( "Invalid Subject 3" , lineNumber , curToken ) ; } else if ( ! foundPredicateCandidate ) { foundPredicateCandidate = true ; if ( isRelationToken ( curToken ) || isSum3Token ( curToken ) ) { tokenStream . add ( new RelationToken ( lineNumber , curToken ) ) ; predToken = curToken ; } else throw new InvalidTokenException ( "Invalid Predicate" , lineNumber , curToken ) ; } else if ( ! foundObjectCandidate ) { foundObjectCandidate = true ; if ( isClassToken ( curToken ) ) tokenStream . add ( new ClassToken ( lineNumber , curToken ) ) ; else throw new InvalidTokenException ( "Invalid Object" , lineNumber , curToken ) ; } else if ( ! foundObjectCandidate2 && ( isRightSum ( predToken ) || isRightSum3 ( predToken ) ) ) { foundObjectCandidate2 = true ; if ( isClassToken ( curToken ) ) tokenStream . add ( new ClassToken ( lineNumber , curToken ) ) ; else throw new InvalidTokenException ( "Invalid Object 2" , lineNumber , curToken ) ; } else if ( ! foundObjectCandidate3 && isRightSum3 ( predToken ) ) { foundObjectCandidate3 = true ; if ( isClassToken ( curToken ) ) tokenStream . add ( new ClassToken ( lineNumber , curToken ) ) ; else throw new InvalidTokenException ( "Invalid Object 3" , lineNumber , curToken ) ; } else throw new InvalidTokenException ( "Too many tokens: " , lineNumber , curToken ) ; } if ( foundSubjectCandidate && ! foundObjectCandidate ) throw new InvalidTokenException ( "Too few tokens: " , lineNumber , " beginning with " + line [ 0 ] ) ; lineNumber ++ ; }
Tokenizes and adds a given line of input to the token stream as tokens
 @param line The line of the input file to tokenize and add
Tokenizes and adds a given line of input to the token stream as tokens @param line The line of the input file to tokenize and add
// renamed for the purpose of readability // remove whitespace (spaces and newlines have already been handled)