src-gu/mir-reasoner/src/common_classes/Tokenizer.java
/** Tokenizes the input file and saves the list of tokens.
	 * @param in The input file to tokenize.
	 * @return True if the file is tokenized successfully, else false.
	 * @throws IOException If the file cannot be opened, read or closed.	*/
Tokenizes the input file and saves the list of tokens.
@param in The input file to tokenize.
@return True if the file is tokenized successfully, else false.
@throws IOException If the file cannot be opened, read or closed.
Tokenizes the input file and saves the list of tokens . @param in The input file to tokenize . @return True if the file is tokenized successfully else false . @throws IOException If the file cannot be opened read or closed .
public boolean tokenize(File in) throws IOException{
		lineNumber = 1;
		Scanner input = new Scanner(in);
		try {
		    while(input.hasNextLine()) {
				String line = input.nextLine();
				int endIndex = line.indexOf(']');
				if(endIndex == -1){
			    	if(!line.startsWith("#"))
						addLineToTokenStream(line.split("[ ]+"));
			    }
			    else{
			    	String subline = line.substring(0, endIndex+1);
			    	if(!subline.startsWith("#"))
						addLineToTokenStream(subline.split("[ ]+"));
				}
			}
		} catch(InvalidTokenException ex) { 
		    System.out.println( ex.getMessage() ); 
		    return false; 
		}
		input.close();
		return true;
	}
public boolean tokenize ( File in ) throws IOException { lineNumber = 1 ; Scanner input = new Scanner ( in ) ; try { while ( input . hasNextLine ( ) ) { String line = input . nextLine ( ) ; int endIndex = line . indexOf ( ']' ) ; if ( endIndex == - 1 ) { if ( ! line . startsWith ( "#" ) ) addLineToTokenStream ( line . split ( "[ ]+" ) ) ; } else { String subline = line . substring ( 0 , endIndex + 1 ) ; if ( ! subline . startsWith ( "#" ) ) addLineToTokenStream ( subline . split ( "[ ]+" ) ) ; } } } catch ( InvalidTokenException ex ) { System . out . println ( ex . getMessage ( ) ) ; return false ; } input . close ( ) ; return true ; }
Tokenizes the input file and saves the list of tokens.
Tokenizes the input file and saves the list of tokens .