def test_tokenize_file(filename, expected):
 tokens = Tokenizer.tokenize(filename)
 if (len(tokens) != len(expected)):
  print 'Tokens and expected are different lengths\n'
  return False
 for idx in range(len(expected)):
  if (tokens[idx]['type'] != expected[idx]):
   print (('Difference at index: ' + str(idx)) + '\n')
   print ('Expected: ' + str(expected[idx]))
   print ('Received: ' + str(tokens[idx]))
   return False
 print 'Tokenizer tests pass\n'
 return True

def test_tokenize_file ( filename , expected ) : tokens = Tokenizer . tokenize ( filename ) if ( len ( tokens ) != len ( expected ) ) : print 'Tokens and expected are different lengths\n' return False for idx in range ( len ( expected ) ) : if ( tokens [ idx ] [ 'type' ] != expected [ idx ] ) : print ( ( 'Difference at index: ' + str ( idx ) ) + '\n' ) print ( 'Expected: ' + str ( expected [ idx ] ) ) print ( 'Received: ' + str ( tokens [ idx ] ) ) return False print 'Tokenizer tests pass\n' return True
parse the file and verify that the types are what we expect .