def test_tokenize_file(filename, expected):

    tokens = Tokenizer.tokenize(filename)
    if len(tokens) != len(expected):
        print("Tokens and expected are different lengths\n")
        return False

    for idx in range(len(expected)):
        if tokens[idx]['type'] != expected[idx]:
            print("Difference at index: " + str(idx) + "\n")
            print("Expected: " + str(expected[idx]))
            print("Received: " + str(tokens[idx]))
            return False

    print("Tokenizer tests pass\n")
    return True
Parse the file and verify that the types are what we expect
dataset/ETHPy150Open samshadwell/TrumpScript/test/test.py/test_tokenize_file