def tokenize(stream):
    def is_delimiter(char):
        return char.isspace() or char in "{}[]:,"

    token = []
    charcode = 0
    completed = False
    now_token = ""

    def process_char(char, charcode):
        nonlocal token, completed, now_token
        advance = True
        add_char = False
        next_state = state
        if state == __TOKENIZER_STATE.WHITESPACE:
            if char == "{":
                completed = True
                now_token = (TOKEN_TYPE.OPERATOR, "{")
            elif char == "}":
                completed = True
                now_token = (TOKEN_TYPE.OPERATOR, "}")
            elif char == "[":
                completed = True
                now_token = (TOKEN_TYPE.OPERATOR, "[")
            elif char == "]":
                completed = True
                now_token = (TOKEN_TYPE.OPERATOR, "]")
            elif char == ",":
                completed = True
                now_token = (TOKEN_TYPE.OPERATOR, ",")
            elif char == ":":
                completed = True
                now_token = (TOKEN_TYPE.OPERATOR, ":")
            elif char == "\"":
                next_state = __TOKENIZER_STATE.STRING
            elif char in "123456789":
                next_state = __TOKENIZER_STATE.INTEGER
                add_char = True
            elif char == "0":
                next_state = __TOKENIZER_STATE.INTEGER_0
                add_char = True
            elif char == "-":
                next_state = __TOKENIZER_STATE.INTEGER_SIGN
                add_char = True
            elif char == "f":
                next_state = __TOKENIZER_STATE.FALSE_1
            elif char == "t":
                next_state = __TOKENIZER_STATE.TRUE_1
            elif char == "n":
                next_state = __TOKENIZER_STATE.NULL_1
            elif not char.isspace():
                raise ValueError("Invalid JSON character: '{0}'".format(char))
        elif state == __TOKENIZER_STATE.INTEGER:
            if char in "0123456789":
                add_char = True
            elif char == ".":
                next_state = __TOKENIZER_STATE.FLOATING_POINT_0
                add_char = True
            elif char == "e" or char == 'E':
                next_state = __TOKENIZER_STATE.INTEGER_EXP_0
                add_char = True
            elif is_delimiter(char):
                next_state = __TOKENIZER_STATE.WHITESPACE
                completed = True
                now_token = (TOKEN_TYPE.NUMBER, int("".join(token)))
                advance = False
            else:
                raise ValueError("A number must contain only digits.  Got '{}'".format(char))
        elif state == __TOKENIZER_STATE.INTEGER_0:
            if char == ".":
                next_state = __TOKENIZER_STATE.FLOATING_POINT_0
                add_char = True
            elif char == "e" or char == 'E':
                next_state = __TOKENIZER_STATE.INTEGER_EXP_0
                add_char = True
            elif is_delimiter(char):
                next_state = __TOKENIZER_STATE.WHITESPACE
                completed = True
                now_token = (TOKEN_TYPE.NUMBER, 0)
                advance = False
            else:
                raise ValueError("A 0 must be followed by a '.' or a 'e'.  Got '{0}'".format(char))
        elif state == __TOKENIZER_STATE.INTEGER_SIGN:
            if char == "0":
                next_state = __TOKENIZER_STATE.INTEGER_0
                add_char = True
            elif char in "123456789":
                next_state = __TOKENIZER_STATE.INTEGER
                add_char = True
            else:
                raise ValueError("A - must be followed by a digit.  Got '{0}'".format(char))
        elif state == __TOKENIZER_STATE.INTEGER_EXP_0:
            if char == "+" or char == "-" or char in "0123456789":
                next_state = __TOKENIZER_STATE.INTEGER_EXP
                add_char = True
            else:
                raise ValueError("An e in a number must be followed by a '+', '-' or digit.  Got '{0}'".format(char))
        elif state == __TOKENIZER_STATE.INTEGER_EXP:
            if char in "0123456789":
                add_char = True
            elif is_delimiter(char):
                completed = True
                now_token = (TOKEN_TYPE.NUMBER, float("".join(token)))
                next_state = __TOKENIZER_STATE.WHITESPACE
                advance = False
            else:
                raise ValueError("A number exponent must consist only of digits.  Got '{}'".format(char))
        elif state == __TOKENIZER_STATE.FLOATING_POINT:
            if char in "0123456789":
                add_char = True
            elif char == "e" or char == "E":
                next_state = __TOKENIZER_STATE.INTEGER_EXP_0
                add_char = True
            elif is_delimiter(char):
                completed = True
                now_token = (TOKEN_TYPE.NUMBER, float("".join(token)))
                next_state = __TOKENIZER_STATE.WHITESPACE
                advance = False
            else:
                raise ValueError("A number must include only digits")
        elif state == __TOKENIZER_STATE.FLOATING_POINT_0:
            if char in "0123456789":
                next_state = __TOKENIZER_STATE.FLOATING_POINT
                add_char = True
            else:
                raise ValueError("A number with a decimal point must be followed by a fractional part")
        elif state == __TOKENIZER_STATE.FALSE_1:
            if char == "a":
                next_state = __TOKENIZER_STATE.FALSE_2
            else:
                raise ValueError("Invalid JSON character: '{0}'".format(char))
        elif state == __TOKENIZER_STATE.FALSE_2:
            if char == "l":
                next_state = __TOKENIZER_STATE.FALSE_3
            else:
                raise ValueError("Invalid JSON character: '{0}'".format(char))
        elif state == __TOKENIZER_STATE.FALSE_3:
            if char == "s":
                next_state = __TOKENIZER_STATE.FALSE_4
            else:
                raise ValueError("Invalid JSON character: '{0}'".format(char))
        elif state == __TOKENIZER_STATE.FALSE_4:
            if char == "e":
                next_state = __TOKENIZER_STATE.WHITESPACE
                completed = True
                now_token = (TOKEN_TYPE.BOOLEAN, False)
            else:
                raise ValueError("Invalid JSON character: '{0}'".format(char))
        elif state == __TOKENIZER_STATE.TRUE_1:
            if char == "r":
                next_state = __TOKENIZER_STATE.TRUE_2
            else:
                raise ValueError("Invalid JSON character: '{0}'".format(char))
        elif state == __TOKENIZER_STATE.TRUE_2:
            if char == "u":
                next_state = __TOKENIZER_STATE.TRUE_3
            else:
                raise ValueError("Invalid JSON character: '{0}'".format(char))
        elif state == __TOKENIZER_STATE.TRUE_3:
            if char == "e":
                next_state = __TOKENIZER_STATE.WHITESPACE
                completed = True
                now_token = (TOKEN_TYPE.BOOLEAN, True)
            else:
                raise ValueError("Invalid JSON character: '{0}'".format(char))
        elif state == __TOKENIZER_STATE.NULL_1:
            if char == "u":
                next_state = __TOKENIZER_STATE.NULL_2
            else:
                raise ValueError("Invalid JSON character: '{0}'".format(char))
        elif state == __TOKENIZER_STATE.NULL_2:
            if char == "l":
                next_state = __TOKENIZER_STATE.NULL_3
            else:
                raise ValueError("Invalid JSON character: '{0}'".format(char))
        elif state == __TOKENIZER_STATE.NULL_3:
            if char == "l":
                next_state = __TOKENIZER_STATE.WHITESPACE
                completed = True
                now_token = (TOKEN_TYPE.NULL, None)
            else:
                raise ValueError("Invalid JSON character: '{0}'".format(char))
        elif state == __TOKENIZER_STATE.STRING:
            if char == "\"":
                completed = True
                now_token = (TOKEN_TYPE.STRING, "".join(token))
                next_state = __TOKENIZER_STATE.STRING_END
            elif char == "\\":
                next_state = __TOKENIZER_STATE.STRING_ESCAPE
            else:
                add_char = True
        elif state == __TOKENIZER_STATE.STRING_END:
            if is_delimiter(char):
                advance = False
                next_state = __TOKENIZER_STATE.WHITESPACE
            else:
                raise ValueError("Expected whitespace or an operator after strin.  Got '{}'".format(char))
        elif state == __TOKENIZER_STATE.STRING_ESCAPE:
            next_state = __TOKENIZER_STATE.STRING
            if char == "\\" or char == "\"":
                add_char = True
            elif char == "b":
                char = "\b"
                add_char = True
            elif char == "f":
                char = "\f"
                add_char = True
            elif char == "n":
                char = "\n"
                add_char = True
            elif char == "t":
                char = "\t"
                add_char = True
            elif char == "r":
                char = "\r"
                add_char = True
            elif char == "/":
                char = "/"
                add_char = True
            elif char == "u":
                next_state = __TOKENIZER_STATE.UNICODE_1
                charcode = 0
            else:
                raise ValueError("Invalid string escape: {}".format(char))
        elif state == __TOKENIZER_STATE.UNICODE_1:
            if char in "0123456789":
                charcode = (ord(char) - 48) * 4096
            elif char in "abcdef":
                charcode = (ord(char) - 87) * 4096
            elif char in "ABCDEF":
                charcode = (ord(char) - 55) * 4096
            else:
                raise ValueError("Invalid character code: {}".format(char))
            next_state = __TOKENIZER_STATE.UNICODE_2
            char = ""
        elif state == __TOKENIZER_STATE.UNICODE_2:
            if char in "0123456789":
                charcode += (ord(char) - 48) * 256
            elif char in "abcdef":
                charcode += (ord(char) - 87) * 256
            elif char in "ABCDEF":
                charcode += (ord(char) - 55) * 256
            else:
                raise ValueError("Invalid character code: {}".format(char))
            next_state = __TOKENIZER_STATE.UNICODE_3
            char = ""
        elif state == __TOKENIZER_STATE.UNICODE_3:
            if char in "0123456789":
                charcode += (ord(char) - 48) * 16
            elif char in "abcdef":
                charcode += (ord(char) - 87) * 16
            elif char in "ABCDEF":
                charcode += (ord(char) - 55) * 16
            else:
                raise ValueError("Invalid character code: {}".format(char))
            next_state = __TOKENIZER_STATE.UNICODE_4
            char = ""
        elif state == __TOKENIZER_STATE.UNICODE_4:
            if char in "0123456789":
                charcode += ord(char) - 48
            elif char in "abcdef":
                charcode += ord(char) - 87
            elif char in "ABCDEF":
                charcode += ord(char) - 55
            else:
                raise ValueError("Invalid character code: {}".format(char))
            next_state = __TOKENIZER_STATE.STRING
            char = chr(charcode)
            add_char = True

        if add_char:
            token.append(char)

        return advance, next_state, charcode
    state = __TOKENIZER_STATE.WHITESPACE
    char = stream.read(1)
    index = 0
    while char:
        try:
            advance, state, charcode = process_char(char, charcode)
        except ValueError as e:
            raise ValueError("".join([e.args[0], " at index {}".format(index)]))
        if completed:
            completed = False
            token = []
            yield now_token
        if advance:
            char = stream.read(1)
            index += 1
    process_char(" ", charcode)
    if completed:
        yield now_token