def tokenize_flat(css_source, ignore_comments=True,
    # Make these local variable to avoid global lookups in the loop
    tokens_dispatch=token_data.TOKEN_DISPATCH,
    unicode_unescape=token_data.UNICODE_UNESCAPE,
    newline_unescape=token_data.NEWLINE_UNESCAPE,
    simple_unescape=token_data.SIMPLE_UNESCAPE,
    find_newlines=token_data.FIND_NEWLINES,
    Token=token_data.Token,
    len=len,
    int=int,
    float=float,
    list=list,
    _None=None,
):

    pos = 0
    line = 1
    column = 1
    source_len = len(css_source)
    tokens = []
    while pos < source_len:
        char = css_source[pos]
        if char in ':;{}()[]':
            type_ = char
            css_value = char
        else:
            codepoint = min(ord(char), 160)
            for _index, type_, regexp in tokens_dispatch[codepoint]:
                match = regexp(css_source, pos)
                if match:
                    # First match is the longest. See comments on TOKENS above.
                    css_value = match.group()
                    break
            else:
                # No match.
                # "Any other character not matched by the above rules,
                #  and neither a single nor a double quote."
                # ... but quotes at the start of a token are always matched
                # by STRING or BAD_STRING. So DELIM is any single character.
                type_ = 'DELIM'
                css_value = char
        length = len(css_value)
        next_pos = pos + length

        # A BAD_COMMENT is a comment at EOF. Ignore it too.
        if not (ignore_comments and type_ in ('COMMENT', 'BAD_COMMENT')):
            # Parse numbers, extract strings and URIs, unescape
            unit = _None
            if type_ == 'DIMENSION':
                value = match.group(1)
                value = float(value) if '.' in value else int(value)
                unit = match.group(2)
                unit = simple_unescape(unit)
                unit = unicode_unescape(unit)
                unit = unit.lower()  # normalize
            elif type_ == 'PERCENTAGE':
                value = css_value[:-1]
                value = float(value) if '.' in value else int(value)
                unit = '%'
            elif type_ == 'NUMBER':
                value = css_value
                if '.' in value:
                    value = float(value)
                else:
                    value = int(value)
                    type_ = 'INTEGER'
            elif type_ in ('IDENT', 'ATKEYWORD', 'HASH', 'FUNCTION'):
                value = simple_unescape(css_value)
                value = unicode_unescape(value)
            elif type_ == 'URI':
                value = match.group(1)
                if value and value[0] in '"\'':
                    value = value[1:-1]  # Remove quotes
                    value = newline_unescape(value)
                value = simple_unescape(value)
                value = unicode_unescape(value)
            elif type_ == 'STRING':
                value = css_value[1:-1]  # Remove quotes
                value = newline_unescape(value)
                value = simple_unescape(value)
                value = unicode_unescape(value)
            # BAD_STRING can only be one of:
            # * Unclosed string at the end of the stylesheet:
            #   Close the string, but this is not an error.
            #   Make it a "good" STRING token.
            # * Unclosed string at the (unescaped) end of the line:
            #   Close the string, but this is an error.
            #   Leave it as a BAD_STRING, don’t bother parsing it.
            # See http://www.w3.org/TR/CSS21/syndata.html#parsing-errors
            elif type_ == 'BAD_STRING' and next_pos == source_len:
                type_ = 'STRING'
                value = css_value[1:]  # Remove quote
                value = newline_unescape(value)
                value = simple_unescape(value)
                value = unicode_unescape(value)
            else:
                value = css_value
            tokens.append(Token(type_, css_value, value, unit, line, column))

        pos = next_pos
        newlines = list(find_newlines(css_value))
        if newlines:
            line += len(newlines)
            # Add 1 to have lines start at column 1, not 0
            column = length - newlines[-1].end() + 1
        else:
            column += length
    return tokens
Strings that are not closed result in an parse error.
dataset/ETHPy150Open FichteFoll/CSScheme/tinycsscheme/tinycss/tokenizer.py/tokenize_flat--crossover with--dataset/ETHPy150Open kvalle/diy-lisp/tests/test_8_final_touches.py/test_parsing_unclosed_strings