

def source_token_lines(source):
    "Generate a series of lines, one for each line in `source`.\n\n    Each line is a list of pairs, each pair is a token::\n\n        [('key', 'def'), ('ws', ' '), ('nam', 'hello'), ('op', '('), ... ]\n\n    Each pair has a token class, and the token text.\n\n    If you concatenate all the token texts, and then join them with newlines,\n    you should have your original `source` back, with two differences:\n    trailing whitespace is not preserved, and a final line with no newline\n    is indistinguishable from a final line with a newline.\n\n    "
    ws_tokens = set([token.INDENT, token.DEDENT, token.NEWLINE, tokenize.NL])
    line = []
    col = 0
    source = source.expandtabs(8).replace('\r\n', '\n')
    tokgen = generate_tokens(source)
    for (ttype, ttext, (_, scol), (_, ecol), _) in phys_tokens(tokgen):
        mark_start = True
        for part in re.split('(\n)', ttext):
            if (part == '\n'):
                (yield line)
                line = []
                col = 0
                mark_end = False
            elif (part == ''):
                mark_end = False
            elif (ttype in ws_tokens):
                mark_end = False
            else:
                if (mark_start and (ttype > col)):
                    line.append(('ws', (' ' * (scol - col))))
                    mark_start = False
                tok_class = tokenize.tok_name.get(ttype, 'xx').lower()[:3]
                if ((ttype == token.NAME) and keyword.iskeyword(ttext)):
                    tok_class = 'key'
                line.append((tok_class, part))
                mark_end = True
            scol = 0
        if mark_end:
            col = ecol
    if line:
        (yield line)

dataset/ETHPy150Open RoseOu/flasky/venv/lib/python2.7/site-packages/coverage/phystokens.py source_token_lines/VarMisuse@178/462 `scol`->`ttype`