def tokenize(string):
    tokens = deque()
    for m in _RE_TOKENIZER.finditer(string):
        if m.group(_CTYPE):
            name, token = parsing.parse_header(m.group(_CTYPE))
        elif m.group(_BOUNDARY):
            token = Boundary(m.group(_BOUNDARY).strip("\t\r\n"),
                             _grab_newline(m.start(), string, -1),
                             _grab_newline(m.end(), string, 1))
        else:
            token = _EMPTY_LINE

        tokens.append(token)
    return _filter_false_tokens(tokens)
Scans the entire message to find all Content-Types and boundaries.
dataset/ETHPy150Open mailgun/flanker/flanker/mime/message/scanner.py/tokenize