

def tokenize(parsed, left_delimiters, right_delimiters):
    last = ''
    for thing in parsed:
        if (thing in left_delimiters):
            (before, last) = (last, '')
            (yield LeftDelimiter(before=before, delimiter=thing))
        elif (thing in right_delimiters):
            if last:
                (yield NonDelimiter(content=last))
                last = ''
            (yield RightDelimiter(delimiter=thing))
            last = ''
        else:
            last = thing
    if last:
        (yield NonDelimiter(content=last))

dataset/ETHPy150Open Julian/Condent/condent.py tokenize/original