def tokenize(str):
    specs = [
        (u'Space', (ur'[ \t\r\n]+',)),
        (u'String', (ur'"(%(unescaped)s | %(escaped)s)*"' % regexps, VERBOSE)),
        (u'Number', (ur'''
            -?                  # Minus
            (0|([1-9][0-9]*))   # Int
            (\.[0-9]+)?         # Frac
            ([Ee][+-][0-9]+)?   # Exp
            ''', VERBOSE)),
        (u'Op', (ur'[{}\[\]\-,:]',)),
        (u'Name', (ur'[A-Za-z_][A-Za-z_0-9]*',)),
    ]
    useless = [u'Space']
    t = make_tokenizer(specs)
    return [x for x in t(str) if x.type not in useless]
Prepare and execute a database operation (query or command).
dataset/ETHPy150Open vlasovskikh/funcparserlib/funcparserlib/tests/json.py/tokenize--crossover with--dataset/ETHPy150Open crate/crate-python/src/crate/client/cursor.py/Cursor.execute