def get_parameter(f):
    dic = dict()

    # read and decode the first line
    line = f.readline().decode().split()

    dic["name"] = line[0]
    dic["subtype"] = line[1]
    dic["basictype"] = line[2]
    dic["maxvalue"] = line[3]
    dic["minvalue"] = line[4]
    dic["stepsize"] = line[5]
    dic["Ggroup"] = line[6]
    dic["Dgroup"] = line[7]
    dic["protection"] = line[8]
    dic["active"] = line[9]
    dic["intptr"] = line[10]

    # read in the values of the parameter
    line = f.readline().decode()
    num = int(line.split()[0])
    values = []

    if dic["basictype"] == "1":     # real values, only one line
        values = line.split()[1:]
    elif dic["basictype"] == "2":   # strings, may have multiple lines
        values.append(line.split("\"")[1])  # split on "s
        for i in range(num - 1):
            values.append(f.readline().decode().split("\"")[1])

    dic["values"] = values
    line = f.readline().decode()

    # read and decode the enumerables
    dic["enumerable"] = line.split()[0]

    if dic["enumerable"] != "0":
        if dic["basictype"] == "1":     # reals
            dic["enumerables"] = line.split()[1:]
        elif dic["basictype"] == "2":   # strings
            dic["enumerables"] = line.split("\"")[1::2]

    return dic
Tokenization/string cleaning. This is specific to tweets, but can be applied for other texts as well.
dataset/ETHPy150Open jjhelmus/nmrglue/nmrglue/fileio/varian.py/get_parameter--crossover with--dataset/ETHPy150Open flipkart-incubator/optimus/cnn_text_trainer/rw/datasets.py/clean_str