

def token_encoder(texts, character=False, max_features=9997, min_df=10):
    df = {
        
    }
    for text in texts:
        if character:
            text = list(text)
        else:
            text = tokenize(text)
        tokens = set(text)
        for token in tokens:
            if (token in df):
                df[token] += 1
            else:
                df[token] = 1
    (k, v) = (df.keys(), np.asarray(df.values()))
    valid = (v >= min_df)
    k = lbf(k, valid)
    v = v[valid]
    sort_mask = np.argsort(v)[::(- 1)]
    k = list_index(k, sort_mask)[:max_features]
    v = v[sort_mask][:max_features]
    xtoi = dict(zip(k, range(3, (len(k) + 3))))
    return xtoi

dataset/ETHPy150Open IndicoDataSolutions/Foxhound/foxhound/preprocessing.py token_encoder/original