<reponame>EleutherAI/pile-CC-filtering
import numpy as np
x = list(map(float, open('scores.txt')))

def ct_accept(alpha):
    ct = 0
    for score in x:
        # from https://arxiv.org/pdf/2005.14165.pdf P.43
        if np.random.pareto(alpha) > 1 - score:
            ct += 1
    return ct

for alpha in [1,2,3,4,5,6,7,8,9]:
    print(alpha, ct_accept(alpha) / len(x))
import numpy as np
x = list(map(float, open('scores.txt')))

def ct_accept(alpha):
    ct = 0
    for score in x:
        # from https://arxiv.org/pdf/2005.14165.pdf P.43
        if np.random.pareto(alpha) > 1 - score:
            ct += 1
    return ct

for alpha in [1,2,3,4,5,6,7,8,9]:
    print(alpha, ct_accept(alpha) / len(x))
# from https://arxiv.org/pdf/2005.14165.pdf P.43