

def udf(sc, data_path, sampling_ratio, seed):
    '\n    This is an example UDF function to load training data. It loads data from a text file\n    with base64-encoded pickled numpy arrays on each line.\n    '
    total_cores = (int(sc._conf.get('spark.executor.instances')) * int(sc._conf.get('spark.executor.cores')))
    d = sc.textFile(data_path, (total_cores * 3)).sample(False, sampling_ratio, seed)
    deserialize_vec = (lambda s: pkl.loads(base64.decodestring(s)))
    vecs = d.map(deserialize_vec)
    return vecs

dataset/ETHPy150Open yahoo/lopq/spark/example_udf.py udf/original