def udf(sc, data_path, sampling_ratio, seed):

    # Compute the number of cores in our cluster - used below to heuristically set the number of partitions
    total_cores = int(sc._conf.get('spark.executor.instances')) * int(sc._conf.get('spark.executor.cores'))

    # Load and sample down the dataset
    d = sc.textFile(data_path, total_cores * 3).sample(False, sampling_ratio, seed)

    deserialize_vec = lambda s: pkl.loads(base64.decodestring(s))
    vecs = d.map(deserialize_vec)

    return vecs
Needs to get the file as well which fields have 
dataset/ETHPy150Open yahoo/lopq/spark/example_udf.py/udf--crossover with--dataset/ETHPy150Open associatedpress/geomancer/geomancer/api.py/geomance_api