bolt/spark/construct.py
def _wrap(func, shape, context=None, axis=(0,), dtype=None, npartitions=None):
        """
        Wrap an existing numpy constructor in a parallelized construction
        """
        if isinstance(shape, int):
            shape = (shape,)
        key_shape, value_shape = get_kv_shape(shape, ConstructSpark._format_axes(axis, shape))
        split = len(key_shape)

        # make the keys
        rdd = context.parallelize(list(product(*[arange(x) for x in key_shape])), npartitions)

        # use a map to make the arrays in parallel
        rdd = rdd.map(lambda x: (x, func(value_shape, dtype, order='C')))
        return BoltArraySpark(rdd, shape=shape, split=split, dtype=dtype)
def _wrap(func, shape, context=None, axis=(0,), dtype=None, npartitions=None):
        if isinstance(shape, int):
            shape = (shape,)
        key_shape, value_shape = get_kv_shape(shape, ConstructSpark._format_axes(axis, shape))
        split = len(key_shape)

        # make the keys
        rdd = context.parallelize(list(product(*[arange(x) for x in key_shape])), npartitions)

        # use a map to make the arrays in parallel
        rdd = rdd.map(lambda x: (x, func(value_shape, dtype, order='C')))
        return BoltArraySpark(rdd, shape=shape, split=split, dtype=dtype)
def _wrap ( func , shape , context = None , axis = ( 0 , ) , dtype = None , npartitions = None ) : if isinstance ( shape , int ) : shape = ( shape , ) key_shape , value_shape = get_kv_shape ( shape , ConstructSpark . _format_axes ( axis , shape ) ) split = len ( key_shape ) # make the keys rdd = context . parallelize ( list ( product ( * [ arange ( x ) for x in key_shape ] ) ) , npartitions ) # use a map to make the arrays in parallel rdd = rdd . map ( lambda x : ( x , func ( value_shape , dtype , order = 'C' ) ) ) return BoltArraySpark ( rdd , shape = shape , split = split , dtype = dtype )
Wrap an existing numpy constructor in a parallelized construction
Wrap an existing numpy constructor in a parallelized construction