bolt/spark/stack.py
def stack(self, size):
        """
        Make an intermediate RDD where all records are combined into a
        list of keys and larger ndarray along a new 0th dimension.
        """
        def tostacks(partition):
            keys = []
            arrs = []
            for key, arr in partition:
                keys.append(key)
                arrs.append(arr)
                if size and 0 <= size <= len(keys):
                    yield (keys, asarray(arrs))
                    keys, arrs = [], []
            if keys:
                yield (keys, asarray(arrs))

        rdd = self._rdd.mapPartitions(tostacks)
        return self._constructor(rdd).__finalize__(self)
def stack(self, size):
        def tostacks(partition):
            keys = []
            arrs = []
            for key, arr in partition:
                keys.append(key)
                arrs.append(arr)
                if size and 0 <= size <= len(keys):
                    yield (keys, asarray(arrs))
                    keys, arrs = [], []
            if keys:
                yield (keys, asarray(arrs))

        rdd = self._rdd.mapPartitions(tostacks)
        return self._constructor(rdd).__finalize__(self)
def stack ( self , size ) : def tostacks ( partition ) : keys = [ ] arrs = [ ] for key , arr in partition : keys . append ( key ) arrs . append ( arr ) if size and 0 <= size <= len ( keys ) : yield ( keys , asarray ( arrs ) ) keys , arrs = [ ] , [ ] if keys : yield ( keys , asarray ( arrs ) ) rdd = self . _rdd . mapPartitions ( tostacks ) return self . _constructor ( rdd ) . __finalize__ ( self )
Make an intermediate RDD where all records are combined into a
        list of keys and larger ndarray along a new 0th dimension.
Make an intermediate RDD where all records are combined into a list of keys and larger ndarray along a new 0th dimension .