def get_widgets_that_need_update(self):
        """
        Returns all widgets that need an update.

        This should be scheduled every minute via crontab.

        """
        result = []
        for widget_name, widget in self.get_widgets().items():
            if widget.should_update():
                result.append(widget)
        return result

def get_widgets_that_need_update ( self ) : result = [ ] for widget_name , widget in self . get_widgets ( ) . items ( ) : if widget . should_update ( ) : result . append ( widget ) return result

Returns all widgets that need an update .


def register_widget(self, widget_cls, **widget_kwargs):
        """
        Registers the given widget.

        Widgets must inherit ``DashboardWidgetBase`` and you cannot register
        the same widget twice.

        :widget_cls: A class that inherits ``DashboardWidgetBase``.

        """
        if not issubclass(widget_cls, DashboardWidgetBase):
            raise ImproperlyConfigured(
                'DashboardWidgets must be subclasses of DashboardWidgetBase,'
                ' {0} is not.'.format(widget_cls))

        widget = widget_cls(**widget_kwargs)
        widget_name = widget.get_name()
        if widget_name in self.widgets:
            raise WidgetAlreadyRegistered(
                'Cannot register {0}, a plugin with this name {1} is already '
                'registered.'.format(widget_cls, widget_name))

        self.widgets[widget_name] = widget


def unregister_widget(self, widget_cls):
        """Unregisters the given widget."""
        if widget_cls.__name__ in self.widgets:
            del self.widgets[widget_cls().get_name()]

def unregister_widget ( self , widget_cls ) : if widget_cls . __name__ in self . widgets : del self . widgets [ widget_cls ( ) . get_name ( ) ]


def get_last_update(self):
        """Gets or creates the last update object for this widget."""
        instance, created = \
            models.DashboardWidgetLastUpdate.objects.get_or_create(
                widget_name=self.get_name())
        return instance

def get_last_update ( self ) : instance , created = models . DashboardWidgetLastUpdate . objects . get_or_create ( widget_name = self . get_name ( ) ) return instance

Gets or creates the last update object for this widget .


def get_setting(self, setting_name, default=None):
        """
        Returns the setting for this widget from the database.

        :setting_name: The name of the setting.
        :default: Optional default value if the setting cannot be found.

        """
        try:
            setting = models.DashboardWidgetSettings.objects.get(
                widget_name=self.get_name(),
                setting_name=setting_name)
        except models.DashboardWidgetSettings.DoesNotExist:
            setting = default
        return setting

def get_setting ( self , setting_name , default = None ) : try : setting = models . DashboardWidgetSettings . objects . get ( widget_name = self . get_name ( ) , setting_name = setting_name ) except models . DashboardWidgetSettings . DoesNotExist : setting = default return setting

Returns the setting for this widget from the database .


def save_setting(self, setting_name, value):
        """Saves the setting value into the database."""
        setting = self.get_setting(setting_name)
        if setting is None:
            setting = models.DashboardWidgetSettings.objects.create(
                widget_name=self.get_name(),
                setting_name=setting_name,
                value=value)
        setting.value = value
        setting.save()
        return setting

def save_setting ( self , setting_name , value ) : setting = self . get_setting ( setting_name ) if setting is None : setting = models . DashboardWidgetSettings . objects . create ( widget_name = self . get_name ( ) , setting_name = setting_name , value = value ) setting . value = value setting . save ( ) return setting

Saves the setting value into the database .


def should_update(self):
        """
        Checks if an update is needed.

        Checks against ``self.update_interval`` and this widgets
        ``DashboardWidgetLastUpdate`` instance if an update is overdue.

        This should be called by
        ``DashboardWidgetPool.get_widgets_that_need_update()``, which in turn
        should be called by an admin command which should be scheduled every
        minute via crontab.

        """
        last_update = self.get_last_update()
        time_since = now() - last_update.last_update
        if time_since.seconds < self.update_interval:
            return False
        return True

def should_update ( self ) : last_update = self . get_last_update ( ) time_since = now ( ) - last_update . last_update if time_since . seconds < self . update_interval : return False return True


def array(a, context=None, axis=(0,), dtype=None, npartitions=None):
        """
        Create a spark bolt array from a local array.

        Parameters
        ----------
        a : array-like
            An array, any object exposing the array interface, an
            object whose __array__ method returns an array, or any
            (nested) sequence.

        context : SparkContext
            A context running Spark. (see pyspark)

        axis : tuple, optional, default=(0,)
            Which axes to distribute the array along. The resulting
            distributed object will use keys to represent these axes,
            with the remaining axes represented by values.

        dtype : data-type, optional, default=None
            The desired data-type for the array. If None, will
            be determined from the data. (see numpy)

        npartitions : int
            Number of partitions for parallization.

        Returns
        -------
        BoltArraySpark
        """
        if dtype is None:
            arry = asarray(a)
            dtype = arry.dtype
        else:
            arry = asarray(a, dtype)
        shape = arry.shape
        ndim = len(shape)

        # handle the axes specification and transpose if necessary
        axes = ConstructSpark._format_axes(axis, arry.shape)
        key_axes, value_axes = get_kv_axes(arry.shape, axes)
        permutation = key_axes + value_axes
        arry = arry.transpose(*permutation)
        split = len(axes)

        if split < 1:
            raise ValueError("split axis must be greater than 0, got %g" % split)
        if split > len(shape):
            raise ValueError("split axis must not exceed number of axes %g, got %g" % (ndim, split))

        key_shape = shape[:split]
        val_shape = shape[split:]

        keys = zip(*unravel_index(arange(0, int(prod(key_shape))), key_shape))
        vals = arry.reshape((prod(key_shape),) + val_shape)

        rdd = context.parallelize(zip(keys, vals), npartitions)
        return BoltArraySpark(rdd, shape=shape, split=split, dtype=dtype)

Create a spark bolt array from a local array .


def ones(shape, context=None, axis=(0,), dtype=float64, npartitions=None):
        """
        Create a spark bolt array of ones.

        Parameters
        ----------
        shape : tuple
            The desired shape of the array.

        context : SparkContext
            A context running Spark. (see pyspark)

        axis : tuple, optional, default=(0,)
            Which axes to distribute the array along. The resulting
            distributed object will use keys to represent these axes,
            with the remaining axes represented by values.

        dtype : data-type, optional, default=float64
            The desired data-type for the array. If None, will
            be determined from the data. (see numpy)

        npartitions : int
            Number of partitions for parallization.

        Returns
        -------
        BoltArraySpark
        """
        from numpy import ones
        return ConstructSpark._wrap(ones, shape, context, axis, dtype, npartitions)

def ones ( shape , context = None , axis = ( 0 , ) , dtype = float64 , npartitions = None ) : from numpy import ones return ConstructSpark . _wrap ( ones , shape , context , axis , dtype , npartitions )


def concatenate(arrays, axis=0):
        """
        Join two bolt arrays together, at least one of which is in spark.

        Parameters
        ----------
        arrays : tuple
            A pair of arrays. At least one must be a spark array,
            the other can be a local bolt array, a local numpy array,
            or an array-like.

        axis : int, optional, default=0
            The axis along which the arrays will be joined.

        Returns
        -------
        BoltArraySpark
        """
        if not isinstance(arrays, tuple):
            raise ValueError("data type not understood")
        if not len(arrays) == 2:
            raise NotImplementedError("spark concatenation only supports two arrays")

        first, second = arrays
        if isinstance(first, BoltArraySpark):
            return first.concatenate(second, axis)
        elif isinstance(second, BoltArraySpark):
            first = ConstructSpark.array(first, second._rdd.context)
            return first.concatenate(second, axis)
        else:
            raise ValueError("at least one array must be a spark bolt array")

Join two bolt arrays together at least one of which is in spark .


def _argcheck(*args, **kwargs):
        """
        Check that arguments are consistent with spark array construction.

        Conditions are:
        (1) a positional argument is a SparkContext
        (2) keyword arg 'context' is a SparkContext
        (3) an argument is a BoltArraySpark, or
        (4) an argument is a nested list containing a BoltArraySpark
        """
        try:
            from pyspark import SparkContext
        except ImportError:
            return False

        cond1 = any([isinstance(arg, SparkContext) for arg in args])
        cond2 = isinstance(kwargs.get('context', None), SparkContext)
        cond3 = any([isinstance(arg, BoltArraySpark) for arg in args])
        cond4 = any([any([isinstance(sub, BoltArraySpark) for sub in arg])
                     if isinstance(arg, (tuple, list)) else False for arg in args])
        return cond1 or cond2 or cond3 or cond4

def _argcheck ( * args , * * kwargs ) : try : from pyspark import SparkContext except ImportError : return False cond1 = any ( [ isinstance ( arg , SparkContext ) for arg in args ] ) cond2 = isinstance ( kwargs . get ( 'context' , None ) , SparkContext ) cond3 = any ( [ isinstance ( arg , BoltArraySpark ) for arg in args ] ) cond4 = any ( [ any ( [ isinstance ( sub , BoltArraySpark ) for sub in arg ] ) if isinstance ( arg , ( tuple , list ) ) else False for arg in args ] ) return cond1 or cond2 or cond3 or cond4

Check that arguments are consistent with spark array construction .


def _format_axes(axes, shape):
        """
        Format target axes given an array shape
        """
        if isinstance(axes, int):
            axes = (axes,)
        elif isinstance(axes, list) or hasattr(axes, '__iter__'):
            axes = tuple(axes)
        if not isinstance(axes, tuple):
            raise ValueError("axes argument %s in the constructor not specified correctly" % str(axes))
        if min(axes) < 0 or max(axes) > len(shape) - 1:
            raise ValueError("invalid key axes %s given shape %s" % (str(axes), str(shape)))
        return axes


def _wrap(func, shape, context=None, axis=(0,), dtype=None, npartitions=None):
        """
        Wrap an existing numpy constructor in a parallelized construction
        """
        if isinstance(shape, int):
            shape = (shape,)
        key_shape, value_shape = get_kv_shape(shape, ConstructSpark._format_axes(axis, shape))
        split = len(key_shape)

        # make the keys
        rdd = context.parallelize(list(product(*[arange(x) for x in key_shape])), npartitions)

        # use a map to make the arrays in parallel
        rdd = rdd.map(lambda x: (x, func(value_shape, dtype, order='C')))
        return BoltArraySpark(rdd, shape=shape, split=split, dtype=dtype)


def _align(self, axes, key_shape=None):
        """
        Align local bolt array so that axes for iteration are in the keys.

        This operation is applied before most functional operators.
        It ensures that the specified axes are valid, and might transpose/reshape
        the underlying array so that the functional operators can be applied
        over the correct records.

        Parameters
        ----------
        axes: tuple[int]
            One or more axes that will be iterated over by a functional operator

        Returns
        -------
        BoltArrayLocal
        """

        # ensure that the key axes are valid for an ndarray of this shape
        inshape(self.shape, axes)

        # compute the set of dimensions/axes that will be used to reshape
        remaining = [dim for dim in range(len(self.shape)) if dim not in axes]
        key_shape = key_shape if key_shape else [self.shape[axis] for axis in axes]
        remaining_shape = [self.shape[axis] for axis in remaining]
        linearized_shape = [prod(key_shape)] + remaining_shape

        # compute the transpose permutation
        transpose_order = axes + remaining

        # transpose the array so that the keys being mapped over come first, then linearize keys
        reshaped = self.transpose(*transpose_order).reshape(*linearized_shape)

        return reshaped

Align local bolt array so that axes for iteration are in the keys .


def tospark(self, sc, axis=0):
        """
        Converts a BoltArrayLocal into a BoltArraySpark

        Parameters
        ----------
        sc : SparkContext
            The SparkContext which will be used to create the BoltArraySpark

        axis : tuple or int, optional, default=0
            The axis (or axes) across which this array will be parallelized

        Returns
        -------
        BoltArraySpark
        """
        from bolt import array
        return array(self.toarray(), sc, axis=axis)

def tospark ( self , sc , axis = 0 ) : from bolt import array return array ( self . toarray ( ) , sc , axis = axis )


def tordd(self, sc, axis=0):
        """
        Converts a BoltArrayLocal into an RDD

        Parameters
        ----------
        sc : SparkContext
            The SparkContext which will be used to create the BoltArraySpark

        axis : tuple or int, optional, default=0
            The axis (or axes) across which this array will be parallelized

        Returns
        -------
        RDD[(tuple, ndarray)]
        """
        from bolt import array
        return array(self.toarray(), sc, axis=axis).tordd()

def tordd ( self , sc , axis = 0 ) : from bolt import array return array ( self . toarray ( ) , sc , axis = axis ) . tordd ( )


def stack(self, size):
        """
        Make an intermediate RDD where all records are combined into a
        list of keys and larger ndarray along a new 0th dimension.
        """
        def tostacks(partition):
            keys = []
            arrs = []
            for key, arr in partition:
                keys.append(key)
                arrs.append(arr)
                if size and 0 <= size <= len(keys):
                    yield (keys, asarray(arrs))
                    keys, arrs = [], []
            if keys:
                yield (keys, asarray(arrs))

        rdd = self._rdd.mapPartitions(tostacks)
        return self._constructor(rdd).__finalize__(self)

def stack ( self , size ) : def tostacks ( partition ) : keys = [ ] arrs = [ ] for key , arr in partition : keys . append ( key ) arrs . append ( arr ) if size and 0 <= size <= len ( keys ) : yield ( keys , asarray ( arrs ) ) keys , arrs = [ ] , [ ] if keys : yield ( keys , asarray ( arrs ) ) rdd = self . _rdd . mapPartitions ( tostacks ) return self . _constructor ( rdd ) . __finalize__ ( self )

Make an intermediate RDD where all records are combined into a list of keys and larger ndarray along a new 0th dimension .


def map(self, func):
        """
        Apply a function on each subarray.

        Parameters
        ----------
        func : function
             This is applied to each value in the intermediate RDD.

        Returns
        -------
        StackedArray
        """
        vshape = self.shape[self.split:]
        x = self._rdd.values().first()
        if x.shape == vshape:
            a, b = asarray([x]), asarray([x, x])
        else:
            a, b = x, concatenate((x, x))

        try:
            atest = func(a)
            btest = func(b)
        except Exception as e:
            raise RuntimeError("Error evaluating function on test array, got error:\n %s" % e)

        if not (isinstance(atest, ndarray) and isinstance(btest, ndarray)):
            raise ValueError("Function must return ndarray")

        # different shapes map to the same new shape
        elif atest.shape == btest.shape:
            if self._rekeyed is True:
                # we've already rekeyed
                rdd = self._rdd.map(lambda kv: (kv[0], func(kv[1])))
                shape = (self.shape[0],) + atest.shape
            else:
                # do the rekeying
                count, rdd = zip_with_index(self._rdd.values())
                rdd = rdd.map(lambda kv: ((kv[1],), func(kv[0])))
                shape = (count,) + atest.shape
            split = 1
            rekeyed = True

        # different shapes stay different (along the first dimension)
        elif atest.shape[0] == a.shape[0] and btest.shape[0] == b.shape[0]:
            shape = self.shape[0:self.split] + atest.shape[1:]
            split = self.split
            rdd = self._rdd.map(lambda kv: (kv[0], func(kv[1])))
            rekeyed = self._rekeyed

        else:
            raise ValueError("Cannot infer effect of function on shape")

        return self._constructor(rdd, rekeyed=rekeyed, shape=shape, split=split).__finalize__(self)


def _chunk(self, size="150", axis=None, padding=None):
        """
        Split values of distributed array into chunks.

        Transforms an underlying pair RDD of (key, value) into
        records of the form: (key, chunk id), (chunked value).
        Here, chunk id is a tuple identifying the chunk and
        chunked value is a subset of the data from each original value,
        that has been divided along the specified dimensions.

        Parameters
        ----------
        size : str or tuple or int
            If str, the average size (in KB) of the chunks in all value dimensions.
            If int or tuple, an explicit specification of the number chunks in
            each value dimension.

        axis : tuple, optional, default=None
            One or more axes to estimate chunks for, if provided any
            other axes will use one chunk.

        padding: tuple or int, default = None
            Number of elements per dimension that will overlap with the adjacent chunk.
            If a tuple, specifies padding along each chunked dimension; if a int, same
            padding will be applied to all chunked dimensions.
        """
        if self.split == len(self.shape) and padding is None:
            self._rdd = self._rdd.map(lambda kv: (kv[0]+(0,), array(kv[1], ndmin=1)))
            self._shape = self._shape + (1,)
            self._plan = (1,)
            self._padding = array([0])
            return self

        rdd = self._rdd
        self._plan, self._padding = self.getplan(size, axis, padding)

        if any([x + y > z for x, y, z in zip(self.plan, self.padding, self.vshape)]):
            raise ValueError("Chunk sizes %s plus padding sizes %s cannot exceed value dimensions %s along any axis"
                             % (tuple(self.plan), tuple(self.padding), tuple(self.vshape)))

        if any([x > y for x, y in zip(self.padding, self.plan)]):
            raise ValueError("Padding sizes %s cannot exceed chunk sizes %s along any axis"
                             % (tuple(self.padding), tuple(self.plan)))

        slices = self.getslices(self.plan, self.padding, self.vshape)
        labels = list(product(*[list(enumerate(s)) for s in slices]))
        scheme = [list(zip(*s)) for s in labels]

        def _chunk(record):
            k, v = record[0], record[1]
            for (chk, slc) in scheme:
                if type(k) is int:
                    k = (k,)
                yield k + chk, v[slc]

        rdd = rdd.flatMap(_chunk)
        return self._constructor(rdd, shape=self.shape, split=self.split,
                                 dtype=self.dtype, plan=self.plan, padding=self.padding, ordered=self._ordered)

Split values of distributed array into chunks .


def map(self, func, value_shape=None, dtype=None):
        """
        Apply an array -> array function on each subarray.

        The function can change the shape of the subarray, but only along
        dimensions that are not chunked.

        Parameters
        ----------
        func : function
            Function of a single subarray to apply

        value_shape:
            Known shape of chunking plan after the map

        dtype: numpy.dtype, optional, default=None
            Known dtype of values resulting from operation

        Returns
        -------
        ChunkedArray
        """

        if value_shape is None or dtype is None:
            # try to compute the size of each mapped element by applying func to a random array
            try:
                mapped = func(random.randn(*self.plan).astype(self.dtype))
            except Exception:
                first = self._rdd.first()
                if first:
                    # eval func on the first element
                    mapped = func(first[1])
            if value_shape is None:
                value_shape = mapped.shape
            if dtype is None:
                dtype = mapped.dtype

        chunked_dims = where(self.plan != self.vshape)[0]
        unchunked_dims = where(self.plan == self.vshape)[0]

        # check that no dimensions are dropped
        if len(value_shape) != len(self.plan):
            raise NotImplementedError('map on ChunkedArray cannot drop dimensions')

        # check that chunked dimensions did not change shape
        if any([value_shape[i] != self.plan[i] for i in chunked_dims]):
            raise ValueError('map cannot change the sizes of chunked dimensions')

        def check_and_apply(v):
            new = func(v)
            if len(unchunked_dims) > 0:
                if any([new.shape[i] != value_shape[i] for i in unchunked_dims]):
                    raise Exception("Map operation did not produce values of uniform shape.")
            if len(chunked_dims) > 0:
                if any([v.shape[i] != new.shape[i] for i in chunked_dims]):
                    raise Exception("Map operation changed the size of a chunked dimension")
            return new

        rdd = self._rdd.mapValues(check_and_apply)

        vshape = [value_shape[i] if i in unchunked_dims else self.vshape[i] for i in range(len(self.vshape))]
        newshape = r_[self.kshape, vshape].astype(int).tolist()

        return self._constructor(rdd, shape=tuple(newshape), dtype=dtype,
                                 plan=asarray(value_shape)).__finalize__(self)

Apply an array - > array function on each subarray .


def map_generic(self, func):
        """
        Apply a generic array -> object to each subarray

        The resulting object is a BoltArraySpark of dtype object where the
        blocked dimensions are replaced with indices indication block ID.
        """
        def process_record(val):
            newval = empty(1, dtype="object")
            newval[0]  = func(val)
            return newval

        rdd = self._rdd.mapValues(process_record)

        nchunks = self.getnumber(self.plan, self.vshape)
        newshape = tuple([int(s) for s in r_[self.kshape, nchunks]])
        newsplit = len(self.shape)
        return BoltArraySpark(rdd, shape=newshape, split=newsplit, ordered=self._ordered, dtype="object")

def map_generic ( self , func ) : def process_record ( val ) : newval = empty ( 1 , dtype = "object" ) newval [ 0 ] = func ( val ) return newval rdd = self . _rdd . mapValues ( process_record ) nchunks = self . getnumber ( self . plan , self . vshape ) newshape = tuple ( [ int ( s ) for s in r_ [ self . kshape , nchunks ] ] ) newsplit = len ( self . shape ) return BoltArraySpark ( rdd , shape = newshape , split = newsplit , ordered = self . _ordered , dtype = "object" )

Apply a generic array - > object to each subarray


def getplan(self, size="150", axes=None, padding=None):
        """
        Identify a plan for chunking values along each dimension.

        Generates an ndarray with the size (in number of elements) of chunks
        in each dimension. If provided, will estimate chunks for only a
        subset of axes, leaving all others to the full size of the axis.

        Parameters
        ----------
        size : string or tuple
             If str, the average size (in KB) of the chunks in all value dimensions.
             If int/tuple, an explicit specification of the number chunks in
             each moving value dimension.

        axes : tuple, optional, default=None
              One or more axes to estimate chunks for, if provided any
              other axes will use one chunk.

        padding : tuple or int, option, default=None
            Size over overlapping padding between chunks in each dimension.
            If tuple, specifies padding along each chunked dimension; if int,
            all dimensions use same padding; if None, no padding
        """
        from numpy import dtype as gettype

        # initialize with all elements in one chunk
        plan = self.vshape

        # check for subset of axes
        if axes is None:
            if isinstance(size, str):
                axes = arange(len(self.vshape))
            else:
                axes = arange(len(size))
        else:
            axes = asarray(axes, 'int')

        # set padding
        pad = array(len(self.vshape)*[0, ])
        if padding is not None:
            pad[axes] = padding

        # set the plan
        if isinstance(size, tuple):
            plan[axes] = size

        elif isinstance(size, str):
            # convert from kilobytes
            size = 1000.0 * float(size)

            # calculate from dtype
            elsize = gettype(self.dtype).itemsize
            nelements = prod(self.vshape)
            dims = self.vshape[self.vmask(axes)]

            if size <= elsize:
                s = ones(len(axes))

            else:
                remsize = 1.0 * nelements * elsize
                s = []
                for (i, d) in enumerate(dims):
                    minsize = remsize/d
                    if minsize >= size:
                        s.append(1)
                        remsize = minsize
                        continue
                    else:
                        s.append(min(d, floor(size/minsize)))
                        s[i+1:] = plan[i+1:]
                        break

            plan[axes] = s

        else:
            raise ValueError("Chunk size not understood, must be tuple or int")

        return plan, pad

Identify a plan for chunking values along each dimension .


def removepad(idx, value, number, padding, axes=None):
        """
        Remove the padding from chunks.

        Given a chunk and its corresponding index, use the plan and padding to remove any
        padding from the chunk along with specified axes.

        Parameters
        ----------
        idx: tuple or array-like
            The chunk index, indicating which chunk this is.

        value: ndarray
            The chunk that goes along with the index.

        number: ndarray or array-like
            The number of chunks along each dimension.

        padding: ndarray or array-like
            The padding scheme.

        axes: tuple, optional, default = None
            The axes (in the values) along which to remove padding.
        """
        if axes is None:
            axes = range(len(number))
        mask = len(number)*[False, ]
        for i in range(len(mask)):
            if i in axes and padding[i] != 0:
                mask[i] = True

        starts = [0 if (i == 0 or not m) else p for (i, m, p) in zip(idx, mask, padding)]
        stops = [None if (i == n-1 or not m) else -p for (i, m, p, n) in zip(idx, mask, padding, number)]
        slices = [slice(i1, i2) for (i1, i2) in zip(starts, stops)]

        return value[slices]

def removepad ( idx , value , number , padding , axes = None ) : if axes is None : axes = range ( len ( number ) ) mask = len ( number ) * [ False , ] for i in range ( len ( mask ) ) : if i in axes and padding [ i ] != 0 : mask [ i ] = True starts = [ 0 if ( i == 0 or not m ) else p for ( i , m , p ) in zip ( idx , mask , padding ) ] stops = [ None if ( i == n - 1 or not m ) else - p for ( i , m , p , n ) in zip ( idx , mask , padding , number ) ] slices = [ slice ( i1 , i2 ) for ( i1 , i2 ) in zip ( starts , stops ) ] return value [ slices ]


def getnumber(plan, shape):
        """
        Obtain number of chunks for the given dimensions and chunk sizes.

        Given a plan for the number of chunks along each dimension,
        calculate the number of chunks that this will lead to.

        Parameters
        ----------
        plan: tuple or array-like
            Size of chunks (in number of elements) along each dimensions.
            Length must be equal to the number of dimensions.

        shape : tuple
             Shape of array to be chunked.
        """
        nchunks = []
        for size, d in zip(plan, shape):
            nchunks.append(int(ceil(1.0 * d/size)))
        return nchunks

def getnumber ( plan , shape ) : nchunks = [ ] for size , d in zip ( plan , shape ) : nchunks . append ( int ( ceil ( 1.0 * d / size ) ) ) return nchunks

Obtain number of chunks for the given dimensions and chunk sizes .


def getslices(plan, padding, shape):
        """
        Obtain slices for the given dimensions, padding, and chunks.

        Given a plan for the number of chunks along each dimension and the amount of padding,
        calculate a list of slices required to generate those chunks.

        Parameters
        ----------
        plan: tuple or array-like
            Size of chunks (in number of elements) along each dimensions.
            Length must be equal to the number of dimensions.

        padding: tuple or array-like
            Size of overlap (in number of elements) between chunks along each dimension.
            Length must be equal to the number of dimensions.

        shape: tuple
             Dimensions of axes to be chunked.
        """
        slices = []
        for size, pad, d in zip(plan, padding, shape):
            nchunks = int(floor(d/size))
            remainder = d % size
            start = 0
            dimslices = []
            for idx in range(nchunks):
                end = start + size
                # left endpoint
                if idx == 0:
                    left = start
                else:
                    left = start - pad
                # right endpoint
                if idx == nchunks:
                    right = end
                else:
                    right = end + pad
                dimslices.append(slice(left, right, 1))
                start = end
            if remainder:
                dimslices.append(slice(end - pad, d, 1))
            slices.append(dimslices)
        return slices

def
getslices
(
plan
,
padding
,
shape
)
:
slices
=
[
]
for
size
,
pad
,
d
in
zip
(
plan
,
padding
,
shape
)
:
nchunks
=
int
(
floor
(
d
/
size
)
)
remainder
=
d
%
size
start
=
0
dimslices
=
[
]
for
idx
in
range
(
nchunks
)
:
end
=
start
+
size
# left endpoint
if
idx
==
0
:
left
=
start
else
:
left
=
start
pad
# right endpoint
if
idx
==
nchunks
:
right
=
end
else
:
right
=
end
+
pad
dimslices
.
append
(
slice
(
left
,
right
,
1
)
)
start
=
end
if
remainder
:
dimslices
.
append
(
slice
(
end
pad
,
d
,
1
)
)
slices
.
append
(
dimslices
)
return
slices

Obtain slices for the given dimensions padding and chunks .


def getmask(inds, n):
        """
        Obtain a binary mask by setting a subset of entries to true.

        Parameters
        ----------
        inds : array-like
            Which indices to set as true.

        n : int
            The length of the target mask.
        """
        inds = asarray(inds, 'int')
        mask = zeros(n, dtype=bool)
        mask[inds] = True
        return mask

def getmask ( inds , n ) : inds = asarray ( inds , 'int' ) mask = zeros ( n , dtype = bool ) mask [ inds ] = True return mask

Obtain a binary mask by setting a subset of entries to true .


def repartition(self, npartitions):
        """
        Repartitions the underlying RDD

        Parameters
        ----------
        npartitions : int
            Number of partitions to repartion the underlying RDD to
        """

        rdd = self._rdd.repartition(npartitions)
        return self._constructor(rdd, ordered=False).__finalize__(self)

def repartition ( self , npartitions ) : rdd = self . _rdd . repartition ( npartitions ) return self . _constructor ( rdd , ordered = False ) . __finalize__ ( self )


def stack(self, size=None):
        """
        Aggregates records of a distributed array.

        Stacking should improve the performance of vectorized operations,
        but the resulting StackedArray object only exposes a restricted set
        of operations (e.g. map, reduce). The unstack method can be used
        to restore the full bolt array.

        Parameters
        ----------
        size : int, optional, default=None
            The maximum size for each stack (number of original records),
            will aggregate groups of records per partition up to this size,
            if None will aggregate all records on each partition.

        Returns
        -------
        StackedArray
        """
        stk = StackedArray(self._rdd, shape=self.shape, split=self.split)
        return stk.stack(size)

def stack ( self , size = None ) : stk = StackedArray ( self . _rdd , shape = self . shape , split = self . split ) return stk . stack ( size )

Aggregates records of a distributed array .


def _align(self, axis):
        """
        Align spark bolt array so that axes for iteration are in the keys.

        This operation is applied before most functional operators.
        It ensures that the specified axes are valid, and swaps
        key/value axes so that functional operators can be applied
        over the correct records.

        Parameters
        ----------
        axis: tuple[int]
            One or more axes that wil be iterated over by a functional operator

        Returns
        -------
        BoltArraySpark
        """
        # ensure that the specified axes are valid
        inshape(self.shape, axis)

        # find the value axes that should be moved into the keys (axis >= split)
        tokeys = [(a - self.split) for a in axis if a >= self.split]

        # find the key axes that should be moved into the values (axis < split)
        tovalues = [a for a in range(self.split) if a not in axis]

        if tokeys or tovalues:
            return self.swap(tovalues, tokeys)
        else:
            return self

Align spark bolt array so that axes for iteration are in the keys .


def first(self):
        """
        Return the first element of an array
        """
        from bolt.local.array import BoltArrayLocal
        rdd = self._rdd if self._ordered else self._rdd.sortByKey()
        return BoltArrayLocal(rdd.values().first())

def first ( self ) : from bolt . local . array import BoltArrayLocal rdd = self . _rdd if self . _ordered else self . _rdd . sortByKey ( ) return BoltArrayLocal ( rdd . values ( ) . first ( ) )


def _stat(self, axis=None, func=None, name=None, keepdims=False):
        """
        Compute a statistic over an axis.

        Can provide either a function (for use in a reduce)
        or a name (for use by a stat counter).

        Parameters
        ----------
        axis : tuple or int, optional, default=None
            Axis to compute statistic over, if None
            will compute over all axes

        func : function, optional, default=None
            Function for reduce, see BoltArraySpark.reduce

        name : str
            A named statistic, see StatCounter

        keepdims : boolean, optional, default=False
            Keep axis remaining after operation with size 1.
        """
        if axis is None:
            axis = list(range(len(self.shape)))
        axis = tupleize(axis)

        if func and not name:
            return self.reduce(func, axis, keepdims)

        if name and not func:
            from bolt.local.array import BoltArrayLocal

            swapped = self._align(axis)

            def reducer(left, right):
                return left.combine(right)

            counter = swapped._rdd.values()\
                             .mapPartitions(lambda i: [StatCounter(values=i, stats=name)])\
                             .treeReduce(reducer, depth=3)

            arr = getattr(counter, name)

            if keepdims:
                for i in axis:
                    arr = expand_dims(arr, axis=i)

            return BoltArrayLocal(arr).toscalar()

        else:
            raise ValueError('Must specify either a function or a statistic name.')


def mean(self, axis=None, keepdims=False):
        """
        Return the mean of the array over the given axis.

        Parameters
        ----------
        axis : tuple or int, optional, default=None
            Axis to compute statistic over, if None
            will compute over all axes

        keepdims : boolean, optional, default=False
            Keep axis remaining after operation with size 1.
        """
        return self._stat(axis, name='mean', keepdims=keepdims)

def mean ( self , axis = None , keepdims = False ) : return self . _stat ( axis , name = 'mean' , keepdims = keepdims )

Return the mean of the array over the given axis .


def var(self, axis=None, keepdims=False):
        """
        Return the variance of the array over the given axis.

        Parameters
        ----------
        axis : tuple or int, optional, default=None
            Axis to compute statistic over, if None
            will compute over all axes

        keepdims : boolean, optional, default=False
            Keep axis remaining after operation with size 1.
        """
        return self._stat(axis, name='variance', keepdims=keepdims)

def var ( self , axis = None , keepdims = False ) : return self . _stat ( axis , name = 'variance' , keepdims = keepdims )

Return the variance of the array over the given axis .


def std(self, axis=None, keepdims=False):
        """
        Return the standard deviation of the array over the given axis.

        Parameters
        ----------
        axis : tuple or int, optional, default=None
            Axis to compute statistic over, if None
            will compute over all axes

        keepdims : boolean, optional, default=False
            Keep axis remaining after operation with size 1.
        """
        return self._stat(axis, name='stdev', keepdims=keepdims)

def std ( self , axis = None , keepdims = False ) : return self . _stat ( axis , name = 'stdev' , keepdims = keepdims )

Return the standard deviation of the array over the given axis .


def sum(self, axis=None, keepdims=False):
        """
        Return the sum of the array over the given axis.

        Parameters
        ----------
        axis : tuple or int, optional, default=None
            Axis to compute statistic over, if None
            will compute over all axes

        keepdims : boolean, optional, default=False
            Keep axis remaining after operation with size 1.
        """
        from operator import add
        return self._stat(axis, func=add, keepdims=keepdims)

def sum ( self , axis = None , keepdims = False ) : from operator import add return self . _stat ( axis , func = add , keepdims = keepdims )

Return the sum of the array over the given axis .


def max(self, axis=None, keepdims=False):
        """
        Return the maximum of the array over the given axis.

        Parameters
        ----------
        axis : tuple or int, optional, default=None
            Axis to compute statistic over, if None
            will compute over all axes

        keepdims : boolean, optional, default=False
            Keep axis remaining after operation with size 1.
        """
        from numpy import maximum
        return self._stat(axis, func=maximum, keepdims=keepdims)

def max ( self , axis = None , keepdims = False ) : from numpy import maximum return self . _stat ( axis , func = maximum , keepdims = keepdims )

Return the maximum of the array over the given axis .


def min(self, axis=None, keepdims=False):
        """
        Return the minimum of the array over the given axis.

        Parameters
        ----------
        axis : tuple or int, optional, default=None
            Axis to compute statistic over, if None
            will compute over all axes

        keepdims : boolean, optional, default=False
            Keep axis remaining after operation with size 1.
        """
        from numpy import minimum
        return self._stat(axis, func=minimum, keepdims=keepdims)

def min ( self , axis = None , keepdims = False ) : from numpy import minimum return self . _stat ( axis , func = minimum , keepdims = keepdims )

Return the minimum of the array over the given axis .


def chunk(self, size="150", axis=None, padding=None):
        """
        Chunks records of a distributed array.

        Chunking breaks arrays into subarrays, using an specified
        size of chunks along each value dimension. Can alternatively
        specify an average chunk byte size (in kilobytes) and the size of
        chunks (as ints) will be computed automatically.

        Parameters
        ----------
        size : tuple, int, or str, optional, default = "150"
            A string giving the size in kilobytes, or a tuple with the size
            of chunks along each dimension.

        axis : int or tuple, optional, default = None
            One or more axis to chunk array along, if None
            will use all axes,

        padding: tuple or int, default = None
            Number of elements per dimension that will overlap with the adjacent chunk.
            If a tuple, specifies padding along each chunked dimension; if a int, same
            padding will be applied to all chunked dimensions.

        Returns
        -------
        ChunkedArray
        """
        if type(size) is not str:
            size = tupleize((size))
        axis = tupleize((axis))
        padding = tupleize((padding))

        from bolt.spark.chunk import ChunkedArray

        chnk = ChunkedArray(rdd=self._rdd, shape=self._shape, split=self._split, dtype=self._dtype)
        return chnk._chunk(size, axis, padding)

def chunk ( self , size = "150" , axis = None , padding = None ) : if type ( size ) is not str : size = tupleize ( ( size ) ) axis = tupleize ( ( axis ) ) padding = tupleize ( ( padding ) ) from bolt . spark . chunk import ChunkedArray chnk = ChunkedArray ( rdd = self . _rdd , shape = self . _shape , split = self . _split , dtype = self . _dtype ) return chnk . _chunk ( size , axis , padding )


def swap(self, kaxes, vaxes, size="150"):
        """
        Swap axes from keys to values.

        This is the core operation underlying shape manipulation
        on the Spark bolt array. It exchanges an arbitrary set of axes
        between the keys and the valeus. If either is None, will only
        move axes in one direction (from keys to values, or values to keys).
        Keys moved to values will be placed immediately after the split;
        values moved to keys will be placed immediately before the split.

        Parameters
        ----------
        kaxes : tuple
            Axes from keys to move to values

        vaxes : tuple
            Axes from values to move to keys

        size : tuple or int, optional, default = "150"
            Can either provide a string giving the size in kilobytes,
            or a tuple with the number of chunks along each
            value dimension being moved

        Returns
        -------
        BoltArraySpark
        """
        kaxes = asarray(tupleize(kaxes), 'int')
        vaxes = asarray(tupleize(vaxes), 'int')
        if type(size) is not str:
            size = tupleize(size)

        if len(kaxes) == self.keys.ndim and len(vaxes) == 0:
            raise ValueError('Cannot perform a swap that would '
                             'end up with all data on a single key')

        if len(kaxes) == 0 and len(vaxes) == 0:
            return self

        from bolt.spark.chunk import ChunkedArray

        chunks = self.chunk(size)

        swapped = chunks.keys_to_values(kaxes).values_to_keys([v+len(kaxes) for v in vaxes])
        barray = swapped.unchunk()

        return barray

def
swap
(
self
,
kaxes
,
vaxes
,
size
=
"150"
)
:
kaxes
=
asarray
(
tupleize
(
kaxes
)
,
'int'
)
vaxes
=
asarray
(
tupleize
(
vaxes
)
,
'int'
)
if
type
(
size
)
is
not
str
:
size
=
tupleize
(
size
)
if
len
(
kaxes
)
==
self
.
keys
.
ndim
and
len
(
vaxes
)
==
0
:
raise
ValueError
(
'Cannot perform a swap that would '
'end up with all data on a single key'
)
if
len
(
kaxes
)
==
0
and
len
(
vaxes
)
==
0
:
return
self
from
bolt
.
spark
.
chunk
import
ChunkedArray
chunks
=
self
.
chunk
(
size
)
swapped
=
chunks
.
keys_to_values
(
kaxes
)
.
values_to_keys
(
[
v
+
len
(
kaxes
)
for
v
in
vaxes
]
)
barray
=
swapped
.
unchunk
(
)
return
barray


def transpose(self, *axes):
        """
        Return an array with the axes transposed.

        This operation will incur a swap unless the
        desiured permutation can be obtained
        only by transpoing the keys or the values.

        Parameters
        ----------
        axes : None, tuple of ints, or n ints
            If None, will reverse axis order.
        """
        if len(axes) == 0:
            p = arange(self.ndim-1, -1, -1)
        else:
            p = asarray(argpack(axes))

        istransposeable(p, range(self.ndim))

        split = self.split

        # compute the keys/value axes that need to be swapped
        new_keys, new_values = p[:split], p[split:]
        swapping_keys = sort(new_values[new_values < split])
        swapping_values = sort(new_keys[new_keys >= split])
        stationary_keys = sort(new_keys[new_keys < split])
        stationary_values = sort(new_values[new_values >= split])

        # compute the permutation that the swap causes
        p_swap = r_[stationary_keys, swapping_values, swapping_keys, stationary_values]

        # compute the extra permutation (p_x)  on top of this that
        # needs to happen to get the full permutation desired
        p_swap_inv = argsort(p_swap)
        p_x = p_swap_inv[p]
        p_keys, p_values = p_x[:split], p_x[split:]-split

        # perform the swap and the the within key/value permutations
        arr = self.swap(swapping_keys, swapping_values-split)
        arr = arr.keys.transpose(tuple(p_keys.tolist()))
        arr = arr.values.transpose(tuple(p_values.tolist()))

        return arr

Return an array with the axes transposed .


def swapaxes(self, axis1, axis2):
        """
        Return the array with two axes interchanged.

        Parameters
        ----------
        axis1 : int
            The first axis to swap

        axis2 : int
            The second axis to swap
        """
        p = list(range(self.ndim))
        p[axis1] = axis2
        p[axis2] = axis1

        return self.transpose(p)

def swapaxes ( self , axis1 , axis2 ) : p = list ( range ( self . ndim ) ) p [ axis1 ] = axis2 p [ axis2 ] = axis1 return self . transpose ( p )

Return the array with two axes interchanged .


def reshape(self, *shape):
        """
        Return an array with the same data but a new shape.

        Currently only supports reshaping that independently
        reshapes the keys, or the values, or both.

        Parameters
        ----------
        shape :  tuple of ints, or n ints
            New shape
        """
        new = argpack(shape)
        isreshapeable(new, self.shape)

        if new == self.shape:
            return self

        i = self._reshapebasic(new)
        if i == -1:
            raise NotImplementedError("Currently no support for reshaping between "
                                      "keys and values for BoltArraySpark")
        else:
            new_key_shape, new_value_shape = new[:i], new[i:]
            return self.keys.reshape(new_key_shape).values.reshape(new_value_shape)

Return an array with the same data but a new shape .


def _reshapebasic(self, shape):
        """
        Check if the requested reshape can be broken into independant reshapes
        on the keys and values. If it can, returns the index in the new shape
        separating keys from values, otherwise returns -1
        """
        new = tupleize(shape)
        old_key_size = prod(self.keys.shape)
        old_value_size = prod(self.values.shape)

        for i in range(len(new)):
            new_key_size = prod(new[:i])
            new_value_size = prod(new[i:])
            if new_key_size == old_key_size and new_value_size == old_value_size:
                return i

        return -1

def _reshapebasic ( self , shape ) : new = tupleize ( shape ) old_key_size = prod ( self . keys . shape ) old_value_size = prod ( self . values . shape ) for i in range ( len ( new ) ) : new_key_size = prod ( new [ : i ] ) new_value_size = prod ( new [ i : ] ) if new_key_size == old_key_size and new_value_size == old_value_size : return i return - 1

Check if the requested reshape can be broken into independant reshapes on the keys and values . If it can returns the index in the new shape separating keys from values otherwise returns - 1


def squeeze(self, axis=None):
        """
        Remove one or more single-dimensional axes from the array.

        Parameters
        ----------
        axis : tuple or int
            One or more singleton axes to remove.
        """
        if not any([d == 1 for d in self.shape]):
            return self

        if axis is None:
            drop = where(asarray(self.shape) == 1)[0]
        elif isinstance(axis, int):
            drop = asarray((axis,))
        elif isinstance(axis, tuple):
            drop = asarray(axis)
        else:
            raise ValueError("an integer or tuple is required for the axis")

        if any([self.shape[i] > 1 for i in drop]):
            raise ValueError("cannot select an axis to squeeze out which has size greater than one")

        if any(asarray(drop) < self.split):
            kmask = set([d for d in drop if d < self.split])
            kfunc = lambda k: tuple([kk for ii, kk in enumerate(k) if ii not in kmask])
        else:
            kfunc = lambda k: k

        if any(asarray(drop) >= self.split):
            vmask = tuple([d - self.split for d in drop if d >= self.split])
            vfunc = lambda v: v.squeeze(vmask)
        else:
            vfunc = lambda v: v

        rdd = self._rdd.map(lambda kv: (kfunc(kv[0]), vfunc(kv[1])))
        shape = tuple([ss for ii, ss in enumerate(self.shape) if ii not in drop])
        split = len([d for d in range(self.keys.ndim) if d not in drop])
        return self._constructor(rdd, shape=shape, split=split).__finalize__(self)

Remove one or more single - dimensional axes from the array .


def astype(self, dtype, casting='unsafe'):
        """
        Cast the array to a specified type.

        Parameters
        ----------
        dtype : str or dtype
            Typecode or data-type to cast the array to (see numpy)
        """
        rdd = self._rdd.mapValues(lambda v: v.astype(dtype, 'K', casting))
        return self._constructor(rdd, dtype=dtype).__finalize__(self)

def astype ( self , dtype , casting = 'unsafe' ) : rdd = self . _rdd . mapValues ( lambda v : v . astype ( dtype , 'K' , casting ) ) return self . _constructor ( rdd , dtype = dtype ) . __finalize__ ( self )


def clip(self, min=None, max=None):
        """
        Clip values above and below.

        Parameters
        ----------
        min : scalar or array-like
            Minimum value. If array, will be broadcasted

        max : scalar or array-like
            Maximum value. If array, will be broadcasted.
        """
        rdd = self._rdd.mapValues(lambda v: v.clip(min=min, max=max))
        return self._constructor(rdd).__finalize__(self)

def clip ( self , min = None , max = None ) : rdd = self . _rdd . mapValues ( lambda v : v . clip ( min = min , max = max ) ) return self . _constructor ( rdd ) . __finalize__ ( self )


def toarray(self):
        """
        Returns the contents as a local array.

        Will likely cause memory problems for large objects.
        """
        rdd = self._rdd if self._ordered else self._rdd.sortByKey()
        x = rdd.values().collect()
        return asarray(x).reshape(self.shape)

def toarray ( self ) : rdd = self . _rdd if self . _ordered else self . _rdd . sortByKey ( ) x = rdd . values ( ) . collect ( ) return asarray ( x ) . reshape ( self . shape )


def tupleize(arg):
    """
    Coerce singletons and lists and ndarrays to tuples.

    Parameters
    ----------
    arg : tuple, list, ndarray, or singleton
        Item to coerce
    """
    if arg is None:
        return None
    if not isinstance(arg, (tuple, list, ndarray, Iterable)):
        return tuple((arg,))
    elif isinstance(arg, (list, ndarray)):
        return tuple(arg)
    elif isinstance(arg, Iterable) and not isinstance(arg, str):
        return tuple(arg)
    else:
        return arg

def tupleize ( arg ) : if arg is None : return None if not isinstance ( arg , ( tuple , list , ndarray , Iterable ) ) : return tuple ( ( arg , ) ) elif isinstance ( arg , ( list , ndarray ) ) : return tuple ( arg ) elif isinstance ( arg , Iterable ) and not isinstance ( arg , str ) : return tuple ( arg ) else : return arg

Coerce singletons and lists and ndarrays to tuples .


def argpack(args):
    """
    Coerce a list of arguments to a tuple.

    Parameters
    ----------
    args : tuple or nested tuple
        Pack arguments into a tuple, converting ((,...),) or (,) -> (,)
    """
    if isinstance(args[0], (tuple, list, ndarray)):
        return tupleize(args[0])
    elif isinstance(args[0], Iterable) and not isinstance(args[0], str):
        # coerce any iterable into a list before calling tupleize (Python 3 compatibility)
        return tupleize(list(args[0]))
    else:
        return tuple(args)


def inshape(shape, axes):
    """
    Checks to see if a list of axes are contained within an array shape.

    Parameters
    ----------
    shape : tuple[int]
        the shape of a BoltArray

    axes : tuple[int]
        the axes to check against shape
    """
    valid = all([(axis < len(shape)) and (axis >= 0) for axis in axes])
    if not valid:
        raise ValueError("axes not valid for an ndarray of shape: %s" % str(shape))

Checks to see if a list of axes are contained within an array shape .


def allclose(a, b):
    """
    Test that a and b are close and match in shape.

    Parameters
    ----------
    a : ndarray
        First array to check

    b : ndarray
        First array to check
    """
    from numpy import allclose
    return (a.shape == b.shape) and allclose(a, b)

def allclose ( a , b ) : from numpy import allclose return ( a . shape == b . shape ) and allclose ( a , b )

Test that a and b are close and match in shape .


def listify(lst, dim):
    """
    Flatten lists of indices and ensure bounded by a known dim.

    Parameters
    ----------
    lst : list
        List of integer indices

    dim : tuple
        Bounds for indices
    """
    if not all([l.dtype == int for l in lst]):
        raise ValueError("indices must be integers")

    if npany(asarray(lst) >= dim):
        raise ValueError("indices out of bounds for axis with size %s" % dim)

    return lst.flatten()

Flatten lists of indices and ensure bounded by a known dim .


def slicify(slc, dim):
    """
    Force a slice to have defined start, stop, and step from a known dim.
    Start and stop will always be positive. Step may be negative.

    There is an exception where a negative step overflows the stop needs to have
    the default value set to -1. This is the only case of a negative start/stop
    value.

    Parameters
    ----------
    slc : slice or int
        The slice to modify, or int to convert to a slice

    dim : tuple
        Bound for slice
    """
    if isinstance(slc, slice):

        # default limits
        start = 0 if slc.start is None else slc.start
        stop = dim if slc.stop is None else slc.stop
        step = 1 if slc.step is None else slc.step
        # account for negative indices
        if start < 0: start += dim
        if stop < 0: stop += dim
        # account for over-flowing the bounds
        if step > 0:
            if start < 0: start = 0
            if stop > dim: stop = dim
        else:
            if stop < 0: stop = -1
            if start > dim: start = dim-1

        return slice(start, stop, step)

    elif isinstance(slc, int):
        if slc < 0:
            slc += dim
        return slice(slc, slc+1, 1)

    else:
        raise ValueError("Type for slice %s not recongized" % type(slc))

def
slicify
(
slc
,
dim
)
:
if
isinstance
(
slc
,
slice
)
:
# default limits
start
=
0
if
slc
.
start
is
else
slc
.
start
stop
=
dim
if
slc
.
stop
is
else
slc
.
stop
step
=
1
if
slc
.
step
is
else
slc
.
step
# account for negative indices
if
start
<
0
:
start
+=
dim
if
stop
<
0
:
stop
+=
dim
# account for over-flowing the bounds
if
step
>
0
:
if
start
<
0
:
start
=
0
if
stop
>
dim
:
stop
=
dim
else
:
if
stop
<
0
:
stop
=
1
if
start
>
dim
:
start
=
dim
1
return
slice
(
start
,
stop
,
step
)
elif
isinstance
(
slc
,
int
)
:
if
slc
<
0
:
slc
+=
dim
return
slice
(
slc
,
slc
+
1
,
1
)
else
:
raise
ValueError
(
"Type for slice %s not recongized"
%
type
(
slc
)
)

Force a slice to have defined start stop and step from a known dim . Start and stop will always be positive . Step may be negative .


def istransposeable(new, old):
    """
    Check to see if a proposed tuple of axes is a valid permutation
    of an old set of axes. Checks length, axis repetion, and bounds.

    Parameters
    ----------
    new : tuple
        tuple of proposed axes

    old : tuple
        tuple of old axes
    """

    new, old = tupleize(new), tupleize(old)

    if not len(new) == len(old):
        raise ValueError("Axes do not match axes of keys")

    if not len(set(new)) == len(set(old)):
        raise ValueError("Repeated axes")

    if any(n < 0 for n in new) or max(new) > len(old) - 1:
        raise ValueError("Invalid axes")

def
istransposeable
(
new
,
old
)
:
new
,
old
=
tupleize
(
new
)
,
tupleize
(
old
)
if
not
len
(
new
)
==
len
(
old
)
:
raise
ValueError
(
"Axes do not match axes of keys"
)
if
not
len
(
set
(
new
)
)
==
len
(
set
(
old
)
)
:
raise
ValueError
(
"Repeated axes"
)
if
any
(
n
<
0
for
n
in
new
)
or
max
(
new
)
>
len
(
old
)
1
:
raise
ValueError
(
"Invalid axes"
)

Check to see if a proposed tuple of axes is a valid permutation of an old set of axes . Checks length axis repetion and bounds .


def isreshapeable(new, old):
    """
    Check to see if a proposed tuple of axes is a valid reshaping of
    the old axes by ensuring that they can be factored.

    Parameters
    ----------
    new : tuple
        tuple of proposed axes

    old : tuple
        tuple of old axes
    """

    new, old = tupleize(new), tupleize(old)

    if not prod(new) == prod(old):
        raise ValueError("Total size of new keys must remain unchanged")

Check to see if a proposed tuple of axes is a valid reshaping of the old axes by ensuring that they can be factored .


def allstack(vals, depth=0):
    """
    If an ndarray has been split into multiple chunks by splitting it along
    each axis at a number of locations, this function rebuilds the
    original array from chunks.

    Parameters
    ----------
    vals : nested lists of ndarrays
        each level of nesting of the lists representing a dimension of
        the original array.
    """
    if type(vals[0]) is ndarray:
        return concatenate(vals, axis=depth)
    else:
        return concatenate([allstack(x, depth+1) for x in vals], axis=depth)

def allstack ( vals , depth = 0 ) : if type ( vals [ 0 ] ) is ndarray : return concatenate ( vals , axis = depth ) else : return concatenate ( [ allstack ( x , depth + 1 ) for x in vals ] , axis = depth )

If an ndarray has been split into multiple chunks by splitting it along each axis at a number of locations this function rebuilds the original array from chunks .


def iterexpand(arry, extra):
    """
    Expand dimensions by iteratively append empty axes.

    Parameters
    ----------
    arry : ndarray
        The original array

    extra : int
        The number of empty axes to append
    """
    for d in range(arry.ndim, arry.ndim+extra):
        arry = expand_dims(arry, axis=d)
    return arry

def iterexpand ( arry , extra ) : for d in range ( arry . ndim , arry . ndim + extra ) : arry = expand_dims ( arry , axis = d ) return arry

Expand dimensions by iteratively append empty axes .


def zip_with_index(rdd):
    """
    Alternate version of Spark's zipWithIndex that eagerly returns count.
    """
    starts = [0]
    if rdd.getNumPartitions() > 1:
        nums = rdd.mapPartitions(lambda it: [sum(1 for _ in it)]).collect()
        count = sum(nums)
        for i in range(len(nums) - 1):
            starts.append(starts[-1] + nums[i])
    else:
        count = rdd.count()

    def func(k, it):
        for i, v in enumerate(it, starts[k]):
            yield v, i

    return count, rdd.mapPartitionsWithIndex(func)

def zip_with_index ( rdd ) : starts = [ 0 ] if rdd . getNumPartitions ( ) > 1 : nums = rdd . mapPartitions ( lambda it : [ sum ( 1 for _ in it ) ] ) . collect ( ) count = sum ( nums ) for i in range ( len ( nums ) - 1 ) : starts . append ( starts [ - 1 ] + nums [ i ] ) else : count = rdd . count ( ) def func ( k , it ) : for i , v in enumerate ( it , starts [ k ] ) : yield v , i return count , rdd . mapPartitionsWithIndex ( func )

Alternate version of Spark s zipWithIndex that eagerly returns count .


def wrapped(f):
    """
    Decorator to append routed docstrings
    """
    import inspect

    def extract(func):
        append = ""
        args = inspect.getargspec(func)
        for i, a in enumerate(args.args):
            if i < (len(args) - len(args.defaults)):
                append += str(a) + ", "
            else:
                default = args.defaults[i-len(args.defaults)]
                if hasattr(default, "__name__"):
                    default = default.__name__
                else:
                    default = str(default)
                append += str(a) + "=" + default + ", "
        append = append[:-2] + ")"
        return append

    doc = f.__doc__ + "\n"
    doc += "    local -> array(" + extract(getattr(ConstructLocal, f.__name__)) + "\n"
    doc += "    spark -> array(" + extract(getattr(ConstructSpark, f.__name__)) + "\n"
    f.__doc__ = doc
    return f

def
wrapped
(
f
)
:
import
inspect
def
extract
(
func
)
:
append
=
""
args
=
inspect
.
getargspec
(
func
)
for
i
,
a
in
enumerate
(
args
.
args
)
:
if
i
<
(
len
(
args
)
len
(
args
.
defaults
)
)
:
append
+=
str
(
a
)
+
", "
else
:
default
=
args
.
defaults
[
i
len
(
args
.
defaults
)
]
if
hasattr
(
default
,
"__name__"
)
:
default
=
default
.
__name__
else
:
default
=
str
(
default
)
append
+=
str
(
a
)
+
"="
+
default
+
", "
append
=
append
[
:
2
]
+
")"
return
append
doc
=
f
.
__doc__
+
"\n"
doc
+=
"    local -> array("
+
extract
(
getattr
(
ConstructLocal
,
f
.
__name__
)
)
+
"\n"
doc
+=
"    spark -> array("
+
extract
(
getattr
(
ConstructSpark
,
f
.
__name__
)
)
+
"\n"
f
.
__doc__
=
doc
return
f


def lookup(*args, **kwargs):
    """
    Use arguments to route constructor.

    Applies a series of checks on arguments to identify constructor,
    starting with known keyword arguments, and then applying
    constructor-specific checks
    """
    if 'mode' in kwargs:
        mode = kwargs['mode']
        if mode not in constructors:
            raise ValueError('Mode %s not supported' % mode)
        del kwargs['mode']
        return constructors[mode]
    else:
        for mode, constructor in constructors:
            if constructor._argcheck(*args, **kwargs):
                return constructor
    return ConstructLocal

def
lookup
(
*
args
,
*
*
kwargs
)
:
if
'mode'
in
kwargs
:
mode
=
kwargs
[
'mode'
]
if
mode
not
in
constructors
:
raise
ValueError
(
'Mode %s not supported'
%
mode
)
del
kwargs
[
'mode'
]
return
constructors
[
mode
]
else
:
for
mode
,
constructor
in
constructors
:
if
constructor
.
_argcheck
(
*
args
,
*
*
kwargs
)
:
return
constructor
return
ConstructLocal


def reshape(self, *shape):
        """
        Reshape just the keys of a BoltArraySpark, returning a
        new BoltArraySpark.

        Parameters
        ----------
        shape : tuple
              New proposed axes.
        """
        new = argpack(shape)
        old = self.shape
        isreshapeable(new, old)

        if new == old:
            return self._barray

        def f(k):
            return unravel_index(ravel_multi_index(k, old), new)

        newrdd = self._barray._rdd.map(lambda kv: (f(kv[0]), kv[1]))
        newsplit = len(new)
        newshape = new + self._barray.values.shape

        return BoltArraySpark(newrdd, shape=newshape, split=newsplit).__finalize__(self._barray)

def reshape ( self , * shape ) : new = argpack ( shape ) old = self . shape isreshapeable ( new , old ) if new == old : return self . _barray def f ( k ) : return unravel_index ( ravel_multi_index ( k , old ) , new ) newrdd = self . _barray . _rdd . map ( lambda kv : ( f ( kv [ 0 ] ) , kv [ 1 ] ) ) newsplit = len ( new ) newshape = new + self . _barray . values . shape return BoltArraySpark ( newrdd , shape = newshape , split = newsplit ) . __finalize__ ( self . _barray )

Reshape just the keys of a BoltArraySpark returning a new BoltArraySpark .


def transpose(self, *axes):
        """
        Transpose just the keys of a BoltArraySpark, returning a
        new BoltArraySpark.

        Parameters
        ----------
        axes : tuple
             New proposed axes.
        """
        new = argpack(axes)
        old = range(self.ndim)
        istransposeable(new, old)

        if new == old:
            return self._barray

        def f(k):
            return tuple(k[i] for i in new)

        newrdd = self._barray._rdd.map(lambda kv: (f(kv[0]), kv[1]))
        newshape = tuple(self.shape[i] for i in new) + self._barray.values.shape

        return BoltArraySpark(newrdd, shape=newshape, ordered=False).__finalize__(self._barray)

def transpose ( self , * axes ) : new = argpack ( axes ) old = range ( self . ndim ) istransposeable ( new , old ) if new == old : return self . _barray def f ( k ) : return tuple ( k [ i ] for i in new ) newrdd = self . _barray . _rdd . map ( lambda kv : ( f ( kv [ 0 ] ) , kv [ 1 ] ) ) newshape = tuple ( self . shape [ i ] for i in new ) + self . _barray . values . shape return BoltArraySpark ( newrdd , shape = newshape , ordered = False ) . __finalize__ ( self . _barray )

Transpose just the keys of a BoltArraySpark returning a new BoltArraySpark .


def reshape(self, *shape):
        """
        Reshape just the values of a BoltArraySpark, returning a
        new BoltArraySpark.

        Parameters
        ----------
        shape : tuple
              New proposed axes.
        """
        new = argpack(shape)
        old = self.shape
        isreshapeable(new, old)

        if new == old:
            return self._barray

        def f(v):
            return v.reshape(new)

        newrdd = self._barray._rdd.mapValues(f)
        newshape = self._barray.keys.shape + new

        return BoltArraySpark(newrdd, shape=newshape).__finalize__(self._barray)

def reshape ( self , * shape ) : new = argpack ( shape ) old = self . shape isreshapeable ( new , old ) if new == old : return self . _barray def f ( v ) : return v . reshape ( new ) newrdd = self . _barray . _rdd . mapValues ( f ) newshape = self . _barray . keys . shape + new return BoltArraySpark ( newrdd , shape = newshape ) . __finalize__ ( self . _barray )

Reshape just the values of a BoltArraySpark returning a new BoltArraySpark .


def transpose(self, *axes):
        """
        Transpose just the values of a BoltArraySpark, returning a
        new BoltArraySpark.

        Parameters
        ----------
        axes : tuple
             New proposed axes.
        """
        new = argpack(axes)
        old = range(self.ndim)
        istransposeable(new, old)

        if new == old:
            return self._barray

        def f(v):
            return v.transpose(new)

        newrdd = self._barray._rdd.mapValues(f)
        newshape = self._barray.keys.shape + tuple(self.shape[i] for i in new)

        return BoltArraySpark(newrdd, shape=newshape).__finalize__(self._barray)

def transpose ( self , * axes ) : new = argpack ( axes ) old = range ( self . ndim ) istransposeable ( new , old ) if new == old : return self . _barray def f ( v ) : return v . transpose ( new ) newrdd = self . _barray . _rdd . mapValues ( f ) newshape = self . _barray . keys . shape + tuple ( self . shape [ i ] for i in new ) return BoltArraySpark ( newrdd , shape = newshape ) . __finalize__ ( self . _barray )

Transpose just the values of a BoltArraySpark returning a new BoltArraySpark .


def ones(shape, dtype=float64, order='C'):
        """
        Create a local bolt array of ones.

        Parameters
        ----------
        shape : tuple
            Dimensions of the desired array

        dtype : data-type, optional, default=float64
            The desired data-type for the array. (see numpy)

        order : {'C', 'F', 'A'}, optional, default='C'
            The order of the array. (see numpy)

        Returns
        -------
        BoltArrayLocal
        """
        from numpy import ones
        return ConstructLocal._wrap(ones, shape, dtype, order)

def ones ( shape , dtype = float64 , order = 'C' ) : from numpy import ones return ConstructLocal . _wrap ( ones , shape , dtype , order )


def zeros(shape, dtype=float64, order='C'):
        """
        Create a local bolt array of zeros.

        Parameters
        ----------
        shape : tuple
            Dimensions of the desired array.

        dtype : data-type, optional, default=float64
            The desired data-type for the array. (see numpy)

        order : {'C', 'F', 'A'}, optional, default='C'
            The order of the array. (see numpy)

        Returns
        -------
        BoltArrayLocal
        """
        from numpy import zeros
        return ConstructLocal._wrap(zeros, shape, dtype, order)

def zeros ( shape , dtype = float64 , order = 'C' ) : from numpy import zeros return ConstructLocal . _wrap ( zeros , shape , dtype , order )


def concatenate(arrays, axis=0):
        """
        Join a sequence of arrays together.

        Parameters
        ----------
        arrays : tuple
            A sequence of array-like e.g. (a1, a2, ...)

        axis : int, optional, default=0
            The axis along which the arrays will be joined.

        Returns
        -------
        BoltArrayLocal
        """
        if not isinstance(arrays, tuple):
            raise ValueError("data type not understood")
        arrays = tuple([asarray(a) for a in arrays])
        from numpy import concatenate
        return BoltArrayLocal(concatenate(arrays, axis))

def
concatenate
(
arrays
,
axis
=
0
)
:
if
not
isinstance
(
arrays
,
tuple
)
:
raise
ValueError
(
"data type not understood"
)
arrays
=
tuple
(
[
asarray
(
a
)
for
a
in
arrays
]
)
from
numpy
import
concatenate
return
BoltArrayLocal
(
concatenate
(
arrays
,
axis
)
)


def discrete_likelihood(data, xmin, alpha):
    """
    Equation B.8 in Clauset

    Given a data set, an xmin value, and an alpha "scaling parameter", computes
    the log-likelihood (the value to be maximized)
    """
    if not scipyOK:
        raise ImportError("Can't import scipy.  Need scipy for zeta function.")
    from scipy.special import zeta as zeta

    zz = data[data>=xmin]
    nn = len(zz)

    sum_log_data = np.log(zz).sum()

    zeta = zeta(alpha, xmin)

    L_of_alpha = -1*nn*log(zeta) - alpha * sum_log_data

    return L_of_alpha


def most_likely_alpha(data, xmin, alpharange=(1.5,3.5), n_alpha=201):
    """
    Return the most likely alpha for the data given an xmin
    """
    alpha_vector = np.linspace(alpharange[0],alpharange[1],n_alpha)
    return alpha_vector[discrete_max_likelihood_arg(data, xmin,
                                                    alpharange=alpharange,
                                                    n_alpha=n_alpha)]

def most_likely_alpha ( data , xmin , alpharange = ( 1.5 , 3.5 ) , n_alpha = 201 ) : alpha_vector = np . linspace ( alpharange [ 0 ] , alpharange [ 1 ] , n_alpha ) return alpha_vector [ discrete_max_likelihood_arg ( data , xmin , alpharange = alpharange , n_alpha = n_alpha ) ]


def discrete_alpha_mle(data, xmin):
    """
    Equation B.17 of Clauset et al 2009

    The Maximum Likelihood Estimator of the "scaling parameter" alpha in the
    discrete case is similar to that in the continuous case
    """
    # boolean indices of positive data
    gexmin = (data>=xmin)
    nn = gexmin.sum()
    if nn < 2:
        return 0
    xx = data[gexmin]
    alpha = 1.0 + float(nn) * (sum(log(xx/(float(xmin)-0.5))))**-1
    return alpha

def
discrete_alpha_mle
(
data
,
xmin
)
:
# boolean indices of positive data
gexmin
=
(
data
>=
xmin
)
nn
=
gexmin
.
sum
(
)
if
nn
<
2
:
return
0
xx
=
data
[
gexmin
]
alpha
=
1.0
+
float
(
nn
)
*
(
sum
(
log
(
xx
/
(
float
(
xmin
)
0.5
)
)
)
)
**
1
return
alpha


def discrete_best_alpha(data, alpharangemults=(0.9,1.1), n_alpha=201, approximate=True, verbose=True):
    """
    Use the maximum L to determine the most likely value of alpha

    *alpharangemults* [ 2-tuple ]
        Pair of values indicating multiplicative factors above and below the
        approximate alpha from the MLE alpha to use when determining the
        "exact" alpha (by directly maximizing the likelihood function)
    """

    xmins = np.unique(data)
    if approximate:
        alpha_of_xmin = [ discrete_alpha_mle(data,xmin) for xmin in xmins ]
    else:
        alpha_approx = [ discrete_alpha_mle(data,xmin) for xmin in xmins ]
        alpharanges = [(0.9*a,1.1*a) for a in alpha_approx]
        alpha_of_xmin = [ most_likely_alpha(data,xmin,alpharange=ar,n_alpha=n_alpha) for xmin,ar in zip(xmins,alpharanges) ]
    ksvalues = [ discrete_ksD(data, xmin, alpha) for xmin,alpha in zip(xmins,alpha_of_xmin) ]

    best_index = argmin(ksvalues)
    best_alpha = alpha_of_xmin[best_index]
    best_xmin = xmins[best_index]
    best_ks = ksvalues[best_index]
    best_likelihood = discrete_likelihood(data, best_xmin, best_alpha)

    if verbose:
        print("alpha = %f   xmin = %f   ksD = %f   L = %f   (n<x) = %i  (n>=x) = %i" % (
                best_alpha, best_xmin, best_ks, best_likelihood,
                (data<best_xmin).sum(), (data>=best_xmin).sum()))

    return best_alpha,best_xmin,best_ks,best_likelihood


def discrete_best_alpha(self, alpharangemults=(0.9,1.1), n_alpha=201,
                            approximate=True, verbose=True, finite=True):
        """
        Use the maximum likelihood to determine the most likely value of alpha

        *alpharangemults* [ 2-tuple ]
            Pair of values indicating multiplicative factors above and below the
            approximate alpha from the MLE alpha to use when determining the
            "exact" alpha (by directly maximizing the likelihood function)
        *n_alpha* [ int ]
            Number of alpha values to use when measuring.  Larger number is more accurate.
        *approximate* [ bool ]
            If False, try to "zoom-in" around the MLE alpha and get the exact
            best alpha value within some range around the approximate best
        *vebose* [ bool ]
        *finite* [ bool ]
            Correction for finite data?
        """

        data = self.data
        self._xmins = xmins = np.unique(data)
        if approximate:
            alpha_of_xmin = [ discrete_alpha_mle(data,xmin) for xmin in xmins ]
        else:
            alpha_approx = [ discrete_alpha_mle(data,xmin) for xmin in xmins ]
            alpharanges = [(0.9*a,1.1*a) for a in alpha_approx]
            alpha_of_xmin = [ most_likely_alpha(data,xmin,alpharange=ar,n_alpha=n_alpha)
                             for xmin,ar in zip(xmins,alpharanges) ]
        ksvalues = np.array([discrete_ksD(data, xmin, alpha)
                                for xmin,alpha in zip(xmins,alpha_of_xmin)
                               ])
        self._alpha_values = np.array(alpha_of_xmin)
        self._xmin_kstest = ksvalues

        ksvalues[np.isnan(ksvalues)] = np.inf

        best_index = argmin(ksvalues)
        self._alpha = best_alpha = alpha_of_xmin[best_index]
        self._xmin = best_xmin = xmins[best_index]
        self._ks = best_ks = ksvalues[best_index]
        self._likelihood = best_likelihood = discrete_likelihood(data, best_xmin, best_alpha)

        if finite:
            self._alpha = self._alpha*(n-1.)/n+1./n

        if verbose:
            print("alpha = %f   xmin = %f   ksD = %f   L = %f   (n<x) = %i  (n>=x) = %i" % (
                    best_alpha, best_xmin, best_ks, best_likelihood,
                    (data<best_xmin).sum(), (data>=best_xmin).sum()))

        self._ngtx = n = (self.data>=self._xmin).sum()
        self._alphaerr = (self._alpha-1.0)/np.sqrt(n)
        if scipyOK:
            self._ks_prob = scipy.stats.ksone.sf(self._ks, n)

        return best_alpha,best_xmin,best_ks,best_likelihood


def plotppf(self,x=None,xmin=None,alpha=None,dolog=True,**kwargs):
        """
        Plots the power-law-predicted value on the Y-axis against the real
        values along the X-axis.  Can be used as a diagnostic of the fit
        quality.
        """
        if not(xmin): xmin=self._xmin
        if not(alpha): alpha=self._alpha
        if not(x): x=np.sort(self.data[self.data>xmin])
        else: x=np.sort(x[x>xmin])

        # N = M^(-alpha+1)
        # M = N^(1/(-alpha+1))

        m0 = min(x)
        N = (1.0+np.arange(len(x)))[::-1]
        xmodel = m0 * N**(1/(1-alpha)) / max(N)**(1/(1-alpha))

        if dolog:
            pylab.loglog(x,xmodel,'.',**kwargs)
            pylab.gca().set_xlim(min(x),max(x))
            pylab.gca().set_ylim(min(x),max(x))
        else:
            pylab.plot(x,xmodel,'.',**kwargs)
        pylab.plot([min(x),max(x)],[min(x),max(x)],'k--')
        pylab.xlabel("Real Value")
        pylab.ylabel("Power-Law Model Value")

def
plotppf
(
self
,
x
=
,
xmin
=
,
alpha
=
,
dolog
=
True
,
*
*
kwargs
)
:
if
not
(
xmin
)
:
xmin
=
self
.
_xmin
if
not
(
alpha
)
:
alpha
=
self
.
_alpha
if
not
(
x
)
:
x
=
np
.
sort
(
self
.
data
[
self
.
data
>
xmin
]
)
else
:
x
=
np
.
sort
(
x
[
x
>
xmin
]
)
# N = M^(-alpha+1)
# M = N^(1/(-alpha+1))
m0
=
min
(
x
)
N
=
(
1.0
+
np
.
arange
(
len
(
x
)
)
)
[
:
:
1
]
xmodel
=
m0
*
N
**
(
1
/
(
1
alpha
)
)
/
max
(
N
)
**
(
1
/
(
1
alpha
)
)
if
dolog
:
pylab
.
loglog
(
x
,
xmodel
,
'.'
,
*
*
kwargs
)
pylab
.
gca
(
)
.
set_xlim
(
min
(
x
)
,
max
(
x
)
)
pylab
.
gca
(
)
.
set_ylim
(
min
(
x
)
,
max
(
x
)
)
else
:
pylab
.
plot
(
x
,
xmodel
,
'.'
,
*
*
kwargs
)
pylab
.
plot
(
[
min
(
x
)
,
max
(
x
)
]
,
[
min
(
x
)
,
max
(
x
)
]
,
'k--'
)
pylab
.
xlabel
(
"Real Value"
)
pylab
.
ylabel
(
"Power-Law Model Value"
)

Plots the power - law - predicted value on the Y - axis against the real values along the X - axis . Can be used as a diagnostic of the fit quality .


def lognormal(self,doprint=True):
        """
        Use the maximum likelihood estimator for a lognormal distribution to
        produce the best-fit lognormal parameters
        """
        # N = float(self.data.shape[0])
        # mu = log(self.data).sum() / N
        # sigmasquared = ( ( log(self.data) - mu )**2 ).sum() / N
        # self.lognormal_mu = mu
        # self.lognormal_sigma = np.sqrt(sigmasquared)
        # self.lognormal_likelihood = -N/2. * log(np.pi*2) - N/2. * log(sigmasquared) - 1/(2*sigmasquared) * (( self.data - mu )**2).sum()
        # if doprint:
        #     print "Best fit lognormal is exp( -(x-%g)^2 / (2*%g^2)" % (mu,np.sqrt(sigmasquared))
        #     print "Likelihood: %g" % (self.lognormal_likelihood)
        if scipyOK:
            fitpars = scipy.stats.lognorm.fit(self.data)
            self.lognormal_dist = scipy.stats.lognorm(*fitpars)
            self.lognormal_ksD,self.lognormal_ksP = scipy.stats.kstest(self.data,self.lognormal_dist.cdf)
            # nnlf = NEGATIVE log likelihood
            self.lognormal_likelihood = -1*scipy.stats.lognorm.nnlf(fitpars,self.data)

            # Is this the right likelihood ratio?
            # Definition of L from eqn. B3 of Clauset et al 2009:
            # L = log(p(x|alpha))
            # _nnlf from scipy.stats.distributions:
            # -sum(log(self._pdf(x, *args)),axis=0)
            # Assuming the pdf and p(x|alpha) are both non-inverted, it looks
            # like the _nnlf and L have opposite signs, which would explain the
            # likelihood ratio I've used here:
            self.power_lognorm_likelihood = (self._likelihood + self.lognormal_likelihood)
            # a previous version had 2*(above).  That is the correct form if you want the likelihood ratio
            # statistic "D": http://en.wikipedia.org/wiki/Likelihood-ratio_test
            # The above explanation makes sense, since nnlf is the *negative* log likelihood function:
            ## nnlf  -- negative log likelihood function (to minimize)
            #
            # Assuming we want the ratio between the POSITIVE likelihoods, the D statistic is:
            # D = -2 log( L_power / L_lognormal )
            self.likelihood_ratio_D = -2 * (log(self._likelihood/self.lognormal_likelihood))

            if doprint:
                print("Lognormal KS D: %g  p(D): %g" % (self.lognormal_ksD,self.lognormal_ksP), end=' ')
                print("  Likelihood Ratio Statistic (powerlaw/lognormal): %g" % self.likelihood_ratio_D)
                print("At this point, have a look at Clauset et al 2009 Appendix C: determining sigma(likelihood_ratio)")

Use the maximum likelihood estimator for a lognormal distribution to produce the best - fit lognormal parameters


def sanitize_turbo(html, allowed_tags=TURBO_ALLOWED_TAGS, allowed_attrs=TURBO_ALLOWED_ATTRS):
    """Sanitizes HTML, removing not allowed tags and attributes.

    :param str|unicode html:

    :param list allowed_tags: List of allowed tags.
    :param dict allowed_attrs: Dictionary with attributes allowed for tags.

    :rtype: unicode
    """
    return clean(html, tags=allowed_tags, attributes=allowed_attrs, strip=True)

def sanitize_turbo ( html , allowed_tags = TURBO_ALLOWED_TAGS , allowed_attrs = TURBO_ALLOWED_ATTRS ) : return clean ( html , tags = allowed_tags , attributes = allowed_attrs , strip = True )

Sanitizes HTML removing not allowed tags and attributes .


def configure_analytics_yandex(self, ident, params=None):
        """Configure Yandex Metrika analytics counter.

        :param str|unicode ident: Metrika counter ID.

        :param dict params: Additional params.

        """
        params = params or {}

        data = {
            'type': 'Yandex',
            'id': ident,
        }

        if params:
            data['params'] = '%s' % params

        self.analytics.append(data)

def configure_analytics_yandex ( self , ident , params = None ) : params = params or { } data = { 'type' : 'Yandex' , 'id' : ident , } if params : data [ 'params' ] = '%s' % params self . analytics . append ( data )

Configure Yandex Metrika analytics counter .


def tag_list(self, tags):
        """
        Generates a list of tags identifying those previously selected.

        Returns a list of tuples of the form (<tag name>, <CSS class name>).

        Uses the string names rather than the tags themselves in order to work
        with tag lists built from forms not fully submitted.
        """
        return [
            (tag.name, "selected taggit-tag" if tag.name in tags else "taggit-tag")
            for tag in self.model.objects.all()
        ]

def
tag_list
(
self
,
tags
)
:
return
[
(
tag
.
name
,
"selected taggit-tag"
if
tag
.
name
in
tags
else
"taggit-tag"
)
for
tag
in
self
.
model
.
objects
.
all
(
)
]

Generates a list of tags identifying those previously selected .


def hash_md5(self):
        """Calculate md5 fingerprint.

        Shamelessly copied from http://stackoverflow.com/questions/6682815/deriving-an-ssh-fingerprint-from-a-public-key-in-python

        For specification, see RFC4716, section 4."""
        fp_plain = hashlib.md5(self._decoded_key).hexdigest()
        return "MD5:" + ':'.join(a + b for a, b in zip(fp_plain[::2], fp_plain[1::2]))

def hash_md5 ( self ) : fp_plain = hashlib . md5 ( self . _decoded_key ) . hexdigest ( ) return "MD5:" + ':' . join ( a + b for a , b in zip ( fp_plain [ : : 2 ] , fp_plain [ 1 : : 2 ] ) )


def hash_sha256(self):
        """Calculate sha256 fingerprint."""
        fp_plain = hashlib.sha256(self._decoded_key).digest()
        return (b"SHA256:" + base64.b64encode(fp_plain).replace(b"=", b"")).decode("utf-8")

def hash_sha256 ( self ) : fp_plain = hashlib . sha256 ( self . _decoded_key ) . digest ( ) return ( b"SHA256:" + base64 . b64encode ( fp_plain ) . replace ( b"=" , b"" ) ) . decode ( "utf-8" )


def hash_sha512(self):
        """Calculates sha512 fingerprint."""
        fp_plain = hashlib.sha512(self._decoded_key).digest()
        return (b"SHA512:" + base64.b64encode(fp_plain).replace(b"=", b"")).decode("utf-8")

def hash_sha512 ( self ) : fp_plain = hashlib . sha512 ( self . _decoded_key ) . digest ( ) return ( b"SHA512:" + base64 . b64encode ( fp_plain ) . replace ( b"=" , b"" ) ) . decode ( "utf-8" )


def _parse_long(cls, data):
        """Calculate two's complement."""
        if sys.version < '3':
            # this does not exist in python 3 - undefined-variable disabled to make pylint happier.
            ret = long(0)  # pylint:disable=undefined-variable
            for byte in data:
                ret = (ret << 8) + ord(byte)
        else:
            ret = 0
            for byte in data:
                ret = (ret << 8) + byte
        return ret


def decode_key(cls, pubkey_content):
        """Decode base64 coded part of the key."""
        try:
            decoded_key = base64.b64decode(pubkey_content.encode("ascii"))
        except (TypeError, binascii.Error):
            raise MalformedDataError("Unable to decode the key")
        return decoded_key

def
decode_key
(
cls
,
pubkey_content
)
:
try
:
decoded_key
=
base64
.
b64decode
(
pubkey_content
.
encode
(
"ascii"
)
)
except
(
TypeError
,
binascii
.
Error
)
:
raise
MalformedDataError
(
"Unable to decode the key"
)
return
decoded_key


def parse_options(self, options):
        """Parses ssh options string."""
        quote_open = False
        parsed_options = {}

        def parse_add_single_option(opt):
            """Parses and validates a single option, and adds it to parsed_options field."""
            if "=" in opt:
                opt_name, opt_value = opt.split("=", 1)
                opt_value = opt_value.replace('"', '')
            else:
                opt_name = opt
                opt_value = True
            if " " in opt_name or not self.OPTION_NAME_RE.match(opt_name):
                raise InvalidOptionNameError("%s is not valid option name." % opt_name)
            if self.strict_mode:
                for valid_opt_name, value_required in self.OPTIONS_SPEC:
                    if opt_name.lower() == valid_opt_name:
                        if value_required and opt_value is True:
                            raise MissingMandatoryOptionValueError("%s is missing mandatory value." % opt_name)
                        break
                else:
                    raise UnknownOptionNameError("%s is unrecognized option name." % opt_name)
            if opt_name not in parsed_options:
                parsed_options[opt_name] = []
            parsed_options[opt_name].append(opt_value)

        start_of_current_opt = 0
        i = 1  # Need to be set for empty options strings
        for i, character in enumerate(options):
            if character == '"':  # only double quotes are allowed, no need to care about single quotes
                quote_open = not quote_open
            if quote_open:
                continue
            if character == ",":
                opt = options[start_of_current_opt:i]
                parse_add_single_option(opt)
                start_of_current_opt = i + 1
                # Data begins after the first space
        if start_of_current_opt + 1 != i:
            opt = options[start_of_current_opt:]
            parse_add_single_option(opt)
        if quote_open:
            raise InvalidOptionsError("Unbalanced quotes.")
        return parsed_options


def _process_ssh_rsa(self, data):
        """Parses ssh-rsa public keys."""
        current_position, raw_e = self._unpack_by_int(data, 0)
        current_position, raw_n = self._unpack_by_int(data, current_position)

        unpacked_e = self._parse_long(raw_e)
        unpacked_n = self._parse_long(raw_n)

        self.rsa = RSAPublicNumbers(unpacked_e, unpacked_n).public_key(default_backend())
        self.bits = self.rsa.key_size

        if self.strict_mode:
            min_length = self.RSA_MIN_LENGTH_STRICT
            max_length = self.RSA_MAX_LENGTH_STRICT
        else:
            min_length = self.RSA_MIN_LENGTH_LOOSE
            max_length = self.RSA_MAX_LENGTH_LOOSE
        if self.bits < min_length:
            raise TooShortKeyError(
                "%s key data can not be shorter than %s bits (was %s)" % (self.key_type, min_length, self.bits)
            )
        if self.bits > max_length:
            raise TooLongKeyError(
                "%s key data can not be longer than %s bits (was %s)" % (self.key_type, max_length, self.bits)
            )
        return current_position


def _process_ssh_dss(self, data):
        """Parses ssh-dsa public keys."""
        data_fields = {}
        current_position = 0
        for item in ("p", "q", "g", "y"):
            current_position, value = self._unpack_by_int(data, current_position)
            data_fields[item] = self._parse_long(value)

        q_bits = self._bits_in_number(data_fields["q"])
        p_bits = self._bits_in_number(data_fields["p"])
        if q_bits != self.DSA_N_LENGTH:
            raise InvalidKeyError("Incorrect DSA key parameters: bits(p)=%s, q=%s" % (self.bits, q_bits))
        if self.strict_mode:
            min_length = self.DSA_MIN_LENGTH_STRICT
            max_length = self.DSA_MAX_LENGTH_STRICT
        else:
            min_length = self.DSA_MIN_LENGTH_LOOSE
            max_length = self.DSA_MAX_LENGTH_LOOSE
        if p_bits < min_length:
            raise TooShortKeyError("%s key can not be shorter than %s bits (was %s)" % (self.key_type, min_length, p_bits))
        if p_bits > max_length:
            raise TooLongKeyError(
                "%s key data can not be longer than %s bits (was %s)" % (self.key_type, max_length, p_bits)
            )

        dsa_parameters = DSAParameterNumbers(data_fields["p"], data_fields["q"], data_fields["g"])
        self.dsa = DSAPublicNumbers(data_fields["y"], dsa_parameters).public_key(default_backend())
        self.bits = self.dsa.key_size

        return current_position


def _process_ecdsa_sha(self, data):
        """Parses ecdsa-sha public keys."""
        current_position, curve_information = self._unpack_by_int(data, 0)
        if curve_information not in self.ECDSA_CURVE_DATA:
            raise NotImplementedError("Invalid curve type: %s" % curve_information)
        curve, hash_algorithm = self.ECDSA_CURVE_DATA[curve_information]

        current_position, key_data = self._unpack_by_int(data, current_position)
        try:
            # data starts with \x04, which should be discarded.
            ecdsa_key = ecdsa.VerifyingKey.from_string(key_data[1:], curve, hash_algorithm)
        except AssertionError:
            raise InvalidKeyError("Invalid ecdsa key")
        self.bits = int(curve_information.replace(b"nistp", b""))
        self.ecdsa = ecdsa_key
        return current_position


def _process_ed25516(self, data):
        """Parses ed25516 keys.

        There is no (apparent) way to validate ed25519 keys. This only
        checks data length (256 bits), but does not try to validate
        the key in any way."""

        current_position, verifying_key = self._unpack_by_int(data, 0)
        verifying_key_length = len(verifying_key) * 8
        verifying_key = self._parse_long(verifying_key)

        if verifying_key < 0:
            raise InvalidKeyError("ed25519 verifying key must be >0.")

        self.bits = verifying_key_length
        if self.bits != 256:
            raise InvalidKeyLengthError("ed25519 keys must be 256 bits (was %s bits)" % self.bits)
        return current_position


def parse(self, keydata=None):
        """Validates SSH public key.

        Throws exception for invalid keys. Otherwise returns None.

        Populates key_type, bits and bits fields.

        For rsa keys, see field "rsa" for raw public key data.
        For dsa keys, see field "dsa".
        For ecdsa keys, see field "ecdsa"."""
        if keydata is None:
            if self.keydata is None:
                raise ValueError("Key data must be supplied either in constructor or to parse()")
            keydata = self.keydata
        else:
            self.reset()
            self.keydata = keydata

        if keydata.startswith("---- BEGIN SSH2 PUBLIC KEY ----"):
            # SSH2 key format
            key_type = None  # There is no redundant key-type field - skip comparing plain-text and encoded data.
            pubkey_content = "".join([line for line in keydata.split("\n") if ":" not in line and "----" not in line])
        else:
            key_parts = self._split_key(keydata)
            key_type = key_parts[0]
            pubkey_content = key_parts[1]

        self._decoded_key = self.decode_key(pubkey_content)

        # Check key type
        current_position, unpacked_key_type = self._unpack_by_int(self._decoded_key, 0)
        if key_type is not None and key_type != unpacked_key_type.decode():
            raise InvalidTypeError("Keytype mismatch: %s != %s" % (key_type, unpacked_key_type))

        self.key_type = unpacked_key_type

        key_data_length = self._process_key(self._decoded_key[current_position:])
        current_position = current_position + key_data_length

        if current_position != len(self._decoded_key):
            raise MalformedDataError("Leftover data: %s bytes" % (len(self._decoded_key) - current_position))

        if self.disallow_options and self.options:
            raise InvalidOptionsError("Options are disallowed.")