def _aggregate_ndoverlay(self, element, agg_fn):
        """
        Optimized aggregation for NdOverlay objects by aggregating each
        Element in an NdOverlay individually avoiding having to concatenate
        items in the NdOverlay. Works by summing sum and count aggregates and
        applying appropriate masking for NaN values. Mean aggregation
        is also supported by dividing sum and count aggregates. count_cat
        aggregates are grouped by the categorical dimension and a separate
        aggregate for each category is generated.
        """
        # Compute overall bounds
        x, y = element.last.dimensions()[0:2]
        info = self._get_sampling(element, x, y)
        (x_range, y_range), (xs, ys), (width, height), (xtype, ytype) = info
        if xtype == 'datetime':
            x_range = tuple((np.array(x_range)/1e3).astype('datetime64[us]'))
        if ytype == 'datetime':
            y_range = tuple((np.array(y_range)/1e3).astype('datetime64[us]'))
        agg_params = dict({k: v for k, v in dict(self.get_param_values(), **self.p).items()
                           if k in aggregate.params()},
                          x_range=x_range, y_range=y_range)
        bbox = BoundingBox(points=[(x_range[0], y_range[0]), (x_range[1], y_range[1])])

        # Optimize categorical counts by aggregating them individually
        if isinstance(agg_fn, ds.count_cat):
            agg_params.update(dict(dynamic=False, aggregator=ds.count()))
            agg_fn1 = aggregate.instance(**agg_params)
            if element.ndims == 1:
                grouped = element
            else:
                grouped = element.groupby([agg_fn.column], container_type=NdOverlay,
                                          group_type=NdOverlay)
            groups = []
            for k, v in grouped.items():
                agg = agg_fn1(v)
                groups.append((k, agg.clone(agg.data, bounds=bbox)))
            return grouped.clone(groups)

        # Create aggregate instance for sum, count operations, breaking mean
        # into two aggregates
        column = agg_fn.column or 'Count'
        if isinstance(agg_fn, ds.mean):
            agg_fn1 = aggregate.instance(**dict(agg_params, aggregator=ds.sum(column)))
            agg_fn2 = aggregate.instance(**dict(agg_params, aggregator=ds.count()))
        else:
            agg_fn1 = aggregate.instance(**agg_params)
            agg_fn2 = None
        is_sum = isinstance(agg_fn1.aggregator, ds.sum)

        # Accumulate into two aggregates and mask
        agg, agg2, mask = None, None, None
        mask = None
        for v in element:
            # Compute aggregates and mask
            new_agg = agg_fn1.process_element(v, None)
            if is_sum:
                new_mask = np.isnan(new_agg.data[column].values)
                new_agg.data = new_agg.data.fillna(0)
            if agg_fn2:
                new_agg2 = agg_fn2.process_element(v, None)

            if agg is None:
                agg = new_agg
                if is_sum: mask = new_mask
                if agg_fn2: agg2 = new_agg2
            else:
                agg.data += new_agg.data
                if is_sum: mask &= new_mask
                if agg_fn2: agg2.data += new_agg2.data

        # Divide sum by count to compute mean
        if agg2 is not None:
            agg2.data.rename({'Count': agg_fn.column}, inplace=True)
            with np.errstate(divide='ignore', invalid='ignore'):
                agg.data /= agg2.data

        # Fill masked with with NaNs
        if is_sum:
            agg.data[column].values[mask] = np.NaN

        return agg.clone(bounds=bbox)
def _aggregate_ndoverlay ( self , element , agg_fn ) : x , y = element . last . dimensions ( ) [ 0 : 2 ] info = self . _get_sampling ( element , x , y ) ( x_range , y_range ) , ( xs , ys ) , ( width , height ) , ( xtype , ytype ) = info if xtype == 'datetime' : x_range = tuple ( ( np . array ( x_range ) / 1e3 ) . astype ( 'datetime64[us]' ) ) if ytype == 'datetime' : y_range = tuple ( ( np . array ( y_range ) / 1e3 ) . astype ( 'datetime64[us]' ) ) agg_params = dict ( { k : v for k , v in dict ( self . get_param_values ( ) , ** self . p ) . items ( ) if k in aggregate . params ( ) } , x_range = x_range , y_range = y_range ) bbox = BoundingBox ( points = [ ( x_range [ 0 ] , y_range [ 0 ] ) , ( x_range [ 1 ] , y_range [ 1 ] ) ] ) if isinstance ( agg_fn , ds . count_cat ) : agg_params . update ( dict ( dynamic = False , aggregator = ds . count ( ) ) ) agg_fn1 = aggregate . instance ( ** agg_params ) if element . ndims == 1 : grouped = element else : grouped = element . groupby ( [ agg_fn . column ] , container_type = NdOverlay , group_type = NdOverlay ) groups = [ ] for k , v in grouped . items ( ) : agg = agg_fn1 ( v ) groups . append ( ( k , agg . clone ( agg . data , bounds = bbox ) ) ) return grouped . clone ( groups ) column = agg_fn . column or 'Count' if isinstance ( agg_fn , ds . mean ) : agg_fn1 = aggregate . instance ( ** dict ( agg_params , aggregator = ds . sum ( column ) ) ) agg_fn2 = aggregate . instance ( ** dict ( agg_params , aggregator = ds . count ( ) ) ) else : agg_fn1 = aggregate . instance ( ** agg_params ) agg_fn2 = None is_sum = isinstance ( agg_fn1 . aggregator , ds . sum ) agg , agg2 , mask = None , None , None mask = None for v in element : new_agg = agg_fn1 . process_element ( v , None ) if is_sum : new_mask = np . isnan ( new_agg . data [ column ] . values ) new_agg . data = new_agg . data . fillna ( 0 ) if agg_fn2 : new_agg2 = agg_fn2 . process_element ( v , None ) if agg is None : agg = new_agg if is_sum : mask = new_mask if agg_fn2 : agg2 = new_agg2 else : agg . data += new_agg . data if is_sum : mask &= new_mask if agg_fn2 : agg2 . data += new_agg2 . data if agg2 is not None : agg2 . data . rename ( { 'Count' : agg_fn . column } , inplace = True ) with np . errstate ( divide = 'ignore' , invalid = 'ignore' ) : agg . data /= agg2 . data if is_sum : agg . data [ column ] . values [ mask ] = np . NaN return agg . clone ( bounds = bbox )
Optimized aggregation for NdOverlay objects by aggregating each
        Element in an NdOverlay individually avoiding having to concatenate
        items in the NdOverlay. Works by summing sum and count aggregates and
        applying appropriate masking for NaN values. Mean aggregation
        is also supported by dividing sum and count aggregates. count_cat
        aggregates are grouped by the categorical dimension and a separate
        aggregate for each category is generated.