def cov_corr_chunk(df, corr=False):
 mat = df.values
 mask = np.isfinite(mat)
 keep = np.bitwise_and(mask[:, None, :], mask[:, :, None])
 x = np.where(keep, mat[:, None, :], np.nan)
 sums = np.nansum(x, 0)
 counts = keep.astype('int').sum(0)
 cov = df.cov().values
 dtype = [('sum', sums.dtype), ('count', counts.dtype), ('cov', cov.dtype)]
 if corr:
  m = np.nansum(((x - (sums / np.where(counts, counts, np.nan))) ** 2), 0)
  dtype.append(('m', m.dtype))
 out = np.empty(counts.shape, dtype=dtype)
 out['sum'] = sums
 out['count'] = counts
 out['cov'] = (cov * (counts - 1))
 if corr:
  out['m'] = m
 return out

def cov_corr_chunk ( df , corr = False ) : mat = df . values mask = np . isfinite ( mat ) keep = np . bitwise_and ( mask [ : , None , : ] , mask [ : , : , None ] ) x = np . where ( keep , mat [ : , None , : ] , np . nan ) sums = np . nansum ( x , 0 ) counts = keep . astype ( 'int' ) . sum ( 0 ) cov = df . cov ( ) . values dtype = [ ( 'sum' , sums . dtype ) , ( 'count' , counts . dtype ) , ( 'cov' , cov . dtype ) ] if corr : m = np . nansum ( ( ( x - ( sums / np . where ( counts , counts , np . nan ) ) ) ** 2 ) , 0 ) dtype . append ( ( 'm' , m . dtype ) ) out = np . empty ( counts . shape , dtype = dtype ) out [ 'sum' ] = sums out [ 'count' ] = counts out [ 'cov' ] = ( cov * ( counts - 1 ) ) if corr : out [ 'm' ] = m return out
chunk part of a covariance or correlation computation .