splearn/feature_selection/variance_threshold.py
def fit(self, Z):
        """Learn empirical variances from X.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape (n_samples, n_features)
            Sample vectors from which to compute variances.

        y : any
            Ignored. This parameter exists only for compatibility with
            sklearn.pipeline.Pipeline.

        Returns
        -------
        self
        """

        X = Z[:, 'X'] if isinstance(Z, DictRDD) else Z
        check_rdd(X, (np.ndarray, sp.spmatrix))

        def mapper(X):
            """Calculate statistics for every numpy or scipy blocks."""
            X = check_array(X, ('csr', 'csc'), dtype=np.float64)
            if hasattr(X, "toarray"):   # sparse matrix
                mean, var = mean_variance_axis(X, axis=0)
            else:
                mean, var = np.mean(X, axis=0), np.var(X, axis=0)
            return X.shape[0], mean, var

        def reducer(a, b):
            """Calculate the combined statistics."""
            n_a, mean_a, var_a = a
            n_b, mean_b, var_b = b
            n_ab = n_a + n_b
            mean_ab = ((mean_a * n_a) + (mean_b * n_b)) / n_ab
            var_ab = (((n_a * var_a) + (n_b * var_b)) / n_ab) + \
                     ((n_a * n_b) * ((mean_b - mean_a) / n_ab) ** 2)
            return (n_ab, mean_ab, var_ab)

        _, _, self.variances_ = X.map(mapper).treeReduce(reducer)

        if np.all(self.variances_ <= self.threshold):
            msg = "No feature in X meets the variance threshold {0:.5f}"
            if X.shape[0] == 1:
                msg += " (X contains only one sample)"
            raise ValueError(msg.format(self.threshold))

        return self
def fit(self, Z):
        X = Z[:, 'X'] if isinstance(Z, DictRDD) else Z
        check_rdd(X, (np.ndarray, sp.spmatrix))

        def mapper(X):
            """Calculate statistics for every numpy or scipy blocks."""
            X = check_array(X, ('csr', 'csc'), dtype=np.float64)
            if hasattr(X, "toarray"):   # sparse matrix
                mean, var = mean_variance_axis(X, axis=0)
            else:
                mean, var = np.mean(X, axis=0), np.var(X, axis=0)
            return X.shape[0], mean, var

        def reducer(a, b):
            """Calculate the combined statistics."""
            n_a, mean_a, var_a = a
            n_b, mean_b, var_b = b
            n_ab = n_a + n_b
            mean_ab = ((mean_a * n_a) + (mean_b * n_b)) / n_ab
            var_ab = (((n_a * var_a) + (n_b * var_b)) / n_ab) + \
                     ((n_a * n_b) * ((mean_b - mean_a) / n_ab) ** 2)
            return (n_ab, mean_ab, var_ab)

        _, _, self.variances_ = X.map(mapper).treeReduce(reducer)

        if np.all(self.variances_ <= self.threshold):
            msg = "No feature in X meets the variance threshold {0:.5f}"
            if X.shape[0] == 1:
                msg += " (X contains only one sample)"
            raise ValueError(msg.format(self.threshold))

        return self
def fit ( self , Z ) : X = Z [ : , 'X' ] if isinstance ( Z , DictRDD ) else Z check_rdd ( X , ( np . ndarray , sp . spmatrix ) ) def mapper ( X ) : """Calculate statistics for every numpy or scipy blocks.""" X = check_array ( X , ( 'csr' , 'csc' ) , dtype = np . float64 ) if hasattr ( X , "toarray" ) : # sparse matrix mean , var = mean_variance_axis ( X , axis = 0 ) else : mean , var = np . mean ( X , axis = 0 ) , np . var ( X , axis = 0 ) return X . shape [ 0 ] , mean , var def reducer ( a , b ) : """Calculate the combined statistics.""" n_a , mean_a , var_a = a n_b , mean_b , var_b = b n_ab = n_a + n_b mean_ab = ( ( mean_a * n_a ) + ( mean_b * n_b ) ) / n_ab var_ab = ( ( ( n_a * var_a ) + ( n_b * var_b ) ) / n_ab ) + ( ( n_a * n_b ) * ( ( mean_b - mean_a ) / n_ab ) ** 2 ) return ( n_ab , mean_ab , var_ab ) _ , _ , self . variances_ = X . map ( mapper ) . treeReduce ( reducer ) if np . all ( self . variances_ <= self . threshold ) : msg = "No feature in X meets the variance threshold {0:.5f}" if X . shape [ 0 ] == 1 : msg += " (X contains only one sample)" raise ValueError ( msg . format ( self . threshold ) ) return self
Learn empirical variances from X.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape (n_samples, n_features)
            Sample vectors from which to compute variances.

        y : any
            Ignored. This parameter exists only for compatibility with
            sklearn.pipeline.Pipeline.

        Returns
        -------
        self
Learn empirical variances from X .