splearn/feature_extraction/dict_vectorizer.py
def fit(self, Z):
        """Learn a list of feature name -> indices mappings.

        Parameters
        ----------
        Z : DictRDD with column 'X'
            Dict(s) or Mapping(s) from feature names (arbitrary Python
            objects) to feature values (strings or convertible to dtype).

        Returns
        -------
        self
        """
        X = Z[:, 'X'] if isinstance(Z, DictRDD) else Z

        """Create vocabulary
        """
        class SetAccum(AccumulatorParam):

            def zero(self, initialValue):
                return set(initialValue)

            def addInPlace(self, v1, v2):
                v1 |= v2
                return v1

        accum = X.context.accumulator(set(), SetAccum())

        def mapper(X, separator=self.separator):
            feature_names = []
            for x in X:
                for f, v in six.iteritems(x):
                    if isinstance(v, six.string_types):
                        f = "%s%s%s" % (f, self.separator, v)
                    feature_names.append(f)
            accum.add(set(feature_names))

        X.foreach(mapper)  # init vocabulary
        feature_names = list(accum.value)

        if self.sort:
            feature_names.sort()

        vocab = dict((f, i) for i, f in enumerate(feature_names))

        self.feature_names_ = feature_names
        self.vocabulary_ = vocab

        return self
def fit(self, Z):
        X = Z[:, 'X'] if isinstance(Z, DictRDD) else Z

        """Create vocabulary
        """
        class SetAccum(AccumulatorParam):

            def zero(self, initialValue):
                return set(initialValue)

            def addInPlace(self, v1, v2):
                v1 |= v2
                return v1

        accum = X.context.accumulator(set(), SetAccum())

        def mapper(X, separator=self.separator):
            feature_names = []
            for x in X:
                for f, v in six.iteritems(x):
                    if isinstance(v, six.string_types):
                        f = "%s%s%s" % (f, self.separator, v)
                    feature_names.append(f)
            accum.add(set(feature_names))

        X.foreach(mapper)  # init vocabulary
        feature_names = list(accum.value)

        if self.sort:
            feature_names.sort()

        vocab = dict((f, i) for i, f in enumerate(feature_names))

        self.feature_names_ = feature_names
        self.vocabulary_ = vocab

        return self
def fit ( self , Z ) : X = Z [ : , 'X' ] if isinstance ( Z , DictRDD ) else Z """Create vocabulary
        """ class SetAccum ( AccumulatorParam ) : def zero ( self , initialValue ) : return set ( initialValue ) def addInPlace ( self , v1 , v2 ) : v1 |= v2 return v1 accum = X . context . accumulator ( set ( ) , SetAccum ( ) ) def mapper ( X , separator = self . separator ) : feature_names = [ ] for x in X : for f , v in six . iteritems ( x ) : if isinstance ( v , six . string_types ) : f = "%s%s%s" % ( f , self . separator , v ) feature_names . append ( f ) accum . add ( set ( feature_names ) ) X . foreach ( mapper ) # init vocabulary feature_names = list ( accum . value ) if self . sort : feature_names . sort ( ) vocab = dict ( ( f , i ) for i , f in enumerate ( feature_names ) ) self . feature_names_ = feature_names self . vocabulary_ = vocab return self
Learn a list of feature name -> indices mappings.

        Parameters
        ----------
        Z : DictRDD with column 'X'
            Dict(s) or Mapping(s) from feature names (arbitrary Python
            objects) to feature values (strings or convertible to dtype).

        Returns
        -------
        self
Learn a list of feature name - > indices mappings .