def _format_heatmap(self, filename, heatmap, execution_count):
        """Formats heatmap for UI."""
        with open(filename) as src_file:
            file_source = src_file.read().split('\n')
            skip_map = self._calc_skips(heatmap, len(file_source))
        run_time = sum(time for time in heatmap.values())
        return {
            'name': filename,
            'heatmap': heatmap,
            'executionCount': execution_count,
            'srcCode': self._skip_lines(file_source, skip_map),
            'runTime': run_time
        }

def _format_heatmap ( self , filename , heatmap , execution_count ) : with open ( filename ) as src_file : file_source = src_file . read ( ) . split ( '\n' ) skip_map = self . _calc_skips ( heatmap , len ( file_source ) ) run_time = sum ( time for time in heatmap . values ( ) ) return { 'name' : filename , 'heatmap' : heatmap , 'executionCount' : execution_count , 'srcCode' : self . _skip_lines ( file_source , skip_map ) , 'runTime' : run_time }


def _profile_module(self):
        """Calculates heatmap for module."""
        with open(self._run_object, 'r') as srcfile:
            src_code = srcfile.read()
            code = compile(src_code, self._run_object, 'exec')
        try:
            with _CodeHeatmapCalculator() as prof:
                exec(code, self._globs, None)
        except SystemExit:
            pass

        heatmaps = []
        for filename, heatmap in prof.heatmap.items():
            if os.path.isfile(filename):
                heatmaps.append(
                    self._format_heatmap(
                        filename, heatmap, prof.execution_count[filename]))

        run_time = sum(heatmap['runTime'] for heatmap in heatmaps)
        return {
            'objectName': self._run_object,
            'runTime': run_time,
            'heatmaps': heatmaps
        }

def _profile_module ( self ) : with open ( self . _run_object , 'r' ) as srcfile : src_code = srcfile . read ( ) code = compile ( src_code , self . _run_object , 'exec' ) try : with _CodeHeatmapCalculator ( ) as prof : exec ( code , self . _globs , None ) except SystemExit : pass heatmaps = [ ] for filename , heatmap in prof . heatmap . items ( ) : if os . path . isfile ( filename ) : heatmaps . append ( self . _format_heatmap ( filename , heatmap , prof . execution_count [ filename ] ) ) run_time = sum ( heatmap [ 'runTime' ] for heatmap in heatmaps ) return { 'objectName' : self . _run_object , 'runTime' : run_time , 'heatmaps' : heatmaps }


def profile_function(self):
        """Calculates heatmap for function."""
        with _CodeHeatmapCalculator() as prof:
            result = self._run_object(*self._run_args, **self._run_kwargs)
        code_lines, start_line = inspect.getsourcelines(self._run_object)

        source_lines = []
        for line in code_lines:
            source_lines.append(('line', start_line, line))
            start_line += 1

        filename = os.path.abspath(inspect.getsourcefile(self._run_object))
        heatmap = prof.heatmap[filename]
        run_time = sum(time for time in heatmap.values())
        return {
            'objectName': self._object_name,
            'runTime': run_time,
            'result': result,
            'timestamp': int(time.time()),
            'heatmaps': [{
                'name': self._object_name,
                'heatmap': heatmap,
                'executionCount': prof.execution_count[filename],
                'srcCode': source_lines,
                'runTime': run_time
            }]
        }

def profile_function ( self ) : with _CodeHeatmapCalculator ( ) as prof : result = self . _run_object ( * self . _run_args , * * self . _run_kwargs ) code_lines , start_line = inspect . getsourcelines ( self . _run_object ) source_lines = [ ] for line in code_lines : source_lines . append ( ( 'line' , start_line , line ) ) start_line += 1 filename = os . path . abspath ( inspect . getsourcefile ( self . _run_object ) ) heatmap = prof . heatmap [ filename ] run_time = sum ( time for time in heatmap . values ( ) ) return { 'objectName' : self . _object_name , 'runTime' : run_time , 'result' : result , 'timestamp' : int ( time . time ( ) ) , 'heatmaps' : [ { 'name' : self . _object_name , 'heatmap' : heatmap , 'executionCount' : prof . execution_count [ filename ] , 'srcCode' : source_lines , 'runTime' : run_time } ] }


def run_profilers(run_object, prof_config, verbose=False):
    """Runs profilers on run_object.

    Args:
        run_object: An object (string or tuple) for profiling.
        prof_config: A string with profilers configuration.
        verbose: True if info about running profilers should be shown.
    Returns:
        An ordered dictionary with collected stats.
    Raises:
        AmbiguousConfigurationError: when prof_config is ambiguous.
        BadOptionError: when unknown options are present in configuration.
    """
    if len(prof_config) > len(set(prof_config)):
        raise AmbiguousConfigurationError(
            'Profiler configuration %s is ambiguous' % prof_config)

    available_profilers = {opt for opt, _ in _PROFILERS}
    for option in prof_config:
        if option not in available_profilers:
            raise BadOptionError('Unknown option: %s' % option)

    run_stats = OrderedDict()
    present_profilers = ((o, p) for o, p in _PROFILERS if o in prof_config)
    for option, prof in present_profilers:
        curr_profiler = prof(run_object)
        if verbose:
            print('Running %s...' % curr_profiler.__class__.__name__)
        run_stats[option] = curr_profiler.run()
    return run_stats


def run(func, options, args=(), kwargs={}, host='localhost', port=8000):  # pylint: disable=dangerous-default-value
    """Runs profilers on a function.

    Args:
        func: A Python function.
        options: A string with profilers configuration (i.e. 'cmh').
        args: func non-keyword arguments.
        kwargs: func keyword arguments.
        host: Host name to send collected data.
        port: Port number to send collected data.

    Returns:
        A result of func execution.
    """
    run_stats = run_profilers((func, args, kwargs), options)

    result = None
    for prof in run_stats:
        if not result:
            result = run_stats[prof]['result']
        del run_stats[prof]['result']  # Don't send result to remote host

    post_data = gzip.compress(
        json.dumps(run_stats).encode('utf-8'))
    urllib.request.urlopen('http://%s:%s' % (host, port), post_data)
    return result


def predict_proba(self, X):
        """
        Return probability estimates for the RDD containing test vector X.

        Parameters
        ----------
        X : RDD containing array-like items, shape = [m_samples, n_features]

        Returns
        -------
        C : RDD with array-like items , shape = [n_samples, n_classes]
            Returns the probability of the samples for each class in
            the models for each RDD block. The columns correspond to the classes
            in sorted order, as they appear in the attribute `classes_`.
        """
        check_rdd(X, (sp.spmatrix, np.ndarray))
        return X.map(
            lambda X: super(SparkBaseNB, self).predict_proba(X))

def predict_proba ( self , X ) : check_rdd ( X , ( sp . spmatrix , np . ndarray ) ) return X . map ( lambda X : super ( SparkBaseNB , self ) . predict_proba ( X ) )

Return probability estimates for the RDD containing test vector X .


def predict_log_proba(self, X):
        """
        Return log-probability estimates for the RDD containing the
        test vector X.

        Parameters
        ----------
        X : RDD containing array-like items, shape = [m_samples, n_features]

        Returns
        -------
        C : RDD with array-like items, shape = [n_samples, n_classes]
            Returns the log-probability of the samples for each class in
            the model for each RDD block. The columns correspond to the classes
            in sorted order, as they appear in the attribute `classes_`.
        """
        # required, scikit call self.predict_log_proba(X) in predict_proba
        # and thus this function is call, it must have the same behavior when
        # not called by sparkit-learn
        if not isinstance(X, BlockRDD):
            return super(SparkBaseNB, self).predict_log_proba(X)

        check_rdd(X, (sp.spmatrix, np.ndarray))
        return X.map(
            lambda X: super(SparkBaseNB, self).predict_log_proba(X))

Return log - probability estimates for the RDD containing the test vector X .


def fit(self, Z, classes=None):
        """Fit Gaussian Naive Bayes according to X, y

        Parameters
        ----------
        X : array-like, shape (n_samples, n_features)
            Training vectors, where n_samples is the number of samples
            and n_features is the number of features.

        y : array-like, shape (n_samples,)
            Target values.

        Returns
        -------
        self : object
            Returns self.
        """
        check_rdd(Z, {'X': (sp.spmatrix, np.ndarray), 'y': (sp.spmatrix, np.ndarray)})
        models = Z[:, ['X', 'y']].map(
            lambda X_y: self.partial_fit(X_y[0], X_y[1], classes))
        avg = models.reduce(operator.add)
        self.__dict__.update(avg.__dict__)
        return self

def fit ( self , Z , classes = None ) : check_rdd ( Z , { 'X' : ( sp . spmatrix , np . ndarray ) , 'y' : ( sp . spmatrix , np . ndarray ) } ) models = Z [ : , [ 'X' , 'y' ] ] . map ( lambda X_y : self . partial_fit ( X_y [ 0 ] , X_y [ 1 ] , classes ) ) avg = models . reduce ( operator . add ) self . __dict__ . update ( avg . __dict__ ) return self

Fit Gaussian Naive Bayes according to X y


def _count_vocab(self, analyzed_docs):
        """Create sparse feature matrix, and vocabulary where fixed_vocab=False
        """
        vocabulary = self.vocabulary_
        j_indices = _make_int_array()
        indptr = _make_int_array()
        indptr.append(0)
        for doc in analyzed_docs:
            for feature in doc:
                try:
                    j_indices.append(vocabulary[feature])
                except KeyError:
                    # Ignore out-of-vocabulary items for fixed_vocab=True
                    continue
            indptr.append(len(j_indices))

        j_indices = frombuffer_empty(j_indices, dtype=np.intc)
        indptr = np.frombuffer(indptr, dtype=np.intc)
        values = np.ones(len(j_indices))

        X = sp.csr_matrix((values, j_indices, indptr),
                          shape=(len(indptr) - 1, len(vocabulary)),
                          dtype=self.dtype)
        X.sum_duplicates()

        if self.binary:
            X.data.fill(1)

        return X

Create sparse feature matrix and vocabulary where fixed_vocab = False


def _sort_features(self, vocabulary):
        """Sort features by name

        Returns a reordered matrix and modifies the vocabulary in place
        """
        sorted_features = sorted(six.iteritems(vocabulary))
        map_index = np.empty(len(sorted_features), dtype=np.int32)
        for new_val, (term, old_val) in enumerate(sorted_features):
            map_index[new_val] = old_val
            vocabulary[term] = new_val

        return map_index

def _sort_features ( self , vocabulary ) : sorted_features = sorted ( six . iteritems ( vocabulary ) ) map_index = np . empty ( len ( sorted_features ) , dtype = np . int32 ) for new_val , ( term , old_val ) in enumerate ( sorted_features ) : map_index [ new_val ] = old_val vocabulary [ term ] = new_val return map_index


def _limit_features(self, X, vocabulary, high=None, low=None,
                        limit=None):
        """Remove too rare or too common features.

        Prune features that are non zero in more samples than high or less
        documents than low, modifying the vocabulary, and restricting it to
        at most the limit most frequent.

        This does not prune samples with zero features.
        """
        if high is None and low is None and limit is None:
            return X, set()

        # Calculate a mask based on document frequencies
        dfs = X.map(_document_frequency).sum()
        tfs = X.map(lambda x: np.asarray(x.sum(axis=0))).sum().ravel()
        mask = np.ones(len(dfs), dtype=bool)
        if high is not None:
            mask &= dfs <= high
        if low is not None:
            mask &= dfs >= low
        if limit is not None and mask.sum() > limit:
            mask_inds = (-tfs[mask]).argsort()[:limit]
            new_mask = np.zeros(len(dfs), dtype=bool)
            new_mask[np.where(mask)[0][mask_inds]] = True
            mask = new_mask

        new_indices = np.cumsum(mask) - 1  # maps old indices to new
        removed_terms = set()
        for term, old_index in list(six.iteritems(vocabulary)):
            if mask[old_index]:
                vocabulary[term] = new_indices[old_index]
            else:
                del vocabulary[term]
                removed_terms.add(term)
        kept_indices = np.where(mask)[0]

        if len(kept_indices) == 0:
            raise ValueError("After pruning, no terms remain. Try a lower"
                             " min_df or a higher max_df.")

        return kept_indices, removed_terms

Remove too rare or too common features .


def fit_transform(self, Z):
        """Learn the vocabulary dictionary and return term-document matrix.

        This is equivalent to fit followed by transform, but more efficiently
        implemented.

        Parameters
        ----------
        Z : iterable or DictRDD with column 'X'
            An iterable of raw_documents which yields either str, unicode or
            file objects; or a DictRDD with column 'X' containing such
            iterables.

        Returns
        -------
        X : array, [n_samples, n_features] or DictRDD
            Document-term matrix.
        """
        self._validate_vocabulary()

        # map analyzer and cache result
        analyze = self.build_analyzer()
        A = Z.transform(lambda X: list(map(analyze, X)), column='X').persist()

        # create vocabulary
        X = A[:, 'X'] if isinstance(A, DictRDD) else A
        self.vocabulary_ = self._init_vocab(X)

        # transform according to vocabulary
        mapper = self.broadcast(self._count_vocab, A.context)
        Z = A.transform(mapper, column='X', dtype=sp.spmatrix)

        if not self.fixed_vocabulary_:
            X = Z[:, 'X'] if isinstance(Z, DictRDD) else Z

            max_df = self.max_df
            min_df = self.min_df
            max_features = self.max_features

            # limit features according to min_df, max_df parameters
            n_doc = X.shape[0]
            max_doc_count = (max_df
                             if isinstance(max_df, numbers.Integral)
                             else max_df * n_doc)
            min_doc_count = (min_df
                             if isinstance(min_df, numbers.Integral)
                             else min_df * n_doc)
            if max_doc_count < min_doc_count:
                raise ValueError(
                    "max_df corresponds to < documents than min_df")
            kept_indices, self.stop_words_ = self._limit_features(
                X, self.vocabulary_, max_doc_count, min_doc_count, max_features)

            # sort features
            map_index = self._sort_features(self.vocabulary_)

            # combined mask
            mask = kept_indices[map_index]

            Z = Z.transform(lambda x: x[:, mask], column='X', dtype=sp.spmatrix)
        A.unpersist()
        return Z

Learn the vocabulary dictionary and return term - document matrix .


def transform(self, Z):
        """Transform documents to document-term matrix.

        Extract token counts out of raw text documents using the vocabulary
        fitted with fit or the one provided to the constructor.

        Parameters
        ----------
        raw_documents : iterable
            An iterable which yields either str, unicode or file objects.

        Returns
        -------
        X : sparse matrix, [n_samples, n_features]
            Document-term matrix.
        """
        if not hasattr(self, 'vocabulary_'):
            self._validate_vocabulary()

        self._check_vocabulary()

        analyze = self.build_analyzer()
        mapper = self.broadcast(self._count_vocab, Z.context)

        Z = Z.transform(lambda X: list(map(analyze, X)), column='X') \
             .transform(mapper, column='X', dtype=sp.spmatrix)

        return Z

def transform ( self , Z ) : if not hasattr ( self , 'vocabulary_' ) : self . _validate_vocabulary ( ) self . _check_vocabulary ( ) analyze = self . build_analyzer ( ) mapper = self . broadcast ( self . _count_vocab , Z . context ) Z = Z . transform ( lambda X : list ( map ( analyze , X ) ) , column = 'X' ) . transform ( mapper , column = 'X' , dtype = sp . spmatrix ) return Z

Transform documents to document - term matrix .


def to_scikit(self):
        """
        Convert to equivalent StandardScaler
        """
        scaler = StandardScaler(with_mean=self.with_mean,
                                with_std=self.with_std,
                                copy=self.copy)
        scaler.__dict__ = self.__dict__
        return scaler

def to_scikit ( self ) : scaler = StandardScaler ( with_mean = self . with_mean , with_std = self . with_std , copy = self . copy ) scaler . __dict__ = self . __dict__ return scaler


def _spark_fit(self, cls, Z, *args, **kwargs):
        """Wraps a Scikit-learn Linear model's fit method to use with RDD
        input.

        Parameters
        ----------
        cls : class object
            The sklearn linear model's class to wrap.
        Z : TupleRDD or DictRDD
            The distributed train data in a DictRDD.

        Returns
        -------
        self: the wrapped class
        """
        mapper = lambda X_y: super(cls, self).fit(
            X_y[0], X_y[1], *args, **kwargs
        )
        models = Z.map(mapper)
        avg = models.reduce(operator.add) / models.count()
        self.__dict__.update(avg.__dict__)
        return self

def _spark_fit ( self , cls , Z , * args , * * kwargs ) : mapper = lambda X_y : super ( cls , self ) . fit ( X_y [ 0 ] , X_y [ 1 ] , * args , * * kwargs ) models = Z . map ( mapper ) avg = models . reduce ( operator . add ) / models . count ( ) self . __dict__ . update ( avg . __dict__ ) return self

Wraps a Scikit - learn Linear model s fit method to use with RDD input .


def _spark_predict(self, cls, X, *args, **kwargs):
        """Wraps a Scikit-learn Linear model's predict method to use with RDD
        input.

        Parameters
        ----------
        cls : class object
            The sklearn linear model's class to wrap.
        Z : ArrayRDD
            The distributed data to predict in a DictRDD.

        Returns
        -------
        self: the wrapped class
        """
        return X.map(lambda X: super(cls, self).predict(X, *args, **kwargs))

def _spark_predict ( self , cls , X , * args , * * kwargs ) : return X . map ( lambda X : super ( cls , self ) . predict ( X , * args , * * kwargs ) )

Wraps a Scikit - learn Linear model s predict method to use with RDD input .


def fit(self, Z):
        """
        Fit linear model.

        Parameters
        ----------
        Z : DictRDD with (X, y) values
            X containing numpy array or sparse matrix - The training data
            y containing the target values

        Returns
        -------
        self : returns an instance of self.
        """
        check_rdd(Z, {'X': (sp.spmatrix, np.ndarray)})
        return self._spark_fit(SparkLinearRegression, Z)

def fit ( self , Z ) : check_rdd ( Z , { 'X' : ( sp . spmatrix , np . ndarray ) } ) return self . _spark_fit ( SparkLinearRegression , Z )


def fit(self, Z, **fit_params):
        """Fit all the transforms one after the other and transform the
        data, then fit the transformed data using the final estimator.

        Parameters
        ----------
        Z : ArrayRDD, TupleRDD or DictRDD
            Input data in blocked distributed format.

        Returns
        -------
        self : SparkPipeline
        """
        Zt, fit_params = self._pre_transform(Z, **fit_params)
        self.steps[-1][-1].fit(Zt, **fit_params)
        Zt.unpersist()
        return self

def fit ( self , Z , * * fit_params ) : Zt , fit_params = self . _pre_transform ( Z , * * fit_params ) self . steps [ - 1 ] [ - 1 ] . fit ( Zt , * * fit_params ) Zt . unpersist ( ) return self

Fit all the transforms one after the other and transform the data then fit the transformed data using the final estimator .


def fit_transform(self, Z, **fit_params):
        """Fit all the transforms one after the other and transform the
        data, then use fit_transform on transformed data using the final
        estimator."""
        Zt, fit_params = self._pre_transform(Z, **fit_params)
        if hasattr(self.steps[-1][-1], 'fit_transform'):
            return self.steps[-1][-1].fit_transform(Zt, **fit_params)
        else:
            return self.steps[-1][-1].fit(Zt, **fit_params).transform(Zt)

def fit_transform ( self , Z , * * fit_params ) : Zt , fit_params = self . _pre_transform ( Z , * * fit_params ) if hasattr ( self . steps [ - 1 ] [ - 1 ] , 'fit_transform' ) : return self . steps [ - 1 ] [ - 1 ] . fit_transform ( Zt , * * fit_params ) else : return self . steps [ - 1 ] [ - 1 ] . fit ( Zt , * * fit_params ) . transform ( Zt )

Fit all the transforms one after the other and transform the data then use fit_transform on transformed data using the final estimator .


def score(self, Z):
        """Applies transforms to the data, and the score method of the
        final estimator. Valid only if the final estimator implements
        score."""
        Zt = Z
        for name, transform in self.steps[:-1]:
            Zt = transform.transform(Zt)
        return self.steps[-1][-1].score(Zt)

def score ( self , Z ) : Zt = Z for name , transform in self . steps [ : - 1 ] : Zt = transform . transform ( Zt ) return self . steps [ - 1 ] [ - 1 ] . score ( Zt )

Applies transforms to the data and the score method of the final estimator . Valid only if the final estimator implements score .


def _fit(self, Z, parameter_iterable):
        """Actual fitting,  performing the search over parameters."""
        self.scorer_ = check_scoring(self.estimator, scoring=self.scoring)

        cv = self.cv
        cv = _check_cv(cv, Z)

        if self.verbose > 0:
            if isinstance(parameter_iterable, Sized):
                n_candidates = len(parameter_iterable)
                print("Fitting {0} folds for each of {1} candidates, totalling"
                      " {2} fits".format(len(cv), n_candidates,
                                         n_candidates * len(cv)))

        base_estimator = clone(self.estimator)

        pre_dispatch = self.pre_dispatch

        out = Parallel(
            n_jobs=self.n_jobs, verbose=self.verbose,
            pre_dispatch=pre_dispatch, backend="threading"
        )(
            delayed(_fit_and_score)(clone(base_estimator), Z, self.scorer_,
                                    train, test, self.verbose, parameters,
                                    self.fit_params, return_parameters=True,
                                    error_score=self.error_score)
            for parameters in parameter_iterable
            for train, test in cv)

        # Out is a list of triplet: score, estimator, n_test_samples
        n_fits = len(out)
        n_folds = len(cv)

        scores = list()
        grid_scores = list()
        for grid_start in range(0, n_fits, n_folds):
            n_test_samples = 0
            score = 0
            all_scores = []
            for this_score, this_n_test_samples, _, parameters in \
                    out[grid_start:grid_start + n_folds]:
                all_scores.append(this_score)
                if self.iid:
                    this_score *= this_n_test_samples
                    n_test_samples += this_n_test_samples
                score += this_score
            if self.iid:
                score /= float(n_test_samples)
            else:
                score /= float(n_folds)
            scores.append((score, parameters))
            # TODO: shall we also store the test_fold_sizes?
            grid_scores.append(_CVScoreTuple(
                parameters,
                score,
                np.array(all_scores)))
        # Store the computed scores
        self.grid_scores_ = grid_scores

        # Find the best parameters by comparing on the mean validation score:
        # note that `sorted` is deterministic in the way it breaks ties
        best = sorted(grid_scores, key=lambda x: x.mean_validation_score,
                      reverse=True)[0]
        self.best_params_ = best.parameters
        self.best_score_ = best.mean_validation_score

        if self.refit:
            # fit the best estimator using the entire dataset
            # clone first to work around broken estimators
            best_estimator = clone(base_estimator).set_params(
                **best.parameters)
            best_estimator.fit(Z, **self.fit_params)
            self.best_estimator_ = best_estimator
        return self

Actual fitting performing the search over parameters .


def _score(estimator, Z_test, scorer):
    """Compute the score of an estimator on a given test set."""
    score = scorer(estimator, Z_test)
    if not isinstance(score, numbers.Number):
        raise ValueError("scoring must return a number, got %s (%s) instead."
                         % (str(score), type(score)))
    return score

Compute the score of an estimator on a given test set .


def fit(self, Z):
        """Compute k-means clustering.

        Parameters
        ----------
        Z : ArrayRDD or DictRDD containing array-like or sparse matrix
            Train data.

        Returns
        -------
        self
        """
        X = Z[:, 'X'] if isinstance(Z, DictRDD) else Z
        check_rdd(X, (np.ndarray, sp.spmatrix))
        if self.init == 'k-means||':
            self._mllib_model = MLlibKMeans.train(
                X.unblock(),
                self.n_clusters,
                maxIterations=self.max_iter,
                initializationMode="k-means||")
            self.cluster_centers_ = self._mllib_model.centers
        else:
            models = X.map(lambda X: super(SparkKMeans, self).fit(X))
            models = models.map(lambda model: model.cluster_centers_).collect()
            return super(SparkKMeans, self).fit(np.concatenate(models))

def fit ( self , Z ) : X = Z [ : , 'X' ] if isinstance ( Z , DictRDD ) else Z check_rdd ( X , ( np . ndarray , sp . spmatrix ) ) if self . init == 'k-means||' : self . _mllib_model = MLlibKMeans . train ( X . unblock ( ) , self . n_clusters , maxIterations = self . max_iter , initializationMode = "k-means||" ) self . cluster_centers_ = self . _mllib_model . centers else : models = X . map ( lambda X : super ( SparkKMeans , self ) . fit ( X ) ) models = models . map ( lambda model : model . cluster_centers_ ) . collect ( ) return super ( SparkKMeans , self ) . fit ( np . concatenate ( models ) )


def predict(self, X):
        """Predict the closest cluster each sample in X belongs to.

        In the vector quantization literature, `cluster_centers_` is called
        the code book and each value returned by `predict` is the index of
        the closest code in the code book.

        Parameters
        ----------
        X : ArrayRDD containing array-like, sparse matrix
            New data to predict.

        Returns
        -------
        labels : ArrayRDD with predictions
            Index of the cluster each sample belongs to.

        """
        check_rdd(X, (np.ndarray, sp.spmatrix))
        if hasattr(self, '_mllib_model'):
            if isinstance(X, ArrayRDD):
                X = X.unblock()
            return X.map(lambda x: self._mllib_model.predict(x))
        else:
            rdd = X.map(lambda X: super(SparkKMeans, self).predict(X))
            return ArrayRDD(rdd)

def predict ( self , X ) : check_rdd ( X , ( np . ndarray , sp . spmatrix ) ) if hasattr ( self , '_mllib_model' ) : if isinstance ( X , ArrayRDD ) : X = X . unblock ( ) return X . map ( lambda x : self . _mllib_model . predict ( x ) ) else : rdd = X . map ( lambda X : super ( SparkKMeans , self ) . predict ( X ) ) return ArrayRDD ( rdd )

Predict the closest cluster each sample in X belongs to .


def predict(self, X):
        """Distributed method to predict class labels for samples in X.

        Parameters
        ----------
        X : ArrayRDD containing {array-like, sparse matrix}
            Samples.

        Returns
        -------
        C : ArrayRDD
            Predicted class label per sample.
        """
        check_rdd(X, (sp.spmatrix, np.ndarray))
        return self._spark_predict(SparkSGDClassifier, X)

def predict ( self , X ) : check_rdd ( X , ( sp . spmatrix , np . ndarray ) ) return self . _spark_predict ( SparkSGDClassifier , X )

Distributed method to predict class labels for samples in X .


def check_rdd_dtype(rdd, expected_dtype):
    """Checks if the blocks in the RDD matches the expected types.

    Parameters:
    -----------
    rdd: splearn.BlockRDD
        The RDD to check
    expected_dtype: {type, list of types, tuple of types, dict of types}
        Expected type(s). If the RDD is a DictRDD the parameter type is
        restricted to dict.

    Returns:
    --------
    accept: bool
        Returns if the types are matched.
    """
    if not isinstance(rdd, BlockRDD):
        raise TypeError("Expected {0} for parameter rdd, got {1}."
                        .format(BlockRDD, type(rdd)))
    if isinstance(rdd, DictRDD):
        if not isinstance(expected_dtype, dict):
            raise TypeError('Expected {0} for parameter '
                            'expected_dtype, got {1}.'
                            .format(dict, type(expected_dtype)))
        accept = True
        types = dict(list(zip(rdd.columns, rdd.dtype)))
        for key, values in expected_dtype.items():
            if not isinstance(values, (tuple, list)):
                values = [values]
            accept = accept and types[key] in values
        return accept

    if not isinstance(expected_dtype, (tuple, list)):
        expected_dtype = [expected_dtype]

    return rdd.dtype in expected_dtype

Checks if the blocks in the RDD matches the expected types .


def fit(self, Z):
        """Learn a list of feature name -> indices mappings.

        Parameters
        ----------
        Z : DictRDD with column 'X'
            Dict(s) or Mapping(s) from feature names (arbitrary Python
            objects) to feature values (strings or convertible to dtype).

        Returns
        -------
        self
        """
        X = Z[:, 'X'] if isinstance(Z, DictRDD) else Z

        """Create vocabulary
        """
        class SetAccum(AccumulatorParam):

            def zero(self, initialValue):
                return set(initialValue)

            def addInPlace(self, v1, v2):
                v1 |= v2
                return v1

        accum = X.context.accumulator(set(), SetAccum())

        def mapper(X, separator=self.separator):
            feature_names = []
            for x in X:
                for f, v in six.iteritems(x):
                    if isinstance(v, six.string_types):
                        f = "%s%s%s" % (f, self.separator, v)
                    feature_names.append(f)
            accum.add(set(feature_names))

        X.foreach(mapper)  # init vocabulary
        feature_names = list(accum.value)

        if self.sort:
            feature_names.sort()

        vocab = dict((f, i) for i, f in enumerate(feature_names))

        self.feature_names_ = feature_names
        self.vocabulary_ = vocab

        return self

Learn a list of feature name - > indices mappings .


def fit(self, Z):
        """Learn empirical variances from X.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape (n_samples, n_features)
            Sample vectors from which to compute variances.

        y : any
            Ignored. This parameter exists only for compatibility with
            sklearn.pipeline.Pipeline.

        Returns
        -------
        self
        """

        X = Z[:, 'X'] if isinstance(Z, DictRDD) else Z
        check_rdd(X, (np.ndarray, sp.spmatrix))

        def mapper(X):
            """Calculate statistics for every numpy or scipy blocks."""
            X = check_array(X, ('csr', 'csc'), dtype=np.float64)
            if hasattr(X, "toarray"):   # sparse matrix
                mean, var = mean_variance_axis(X, axis=0)
            else:
                mean, var = np.mean(X, axis=0), np.var(X, axis=0)
            return X.shape[0], mean, var

        def reducer(a, b):
            """Calculate the combined statistics."""
            n_a, mean_a, var_a = a
            n_b, mean_b, var_b = b
            n_ab = n_a + n_b
            mean_ab = ((mean_a * n_a) + (mean_b * n_b)) / n_ab
            var_ab = (((n_a * var_a) + (n_b * var_b)) / n_ab) + \
                     ((n_a * n_b) * ((mean_b - mean_a) / n_ab) ** 2)
            return (n_ab, mean_ab, var_ab)

        _, _, self.variances_ = X.map(mapper).treeReduce(reducer)

        if np.all(self.variances_ <= self.threshold):
            msg = "No feature in X meets the variance threshold {0:.5f}"
            if X.shape[0] == 1:
                msg += " (X contains only one sample)"
            raise ValueError(msg.format(self.threshold))

        return self


def fit_transform(self, Z):
        """Fit LSI model to X and perform dimensionality reduction on X.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape (n_samples, n_features)
            Training data.

        Returns
        -------
        X_new : array, shape (n_samples, n_components)
            Reduced version of X. This will always be a dense array.
        """
        X = Z[:, 'X'] if isinstance(Z, DictRDD) else Z
        check_rdd(X, (sp.spmatrix, np.ndarray))
        if self.algorithm == "em":
            X = X.persist()  # boosting iterative svm
            Sigma, V = svd_em(X, k=self.n_components, maxiter=self.n_iter,
                              tol=self.tol, compute_u=False,
                              seed=self.random_state)
            self.components_ = V
            X.unpersist()
            return self.transform(Z)
        else:
            # TODO: raise warning non distributed
            return super(SparkTruncatedSVD, self).fit_transform(X.tosparse())

def
fit_transform
(
self
,
Z
)
:
X
=
Z
[
:
,
'X'
]
if
isinstance
(
Z
,
DictRDD
)
else
Z
check_rdd
(
X
,
(
sp
.
spmatrix
,
np
.
ndarray
)
)
if
self
.
algorithm
==
"em"
:
X
=
X
.
persist
(
)
# boosting iterative svm
Sigma
,
V
=
svd_em
(
X
,
k
=
self
.
n_components
,
maxiter
=
self
.
n_iter
,
tol
=
self
.
tol
,
compute_u
=
False
,
seed
=
self
.
random_state
)
self
.
components_
=
V
X
.
unpersist
(
)
return
self
.
transform
(
Z
)
else
:
# TODO: raise warning non distributed
return
super
(
SparkTruncatedSVD
,
self
)
.
fit_transform
(
X
.
tosparse
(
)
)

Fit LSI model to X and perform dimensionality reduction on X .


def transform(self, Z):
        """Perform dimensionality reduction on X.

        Parameters
        ----------
        X : {array-like, sparse matrix}, shape (n_samples, n_features)
            New data.

        Returns
        -------
        X_new : array, shape (n_samples, n_components)
            Reduced version of X. This will always be a dense array.
        """
        X = Z[:, 'X'] if isinstance(Z, DictRDD) else Z
        check_rdd(X, (sp.spmatrix, np.ndarray))

        mapper = self.broadcast(
            super(SparkTruncatedSVD, self).transform, Z.context)
        return Z.transform(mapper, column='X', dtype=np.ndarray)

def transform ( self , Z ) : X = Z [ : , 'X' ] if isinstance ( Z , DictRDD ) else Z check_rdd ( X , ( sp . spmatrix , np . ndarray ) ) mapper = self . broadcast ( super ( SparkTruncatedSVD , self ) . transform , Z . context ) return Z . transform ( mapper , column = 'X' , dtype = np . ndarray )


def _block_collection(iterator, dtype, bsize=-1):
    """Pack rdd with a specific collection constructor."""
    i = 0
    accumulated = []
    for a in iterator:
        if (bsize > 0) and (i >= bsize):
            yield _pack_accumulated(accumulated, dtype)
            accumulated = []
            i = 0
        accumulated.append(a)
        i += 1
    if i > 0:
        yield _pack_accumulated(accumulated, dtype)

def _block_collection ( iterator , dtype , bsize = - 1 ) : i = 0 accumulated = [ ] for a in iterator : if ( bsize > 0 ) and ( i >= bsize ) : yield _pack_accumulated ( accumulated , dtype ) accumulated = [ ] i = 0 accumulated . append ( a ) i += 1 if i > 0 : yield _pack_accumulated ( accumulated , dtype )

Pack rdd with a specific collection constructor .


def _block_tuple(iterator, dtypes, bsize=-1):
    """Pack rdd of tuples as tuples of arrays or scipy.sparse matrices."""
    i = 0
    blocked_tuple = None
    for tuple_i in iterator:
        if blocked_tuple is None:
            blocked_tuple = tuple([] for _ in range(len(tuple_i)))

        if (bsize > 0) and (i >= bsize):
            yield tuple(_pack_accumulated(x, dtype)
                        for x, dtype in zip(blocked_tuple, dtypes))
            blocked_tuple = tuple([] for _ in range(len(tuple_i)))
            i = 0

        for x_j, x in zip(tuple_i, blocked_tuple):
            x.append(x_j)
        i += 1
    if i > 0:
        yield tuple(_pack_accumulated(x, dtype)
                    for x, dtype in zip(blocked_tuple, dtypes))

def _block_tuple ( iterator , dtypes , bsize = - 1 ) : i = 0 blocked_tuple = None for tuple_i in iterator : if blocked_tuple is None : blocked_tuple = tuple ( [ ] for _ in range ( len ( tuple_i ) ) ) if ( bsize > 0 ) and ( i >= bsize ) : yield tuple ( _pack_accumulated ( x , dtype ) for x , dtype in zip ( blocked_tuple , dtypes ) ) blocked_tuple = tuple ( [ ] for _ in range ( len ( tuple_i ) ) ) i = 0 for x_j , x in zip ( tuple_i , blocked_tuple ) : x . append ( x_j ) i += 1 if i > 0 : yield tuple ( _pack_accumulated ( x , dtype ) for x , dtype in zip ( blocked_tuple , dtypes ) )

Pack rdd of tuples as tuples of arrays or scipy . sparse matrices .


def block(rdd, bsize=-1, dtype=None):
    """Block an RDD

    Parameters
    ----------

    rdd : RDD
        RDD of data points to block into either numpy arrays,
        scipy sparse matrices, or pandas data frames.
        Type of data point will be automatically inferred
        and blocked accordingly.

    bsize : int, optional, default None
        Size of each block (number of elements), if None all data points
        from each partition will be combined in a block.

    Returns
    -------

    rdd : ArrayRDD or TupleRDD or DictRDD
        The transformed rdd with added functionality
    """
    try:
        entry = rdd.first()
    except IndexError:
        # empty RDD: do not block
        return rdd

    # do different kinds of block depending on the type
    if isinstance(entry, dict):
        rdd = rdd.map(lambda x: list(x.values()))
        return DictRDD(rdd, list(entry.keys()), bsize, dtype)
    elif isinstance(entry, tuple):
        return DictRDD(rdd, bsize=bsize, dtype=dtype)
    elif sp.issparse(entry):
        return SparseRDD(rdd, bsize)
    elif isinstance(entry, np.ndarray):
        return ArrayRDD(rdd, bsize)
    else:
        return BlockRDD(rdd, bsize, dtype)


def transform(self, fn, dtype=None, *args, **kwargs):
        """Equivalent to map, compatibility purpose only.
        Column parameter ignored.
        """
        rdd = self._rdd.map(fn)

        if dtype is None:
            return self.__class__(rdd, noblock=True, **self.get_params())
        if dtype is np.ndarray:
            return ArrayRDD(rdd, bsize=self.bsize, noblock=True)
        elif dtype is sp.spmatrix:
            return SparseRDD(rdd, bsize=self.bsize, noblock=True)
        else:
            return BlockRDD(rdd, bsize=self.bsize, dtype=dtype, noblock=True)

def transform ( self , fn , dtype = None , * args , * * kwargs ) : rdd = self . _rdd . map ( fn ) if dtype is None : return self . __class__ ( rdd , noblock = True , * * self . get_params ( ) ) if dtype is np . ndarray : return ArrayRDD ( rdd , bsize = self . bsize , noblock = True ) elif dtype is sp . spmatrix : return SparseRDD ( rdd , bsize = self . bsize , noblock = True ) else : return BlockRDD ( rdd , bsize = self . bsize , dtype = dtype , noblock = True )

Equivalent to map compatibility purpose only . Column parameter ignored .


def shape(self):
        """Returns the shape of the data."""
        # TODO cache
        first = self.first().shape
        shape = self._rdd.map(lambda x: x.shape[0]).sum()
        return (shape,) + first[1:]

def
shape
(
self
)
:
# TODO cache
first
=
self
.
first
(
)
.
shape
shape
=
self
.
_rdd
.
map
(
lambda
x
:
x
.
shape
[
0
]
)
.
sum
(
)
return
(
shape
,
)
+
first
[
1
:
]


def toarray(self):
        """Returns the data as numpy.array from each partition."""
        rdd = self._rdd.map(lambda x: x.toarray())
        return np.concatenate(rdd.collect())

def toarray ( self ) : rdd = self . _rdd . map ( lambda x : x . toarray ( ) ) return np . concatenate ( rdd . collect ( ) )

Returns the data as numpy . array from each partition .


def transform(self, fn, column=None, dtype=None):
        """Execute a transformation on a column or columns. Returns the modified
        DictRDD.

        Parameters
        ----------
        f : function
            The function to execute on the columns.
        column : {str, list or None}
            The column(s) to transform. If None is specified the method is
            equivalent to map.
        column : {str, list or None}
            The dtype of the column(s) to transform.

        Returns
        -------
        result : DictRDD
            DictRDD with transformed column(s).

        TODO: optimize
        """
        dtypes = self.dtype
        if column is None:
            indices = list(range(len(self.columns)))
        else:
            if not type(column) in (list, tuple):
                column = [column]
            indices = [self.columns.index(c) for c in column]

        if dtype is not None:
            if not type(dtype) in (list, tuple):
                dtype = [dtype]
            dtypes = [dtype[indices.index(i)] if i in indices else t
                      for i, t in enumerate(self.dtype)]

        def mapper(values):
            result = fn(*[values[i] for i in indices])

            if len(indices) == 1:
                result = (result,)
            elif not isinstance(result, (tuple, list)):
                raise ValueError("Transformer function must return an"
                                 " iterable!")
            elif len(result) != len(indices):
                raise ValueError("Transformer result's length must be"
                                 " equal to the given columns length!")

            return tuple(result[indices.index(i)] if i in indices else v
                         for i, v in enumerate(values))

        return DictRDD(self._rdd.map(mapper),
                       columns=self.columns, dtype=dtypes,
                       bsize=self.bsize, noblock=True)

def
transform
(
self
,
fn
,
column
=
,
dtype
=
)
:
dtypes
=
self
.
dtype
if
column
is
:
indices
=
list
(
range
(
len
(
self
.
columns
)
)
)
else
:
if
not
type
(
column
)
in
(
list
,
tuple
)
:
column
=
[
column
]
indices
=
[
self
.
columns
.
index
(
c
)
for
c
in
column
]
if
dtype
is
not
:
if
not
type
(
dtype
)
in
(
list
,
tuple
)
:
dtype
=
[
dtype
]
dtypes
=
[
dtype
[
indices
.
index
(
i
)
]
if
i
in
indices
else
t
for
i
,
t
in
enumerate
(
self
.
dtype
)
]
def
mapper
(
values
)
:
result
=
fn
(
*
[
values
[
i
]
for
i
in
indices
]
)
if
len
(
indices
)
==
1
:
result
=
(
result
,
)
elif
not
isinstance
(
result
,
(
tuple
,
list
)
)
:
raise
ValueError
(
"Transformer function must return an"
" iterable!"
)
elif
len
(
result
)
!=
len
(
indices
)
:
raise
ValueError
(
"Transformer result's length must be"
" equal to the given columns length!"
)
return
tuple
(
result
[
indices
.
index
(
i
)
]
if
i
in
indices
else
v
for
i
,
v
in
enumerate
(
values
)
)
return
DictRDD
(
self
.
_rdd
.
map
(
mapper
)
,
columns
=
self
.
columns
,
dtype
=
dtypes
,
bsize
=
self
.
bsize
,
noblock
=
True
)

Execute a transformation on a column or columns . Returns the modified DictRDD .


def bitperm(s, perm, pos):
    """Returns zero if there are no permissions for a bit of the perm. of a file. Otherwise it returns a positive value

    :param os.stat_result s: os.stat(file) object
    :param str perm: R (Read) or W (Write) or X (eXecute)
    :param str pos: USR (USeR) or GRP (GRouP) or OTH (OTHer)
    :return: mask value
    :rtype: int
    """
    perm = perm.upper()
    pos = pos.upper()
    assert perm in ['R', 'W', 'X']
    assert pos in ['USR', 'GRP', 'OTH']
    return s.st_mode & getattr(stat, 'S_I{}{}'.format(perm, pos))

def bitperm ( s , perm , pos ) : perm = perm . upper ( ) pos = pos . upper ( ) assert perm in [ 'R' , 'W' , 'X' ] assert pos in [ 'USR' , 'GRP' , 'OTH' ] return s . st_mode & getattr ( stat , 'S_I{}{}' . format ( perm , pos ) )

Returns zero if there are no permissions for a bit of the perm . of a file . Otherwise it returns a positive value


def only_root_write(path):
    """File is only writable by root

    :param str path: Path to file
    :return: True if only root can write
    :rtype: bool
    """
    s = os.stat(path)
    for ug, bp in [(s.st_uid, bitperm(s, 'w', 'usr')), (s.st_gid, bitperm(s, 'w', 'grp'))]:
        # User id (is not root) and bit permission
        if ug and bp:
            return False
    if bitperm(s, 'w', 'oth'):
        return False
    return True


def check_config(file, printfn=print):
    """Command to check configuration file. Raises InvalidConfig on error

    :param str file: path to config file
    :param printfn: print function for success message
    :return: None
    """
    Config(file).read()
    printfn('The configuration file "{}" is correct'.format(file))

Command to check configuration file . Raises InvalidConfig on error


def read(self):
        """Parse and validate the config file. The read data is accessible as a dictionary in this instance

        :return: None
        """
        try:
            data = load(open(self.file), Loader)
        except (UnicodeDecodeError, YAMLError) as e:
            raise InvalidConfig(self.file, '{}'.format(e))
        try:
            validate(data, SCHEMA)
        except ValidationError as e:
            raise InvalidConfig(self.file, e)
        self.update(data)

def read ( self ) : try : data = load ( open ( self . file ) , Loader ) except ( UnicodeDecodeError , YAMLError ) as e : raise InvalidConfig ( self . file , '{}' . format ( e ) ) try : validate ( data , SCHEMA ) except ValidationError as e : raise InvalidConfig ( self . file , e ) self . update ( data )

Parse and validate the config file . The read data is accessible as a dictionary in this instance


def run_as_cmd(cmd, user, shell='bash'):
    """Get the arguments to execute a command as a user

    :param str cmd: command to execute
    :param user: User for use
    :param shell: Bash, zsh, etc.
    :return: arguments
    :rtype: list
    """
    to_execute = get_shell(shell) + [EXECUTE_SHELL_PARAM, cmd]
    if user == 'root':
        return to_execute
    return ['sudo', '-s', '--set-home', '-u', user] + to_execute

def run_as_cmd ( cmd , user , shell = 'bash' ) : to_execute = get_shell ( shell ) + [ EXECUTE_SHELL_PARAM , cmd ] if user == 'root' : return to_execute return [ 'sudo' , '-s' , '--set-home' , '-u' , user ] + to_execute


def execute_cmd(cmd, cwd=None, timeout=5):
    """Excecute command on thread

    :param cmd: Command to execute
    :param cwd: current working directory
    :return: None
    """
    p = subprocess.Popen(cmd, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    try:
        p.wait(timeout=timeout)
    except subprocess.TimeoutExpired:
        return None
    else:
        stdout, stderr = p.stdout.read(), p.stderr.read()
        if sys.version_info >= (3,):
            stdout, stderr = stdout.decode('utf-8', errors='ignore'), stderr.decode('utf-8', errors='ignore')
        if p.returncode:
            raise ExecuteError('Error running command {}: The error code {} has returned. Stderr: {}'.format(
                ' '.join(cmd), p.returncode, stderr
            ))
        else:
            return stdout, stderr


def execute_over_ssh(cmd, ssh, cwd=None, shell='bash'):
    """Excecute command on remote machine using SSH

    :param cmd: Command to execute
    :param ssh: Server to connect. Port is optional
    :param cwd: current working directory
    :return: None
    """
    port = None
    parts = ssh.split(':', 1)
    if len(parts) > 1 and not parts[1].isdigit():
        raise InvalidConfig(extra_body='Invalid port number on ssh config: {}'.format(parts[1]))
    elif len(parts) > 1:
        port = parts[1]
    quoted_cmd = ' '.join([x.replace("'", """'"'"'""") for x in cmd.split(' ')])
    remote_cmd = ' '.join([
        ' '.join(get_shell(shell)), # /usr/bin/env bash
        ' '.join([EXECUTE_SHELL_PARAM, "'", ' '.join((['cd', cwd, ';'] if cwd else []) + [quoted_cmd]), "'"])],
    )
    return ['ssh', parts[0]] + (['-p', port] if port else []) + ['-C'] + [remote_cmd]

def
execute_over_ssh
(
cmd
,
ssh
,
cwd
=
,
shell
=
'bash'
)
:
port
=
parts
=
ssh
.
split
(
':'
,
1
)
if
len
(
parts
)
>
1
and
not
parts
[
1
]
.
isdigit
(
)
:
raise
InvalidConfig
(
extra_body
=
'Invalid port number on ssh config: {}'
.
format
(
parts
[
1
]
)
)
elif
len
(
parts
)
>
1
:
port
=
parts
[
1
]
quoted_cmd
=
' '
.
join
(
[
x
.
replace
(
"'"
,
"""'"'"'"""
)
for
x
in
cmd
.
split
(
' '
)
]
)
remote_cmd
=
' '
.
join
(
[
' '
.
join
(
get_shell
(
shell
)
)
,
# /usr/bin/env bash
' '
.
join
(
[
EXECUTE_SHELL_PARAM
,
"'"
,
' '
.
join
(
(
[
'cd'
,
cwd
,
';'
]
if
cwd
else
[
]
)
+
[
quoted_cmd
]
)
,
"'"
]
)
]
,
)
return
[
'ssh'
,
parts
[
0
]
]
+
(
[
'-p'
,
port
]
if
port
else
[
]
)
+
[
'-C'
]
+
[
remote_cmd
]


def validate(self):
        """Check self.data. Raise InvalidConfig on error

        :return: None
        """
        if (self.data.get('content-type') or self.data.get('body')) and \
                self.data.get('method', '').lower() not in CONTENT_TYPE_METHODS:
            raise InvalidConfig(
                extra_body='The body/content-type option only can be used with the {} methods. The device is {}. '
                           'Check the configuration file.'.format(', '.join(CONTENT_TYPE_METHODS), self.name)
            )
        self.data['content-type'] = CONTENT_TYPE_ALIASES.get(self.data.get('content-type'),
                                                             self.data.get('content-type'))
        form_type = CONTENT_TYPE_ALIASES['form']
        if self.data.get('body') and (self.data.get('content-type') or form_type) == form_type:
            try:
                self.data['body'] = json.loads(self.data['body'])
            except JSONDecodeError:
                raise InvalidConfig(
                    extra_body='Invalid JSON body on {} device.'.format(self.name)
                )

Check self . data . Raise InvalidConfig on error


def get_headers(self):
        """Get HTTP Headers to send. By default default_headers

        :return: HTTP Headers
        :rtype: dict
        """
        headers = copy.copy(self.default_headers or {})
        headers.update(self.data.get('headers') or {})
        return headers

def get_headers ( self ) : headers = copy . copy ( self . default_headers or { } ) headers . update ( self . data . get ( 'headers' ) or { } ) return headers

Get HTTP Headers to send . By default default_headers


def get_body(self):
        """Return "data" value on self.data

        :return: data to send
        :rtype: str
        """
        if self.default_body:
            return self.default_body
        data = self.data.get('data')
        if isinstance(data, dict):
            return json.dumps(data)
        return data

def get_body ( self ) : if self . default_body : return self . default_body data = self . data . get ( 'data' ) if isinstance ( data , dict ) : return json . dumps ( data ) return data


def get_url(self):
        """Home assistant url

        :return: url
        :rtype: str
        """
        url = super(ExecuteHomeAssistant, self).get_url()
        if not self.data.get('event'):
            raise InvalidConfig(extra_body='Event option is required for HomeAsistant on {} device.'.format(self.name))
        url += '/api/events/{}'.format(self.data['event'])
        return url


def get_url(self):
        """IFTTT Webhook url

        :return: url
        :rtype: str
        """
        if not self.data[self.execute_name]:
            raise InvalidConfig(extra_body='Value for IFTTT is required on {} device. Get your key here: '
                                           'https://ifttt.com/services/maker_webhooks/settings'.format(self.name))
        if not self.data.get('event'):
            raise InvalidConfig(extra_body='Event option is required for IFTTT on {} device. '
                                           'You define the event name when creating a Webhook '
                                           'applet'.format(self.name))
        url = self.url_pattern.format(event=self.data['event'], key=self.data[self.execute_name])
        return url


def pkt_text(pkt):
    """Return source mac address for this Scapy Packet

    :param scapy.packet.Packet pkt: Scapy Packet
    :return: Mac address. Include (Amazon Device) for these devices
    :rtype: str
    """
    if pkt.src.upper() in BANNED_DEVICES:
        body = ''
    elif pkt.src.upper()[:8] in AMAZON_DEVICES:
        body = '{} (Amazon Device)'.format(pkt.src)
    else:
        body = pkt.src
    return body

def
pkt_text
(
pkt
)
:
if
pkt
.
src
.
upper
(
)
in
BANNED_DEVICES
:
body
=
''
elif
pkt
.
src
.
upper
(
)
[
:
8
]
in
AMAZON_DEVICES
:
body
=
'{} (Amazon Device)'
.
format
(
pkt
.
src
)
else
:
body
=
pkt
.
src
return
body


def discovery_print(pkt):
    """Scandevice callback. Register src mac to avoid src repetition.
    Print device on screen.

    :param scapy.packet.Packet pkt: Scapy Packet
    :return: None
    """
    if pkt.src in mac_id_list:
        return
    mac_id_list.append(pkt.src)
    text = pkt_text(pkt)
    click.secho(text, fg='magenta') if 'Amazon' in text else click.echo(text)

def discovery_print ( pkt ) : if pkt . src in mac_id_list : return mac_id_list . append ( pkt . src ) text = pkt_text ( pkt ) click . secho ( text , fg = 'magenta' ) if 'Amazon' in text else click . echo ( text )

Scandevice callback . Register src mac to avoid src repetition . Print device on screen .


def discover(interface=None):
    """Print help and scan devices on screen.

    :return: None
    """
    click.secho(HELP, fg='yellow')
    scan_devices(discovery_print, lfilter=lambda d: d.src not in mac_id_list, iface=interface)

def discover ( interface = None ) : click . secho ( HELP , fg = 'yellow' ) scan_devices ( discovery_print , lfilter = lambda d : d . src not in mac_id_list , iface = interface )


def execute(self, root_allowed=False):
        """Execute this device

        :param bool root_allowed: Only used for ExecuteCmd
        :return: None
        """
        logger.debug('%s device executed (mac %s)', self.name, self.src)
        if not self.execute_instance:
            msg = '%s: There is not execution method in device conf.'
            logger.warning(msg, self.name)
            self.send_confirmation(msg % self.name, False)
            return
        try:
            result = self.execute_instance.execute(root_allowed)
        except Exception as e:
            self.send_confirmation('Error executing the device {}: {}'.format(self.name, e), False)
            raise
        else:
            result = 'The {} device has been started and is running right now'.format(self.name) \
                if result is None else result
            result = result or 'The {} device has been executed successfully'.format(self.name)
            self.send_confirmation(result)
        return result


def send_confirmation(self, message, success=True):
        """Send success or error message to configured confirmation

        :param str message: Body message to send
        :param bool success: Device executed successfully to personalize message
        :return: None
        """
        message = message.strip()
        if not self.confirmation:
            return
        try:
            self.confirmation.send(message, success)
        except Exception as e:
            logger.warning('Error sending confirmation on device {}: {}'.format(self.name, e))


def on_push(self, device):
        """Press button. Check DEFAULT_DELAY.

        :param scapy.packet.Packet device: Scapy packet
        :return: None
        """
        src = device.src.lower()
        if last_execution[src] + self.settings.get('delay', DEFAULT_DELAY) > time.time():
            return
        last_execution[src] = time.time()
        self.execute(device)

def on_push ( self , device ) : src = device . src . lower ( ) if last_execution [ src ] + self . settings . get ( 'delay' , DEFAULT_DELAY ) > time . time ( ) : return last_execution [ src ] = time . time ( ) self . execute ( device )


def execute(self, device):
        """Execute a device. Used if the time between executions is greater than DEFAULT_DELAY

        :param scapy.packet.Packet device: Scapy packet
        :return: None
        """
        src = device.src.lower()
        device = self.devices[src]
        threading.Thread(target=device.execute, kwargs={
            'root_allowed': self.root_allowed
        }).start()

def execute ( self , device ) : src = device . src . lower ( ) device = self . devices [ src ] threading . Thread ( target = device . execute , kwargs = { 'root_allowed' : self . root_allowed } ) . start ( )

Execute a device . Used if the time between executions is greater than DEFAULT_DELAY


def run(self, root_allowed=False):
        """Start daemon mode

        :param bool root_allowed: Only used for ExecuteCmd
        :return: loop
        """
        self.root_allowed = root_allowed
        scan_devices(self.on_push, lambda d: d.src.lower() in self.devices, self.settings.get('interface'))

def run ( self , root_allowed = False ) : self . root_allowed = root_allowed scan_devices ( self . on_push , lambda d : d . src . lower ( ) in self . devices , self . settings . get ( 'interface' ) )


def convert(self, txn):
        """
        Convert an OFX Transaction to a posting
        """

        ofxid = self.mk_ofxid(txn.id)
        metadata = {}
        posting_metadata = {"ofxid": ofxid}

        if isinstance(txn, OfxTransaction):
            posting = Posting(self.name,
                              Amount(txn.amount, self.currency),
                              metadata=posting_metadata)
            return Transaction(
                date=txn.date,
                payee=self.format_payee(txn),
                postings=[
                    posting,
                    posting.clone_inverted(
                        self.mk_dynamic_account(self.format_payee(txn),
                                                exclude=self.name))])
        elif isinstance(txn, InvestmentTransaction):
            acct1 = self.name
            acct2 = self.name

            posting1 = None
            posting2 = None

            security = self.maybe_get_ticker(txn.security)

            if isinstance(txn.type, str):
                # recent versions of ofxparse
                if re.match('^(buy|sell)', txn.type):
                    acct2 = self.unknownaccount or 'Assets:Unknown'
                elif txn.type == 'transfer':
                    acct2 = 'Transfer'
                elif txn.type == 'reinvest':
                    # reinvestment of income
                    # TODO: make this configurable
                    acct2 = 'Income:Interest'
                elif txn.type == 'income' and txn.income_type == 'DIV':
                    # Fidelity lists non-reinvested dividend income as
                    # type: income, income_type: DIV
                    # TODO: determine how dividend income is listed from other institutions
                    # income/DIV transactions do not involve buying or selling a security
                    # so their postings need special handling compared to
                    # others
                    metadata['dividend_from'] = security
                    acct2 = 'Income:Dividends'
                    posting1 = Posting(acct1,
                                       Amount(txn.total, self.currency),
                                       metadata=posting_metadata)
                    posting2 = posting1.clone_inverted(acct2)
                else:
                    # ???
                    pass
            else:
                # Old version of ofxparse
                if (txn.type in [0, 1, 3, 4]):
                    # buymf, sellmf, buystock, sellstock
                    acct2 = self.unknownaccount or 'Assets:Unknown'
                elif (txn.type == 2):
                    # reinvest
                    acct2 = 'Income:Interest'
                else:
                    # ???
                    pass

            aux_date = None
            if txn.settleDate is not None and \
               txn.settleDate != txn.tradeDate:
                aux_date = txn.settleDate

            # income/DIV already defined above;
            # this block defines all other posting types
            if posting1 is None and posting2 is None:
                posting1 = Posting(
                    acct1,
                    Amount(
                        txn.units,
                        security,
                        unlimited=True),
                    unit_price=Amount(
                        txn.unit_price,
                        self.currency,
                        unlimited=True),
                    metadata=posting_metadata)
                posting2 = Posting(
                    acct2,
                    Amount(
                        txn.units *
                        txn.unit_price,
                        self.currency,
                        reverse=True))
            else:
                # Previously defined if type:income income_type/DIV
                pass

            return Transaction(
                date=txn.tradeDate,
                aux_date=aux_date,
                payee=self.format_payee(txn),
                metadata=metadata,
                postings=[posting1, posting2]
            )


def find_ledger_file(ledgerrcpath=None):
    """Returns main ledger file path or raise exception if it cannot be \
found."""
    if ledgerrcpath is None:
        ledgerrcpath = os.path.abspath(os.path.expanduser("~/.ledgerrc"))
    if "LEDGER_FILE" in os.environ:
        return os.path.abspath(os.path.expanduser(os.environ["LEDGER_FILE"]))
    elif os.path.exists(ledgerrcpath):
        # hacky
        ledgerrc = open(ledgerrcpath)
        for line in ledgerrc.readlines():
            md = re.match(r"--file\s+([^\s]+).*", line)
            if md is not None:
                return os.path.abspath(os.path.expanduser(md.group(1)))
    else:
        return None

def
find_ledger_file
(
ledgerrcpath
=
)
:
if
ledgerrcpath
is
:
ledgerrcpath
=
os
.
path
.
abspath
(
os
.
path
.
expanduser
(
"~/.ledgerrc"
)
)
if
"LEDGER_FILE"
in
os
.
environ
:
return
os
.
path
.
abspath
(
os
.
path
.
expanduser
(
os
.
environ
[
"LEDGER_FILE"
]
)
)
elif
os
.
path
.
exists
(
ledgerrcpath
)
:
# hacky
ledgerrc
=
open
(
ledgerrcpath
)
for
line
in
ledgerrc
.
readlines
(
)
:
md
=
re
.
match
(
r"--file\s+([^\s]+).*"
,
line
)
if
md
is
not
:
return
os
.
path
.
abspath
(
os
.
path
.
expanduser
(
md
.
group
(
1
)
)
)
else
:
return

Returns main ledger file path or raise exception if it cannot be \ found .


def compatibility(session, install):
    """Run the unit test suite with each support library and Python version."""

    session.install('-e', '.[dev]')
    session.install(install)
    _run_tests(session)

def compatibility ( session , install ) : session . install ( '-e' , '.[dev]' ) session . install ( install ) _run_tests ( session )

Run the unit test suite with each support library and Python version .


def get_long_description():
    """Transform README.md into a usable long description.

    Replaces relative references to svg images to absolute https references.
    """

    with open('README.md') as f:
        read_me = f.read()

    def replace_relative_with_absolute(match):
        svg_path = match.group(0)[1:-1]
        return ('(https://github.com/google/pybadges/raw/master/'
                '%s?sanitize=true)' % svg_path)

    return re.sub(r'\(tests/golden-images/.*?\.svg\)',
                  replace_relative_with_absolute,
                  read_me)

Transform README . md into a usable long description .


def from_json(f: TextIO) -> 'PrecalculatedTextMeasurer':
        """Return a PrecalculatedTextMeasurer given a JSON stream.

        See precalculate_text.py for details on the required format.
        """
        o = json.load(f)
        return PrecalculatedTextMeasurer(o['mean-character-length'],
                                         o['character-lengths'],
                                         o['kerning-pairs'])

def from_json ( f : TextIO ) -> 'PrecalculatedTextMeasurer' : o = json . load ( f ) return PrecalculatedTextMeasurer ( o [ 'mean-character-length' ] , o [ 'character-lengths' ] , o [ 'kerning-pairs' ] )

Return a PrecalculatedTextMeasurer given a JSON stream .


def default(cls) -> 'PrecalculatedTextMeasurer':
        """Returns a reasonable default PrecalculatedTextMeasurer."""
        if cls._default_cache is not None:
            return cls._default_cache

        if pkg_resources.resource_exists(__name__, 'default-widths.json.xz'):
            import lzma
            with pkg_resources.resource_stream(__name__,
                                               'default-widths.json.xz') as f:
                with lzma.open(f, "rt") as g:
                    cls._default_cache = PrecalculatedTextMeasurer.from_json(
                        cast(TextIO, g))
                    return cls._default_cache
        elif pkg_resources.resource_exists(__name__, 'default-widths.json'):
            with pkg_resources.resource_stream(__name__,
                                               'default-widths.json') as f:
                cls._default_cache = PrecalculatedTextMeasurer.from_json(
                    io.TextIOWrapper(f, encoding='utf-8'))
                return cls._default_cache
        else:
            raise ValueError('could not load default-widths.json')

def
default
(
cls
)
->
'PrecalculatedTextMeasurer'
:
if
cls
.
_default_cache
is
not
:
return
cls
.
_default_cache
if
pkg_resources
.
resource_exists
(
__name__
,
'default-widths.json.xz'
)
:
import
lzma
with
pkg_resources
.
resource_stream
(
__name__
,
'default-widths.json.xz'
)
as
f
:
with
lzma
.
open
(
f
,
"rt"
)
as
g
:
cls
.
_default_cache
=
PrecalculatedTextMeasurer
.
from_json
(
cast
(
TextIO
,
g
)
)
return
cls
.
_default_cache
elif
pkg_resources
.
resource_exists
(
__name__
,
'default-widths.json'
)
:
with
pkg_resources
.
resource_stream
(
__name__
,
'default-widths.json'
)
as
f
:
cls
.
_default_cache
=
PrecalculatedTextMeasurer
.
from_json
(
io
.
TextIOWrapper
(
f
,
encoding
=
'utf-8'
)
)
return
cls
.
_default_cache
else
:
raise
ValueError
(
'could not load default-widths.json'
)

Returns a reasonable default PrecalculatedTextMeasurer .


def badge(left_text: str, right_text: str, left_link: Optional[str] = None,
          right_link: Optional[str] = None,
          whole_link: Optional[str] = None, logo: Optional[str] = None,
          left_color: str = '#555', right_color: str = '#007ec6',
          measurer: Optional[text_measurer.TextMeasurer] = None,
          embed_logo: bool = False) -> str:
    """Creates a github-style badge as an SVG image.

    >>> badge(left_text='coverage', right_text='23%', right_color='red')
    '<svg...</svg>'
    >>> badge(left_text='build', right_text='green', right_color='green',
    ...       whole_link="http://www.example.com/")
    '<svg...</svg>'

    Args:
        left_text: The text that should appear on the left-hand-side of the
            badge e.g. "coverage".
        right_text: The text that should appear on the right-hand-side of the
            badge e.g. "23%".
        left_link: The URL that should be redirected to when the left-hand text
            is selected.
        right_link: The URL that should be redirected to when the right-hand
            text is selected.
        whole_link: The link that should be redirected to when the badge is
            selected. If set then left_link and right_right may not be set.
        logo: A url representing a logo that will be displayed inside the
            badge. Can be a data URL e.g. "data:image/svg+xml;utf8,<svg..."
        left_color: The color of the part of the badge containing the left-hand
            text. Can be an valid CSS color
            (see https://developer.mozilla.org/en-US/docs/Web/CSS/color) or a
            color name defined here:
            https://github.com/badges/shields/blob/master/lib/colorscheme.json
        right_color: The color of the part of the badge containing the
            right-hand text. Can be an valid CSS color
            (see https://developer.mozilla.org/en-US/docs/Web/CSS/color) or a
            color name defined here:
            https://github.com/badges/shields/blob/master/lib/colorscheme.json
        measurer: A text_measurer.TextMeasurer that can be used to measure the
            width of left_text and right_text.
        embed_logo: If True then embed the logo image directly in the badge.
            This can prevent an HTTP request and some browsers will not render
            external image referenced. When True, `logo` must be a HTTP/HTTPS
            URI or a filesystem path. Also, the `badge` call may raise an
            exception if the logo cannot be loaded, is not an image, etc.
    """
    if measurer is None:
        measurer = (
            precalculated_text_measurer.PrecalculatedTextMeasurer
                .default())

    if (left_link or right_link) and whole_link:
        raise ValueError(
            'whole_link may not bet set with left_link or right_link')
    template = _JINJA2_ENVIRONMENT.get_template('badge-template-full.svg')

    if logo and embed_logo:
        logo = _embed_image(logo)

    svg = template.render(
        left_text=left_text,
        right_text=right_text,
        left_text_width=measurer.text_width(left_text) / 10.0,
        right_text_width=measurer.text_width(right_text) / 10.0,
        left_link=left_link,
        right_link=right_link,
        whole_link=whole_link,
        logo=logo,
        left_color=_NAME_TO_COLOR.get(left_color, left_color),
        right_color=_NAME_TO_COLOR.get(right_color, right_color),
    )
    xml = minidom.parseString(svg)
    _remove_blanks(xml)
    xml.normalize()
    return xml.documentElement.toxml()

Creates a github - style badge as an SVG image .


def generate_supported_characters(deja_vu_sans_path: str) -> Iterable[str]:
    """Generate the characters support by the font at the given path."""
    font = ttLib.TTFont(deja_vu_sans_path)
    for cmap in font['cmap'].tables:
        if cmap.isUnicode():
            for code in cmap.cmap:
                yield chr(code)

def generate_supported_characters ( deja_vu_sans_path : str ) -> Iterable [ str ] : font = ttLib . TTFont ( deja_vu_sans_path ) for cmap in font [ 'cmap' ] . tables : if cmap . isUnicode ( ) : for code in cmap . cmap : yield chr ( code )

Generate the characters support by the font at the given path .


def generate_encodeable_characters(characters: Iterable[str],
                                   encodings: Iterable[str]) -> Iterable[str]:
    """Generates the subset of 'characters' that can be encoded by 'encodings'.

    Args:
        characters: The characters to check for encodeability e.g. 'abcd'.
        encodings: The encodings to check against e.g. ['cp1252', 'iso-8859-5'].

    Returns:
        The subset of 'characters' that can be encoded using one of the provided
        encodings.
    """
    for c in characters:
        for encoding in encodings:
            try:
                c.encode(encoding)
                yield c
            except UnicodeEncodeError:
                pass

def generate_encodeable_characters ( characters : Iterable [ str ] , encodings : Iterable [ str ] ) -> Iterable [ str ] : for c in characters : for encoding in encodings : try : c . encode ( encoding ) yield c except UnicodeEncodeError : pass

Generates the subset of characters that can be encoded by encodings .


def calculate_character_to_length_mapping(
        measurer: text_measurer.TextMeasurer,
        characters: Iterable[str]) -> Mapping[str, float]:
    """Return a mapping between each given character and its length.

    Args:
        measurer: The TextMeasurer used to measure the width of the text in
            pixels.
        characters: The characters to measure e.g. "ml".

    Returns:
        A mapping from the given characters to their length in pixels, as
        determined by 'measurer' e.g. {'m': 5.2, 'l', 1.2}.
    """
    char_to_length = {}

    for c in characters:
        char_to_length[c] = measurer.text_width(c)
    return char_to_length

def calculate_character_to_length_mapping ( measurer : text_measurer . TextMeasurer , characters : Iterable [ str ] ) -> Mapping [ str , float ] : char_to_length = { } for c in characters : char_to_length [ c ] = measurer . text_width ( c ) return char_to_length

Return a mapping between each given character and its length .


def write_json(f: TextIO, deja_vu_sans_path: str,
               measurer: text_measurer.TextMeasurer,
               encodings: Iterable[str]) -> None:
    """Write the data required by PrecalculatedTextMeasurer to a stream."""
    supported_characters = list(
        generate_supported_characters(deja_vu_sans_path))
    kerning_characters = ''.join(
        generate_encodeable_characters(supported_characters, encodings))
    char_to_length = calculate_character_to_length_mapping(measurer,
                                                           supported_characters)
    pair_to_kerning = calculate_pair_to_kern_mapping(measurer, char_to_length,
                                                     kerning_characters)
    json.dump(
        {'mean-character-length': statistics.mean(char_to_length.values()),
         'character-lengths': char_to_length,
         'kerning-characters': kerning_characters,
         'kerning-pairs': pair_to_kerning},
        f, sort_keys=True, indent=1)

def write_json ( f : TextIO , deja_vu_sans_path : str , measurer : text_measurer . TextMeasurer , encodings : Iterable [ str ] ) -> None : supported_characters = list ( generate_supported_characters ( deja_vu_sans_path ) ) kerning_characters = '' . join ( generate_encodeable_characters ( supported_characters , encodings ) ) char_to_length = calculate_character_to_length_mapping ( measurer , supported_characters ) pair_to_kerning = calculate_pair_to_kern_mapping ( measurer , char_to_length , kerning_characters ) json . dump ( { 'mean-character-length' : statistics . mean ( char_to_length . values ( ) ) , 'character-lengths' : char_to_length , 'kerning-characters' : kerning_characters , 'kerning-pairs' : pair_to_kerning } , f , sort_keys = True , indent = 1 )

Write the data required by PrecalculatedTextMeasurer to a stream .


def convolve_gaussian_2d(image, gaussian_kernel_1d):
    """Convolve 2d gaussian."""
    result = scipy.ndimage.filters.correlate1d(
        image, gaussian_kernel_1d, axis=0)
    result = scipy.ndimage.filters.correlate1d(
        result, gaussian_kernel_1d, axis=1)
    return result

def convolve_gaussian_2d ( image , gaussian_kernel_1d ) : result = scipy . ndimage . filters . correlate1d ( image , gaussian_kernel_1d , axis = 0 ) result = scipy . ndimage . filters . correlate1d ( result , gaussian_kernel_1d , axis = 1 ) return result


def get_gaussian_kernel(gaussian_kernel_width=11, gaussian_kernel_sigma=1.5):
    """Generate a gaussian kernel."""
    # 1D Gaussian kernel definition
    gaussian_kernel_1d = numpy.ndarray((gaussian_kernel_width))
    norm_mu = int(gaussian_kernel_width / 2)

    # Fill Gaussian kernel
    for i in range(gaussian_kernel_width):
        gaussian_kernel_1d[i] = (exp(-(((i - norm_mu) ** 2)) /
                                     (2 * (gaussian_kernel_sigma ** 2))))
    return gaussian_kernel_1d / numpy.sum(gaussian_kernel_1d)

def
get_gaussian_kernel
(
gaussian_kernel_width
=
11
,
gaussian_kernel_sigma
=
1.5
)
:
# 1D Gaussian kernel definition
gaussian_kernel_1d
=
numpy
.
ndarray
(
(
gaussian_kernel_width
)
)
norm_mu
=
int
(
gaussian_kernel_width
/
2
)
# Fill Gaussian kernel
for
i
in
range
(
gaussian_kernel_width
)
:
gaussian_kernel_1d
[
i
]
=
(
exp
(
(
(
(
i
norm_mu
)
**
2
)
)
/
(
2
*
(
gaussian_kernel_sigma
**
2
)
)
)
)
return
gaussian_kernel_1d
/
numpy
.
sum
(
gaussian_kernel_1d
)


def to_grayscale(img):
    """Convert PIL image to numpy grayscale array and numpy alpha array.

    Args:
      img (PIL.Image): PIL Image object.

    Returns:
      (gray, alpha): both numpy arrays.
    """
    gray = numpy.asarray(ImageOps.grayscale(img)).astype(numpy.float)

    imbands = img.getbands()
    alpha = None
    if 'A' in imbands:
        alpha = numpy.asarray(img.split()[-1]).astype(numpy.float)

    return gray, alpha

def to_grayscale ( img ) : gray = numpy . asarray ( ImageOps . grayscale ( img ) ) . astype ( numpy . float ) imbands = img . getbands ( ) alpha = None if 'A' in imbands : alpha = numpy . asarray ( img . split ( ) [ - 1 ] ) . astype ( numpy . float ) return gray , alpha

Convert PIL image to numpy grayscale array and numpy alpha array .


def main():
    """Main function for pyssim."""

    description = '\n'.join([
        'Compares an image with a list of images using the SSIM metric.',
        '  Example:',
        '    pyssim test-images/test1-1.png "test-images/*"'
    ])

    parser = argparse.ArgumentParser(
        prog='pyssim', formatter_class=argparse.RawTextHelpFormatter,
        description=description)
    parser.add_argument('--cw', help='compute the complex wavelet SSIM',
                        action='store_true')
    parser.add_argument(
        'base_image', metavar='image1.png', type=argparse.FileType('r'))
    parser.add_argument(
        'comparison_images', metavar='image path with* or image2.png')
    parser.add_argument('--width', type=int, default=None,
                        help='scales the image before computing SSIM')
    parser.add_argument('--height', type=int, default=None,
                        help='scales the image before computing SSIM')

    args = parser.parse_args()

    if args.width and args.height:
        size = (args.width, args.height)
    else:
        size = None

    if not args.cw:
        gaussian_kernel_sigma = 1.5
        gaussian_kernel_width = 11
        gaussian_kernel_1d = get_gaussian_kernel(
            gaussian_kernel_width, gaussian_kernel_sigma)

    comparison_images = glob.glob(args.comparison_images)
    is_a_single_image = len(comparison_images) == 1

    for comparison_image in comparison_images:

        if args.cw:
            ssim = SSIM(args.base_image.name, size=size)
            ssim_value = ssim.cw_ssim_value(comparison_image)
        else:
            ssim = SSIM(args.base_image.name, gaussian_kernel_1d, size=size)
            ssim_value = ssim.ssim_value(comparison_image)

        if is_a_single_image:
            sys.stdout.write('%.7g' % ssim_value)
        else:
            sys.stdout.write('%s - %s: %.7g' % (
                args.base_image.name, comparison_image, ssim_value))
        sys.stdout.write('\n')


def ssim_value(self, target):
        """Compute the SSIM value from the reference image to the target image.

        Args:
          target (str or PIL.Image): Input image to compare the reference image
          to. This may be a PIL Image object or, to save time, an SSIMImage
          object (e.g. the img member of another SSIM object).

        Returns:
          Computed SSIM float value.
        """
        # Performance boost if handed a compatible SSIMImage object.
        if not isinstance(target, SSIMImage) \
          or not np.array_equal(self.gaussian_kernel_1d,
                                target.gaussian_kernel_1d):
            target = SSIMImage(target, self.gaussian_kernel_1d, self.img.size)

        img_mat_12 = self.img.img_gray * target.img_gray
        img_mat_sigma_12 = convolve_gaussian_2d(
            img_mat_12, self.gaussian_kernel_1d)
        img_mat_mu_12 = self.img.img_gray_mu * target.img_gray_mu
        img_mat_sigma_12 = img_mat_sigma_12 - img_mat_mu_12

        # Numerator of SSIM
        num_ssim = ((2 * img_mat_mu_12 + self.c_1) *
                    (2 * img_mat_sigma_12 + self.c_2))

        # Denominator of SSIM
        den_ssim = (
            (self.img.img_gray_mu_squared + target.img_gray_mu_squared +
             self.c_1) *
            (self.img.img_gray_sigma_squared +
             target.img_gray_sigma_squared + self.c_2))

        ssim_map = num_ssim / den_ssim
        index = np.average(ssim_map)
        return index

Compute the SSIM value from the reference image to the target image .


def compute_ssim(image1, image2, gaussian_kernel_sigma=1.5,
                 gaussian_kernel_width=11):
    """Computes SSIM.

    Args:
      im1: First PIL Image object to compare.
      im2: Second PIL Image object to compare.

    Returns:
      SSIM float value.
    """
    gaussian_kernel_1d = get_gaussian_kernel(
        gaussian_kernel_width, gaussian_kernel_sigma)
    return SSIM(image1, gaussian_kernel_1d).ssim_value(image2)


def destroy(self):
        """
        Correctly destroy SyncObj. Stop autoTickThread, close connections, etc.
        """
        if self.__conf.autoTick:
            self.__destroying = True
        else:
            self._doDestroy()

def destroy ( self ) : if self . __conf . autoTick : self . __destroying = True else : self . _doDestroy ( )

Correctly destroy SyncObj . Stop autoTickThread close connections etc .


def setCodeVersion(self, newVersion, callback = None):
        """Switch to a new code version on all cluster nodes. You
        should ensure that cluster nodes are updated, otherwise they
        won't be able to apply commands.

        :param newVersion: new code version
        :type int
        :param callback: will be called on cussess or fail
        :type callback: function(`FAIL_REASON <#pysyncobj.FAIL_REASON>`_, None)
        """
        assert isinstance(newVersion, int)
        if newVersion > self.__selfCodeVersion:
            raise Exception('wrong version, current version is %d, requested version is %d' % (self.__selfCodeVersion, newVersion))
        if newVersion < self.__enabledCodeVersion:
            raise Exception('wrong version, enabled version is %d, requested version is %d' % (self.__enabledCodeVersion, newVersion))
        self._applyCommand(pickle.dumps(newVersion), callback, _COMMAND_TYPE.VERSION)

Switch to a new code version on all cluster nodes . You should ensure that cluster nodes are updated otherwise they won t be able to apply commands .


def getStatus(self):
        """Dumps different debug info about cluster to dict and return it"""

        status = {}
        status['version'] = VERSION
        status['revision'] = REVISION
        status['self'] = self.__selfNode
        status['state'] = self.__raftState
        status['leader'] = self.__raftLeader
        status['partner_nodes_count'] = len(self.__otherNodes)
        for node in self.__otherNodes:
            status['partner_node_status_server_' + node.id] = 2 if node in self.__connectedNodes else 0
        status['readonly_nodes_count'] = len(self.__readonlyNodes)
        for node in self.__readonlyNodes:
            status['readonly_node_status_server_' + node.id] = 2 if node in self.__connectedNodes else 0
        status['log_len'] = len(self.__raftLog)
        status['last_applied'] = self.__raftLastApplied
        status['commit_idx'] = self.__raftCommitIndex
        status['raft_term'] = self.__raftCurrentTerm
        status['next_node_idx_count'] = len(self.__raftNextIndex)
        for node, idx in iteritems(self.__raftNextIndex):
            status['next_node_idx_server_' + node.id] = idx
        status['match_idx_count'] = len(self.__raftMatchIndex)
        for node, idx in iteritems(self.__raftMatchIndex):
            status['match_idx_server_' + node.id] = idx
        status['leader_commit_idx'] = self.__leaderCommitIndex
        status['uptime'] = int(time.time() - self.__startTime)
        status['self_code_version'] = self.__selfCodeVersion
        status['enabled_code_version'] = self.__enabledCodeVersion
        return status

def getStatus ( self ) : status = { } status [ 'version' ] = VERSION status [ 'revision' ] = REVISION status [ 'self' ] = self . __selfNode status [ 'state' ] = self . __raftState status [ 'leader' ] = self . __raftLeader status [ 'partner_nodes_count' ] = len ( self . __otherNodes ) for node in self . __otherNodes : status [ 'partner_node_status_server_' + node . id ] = 2 if node in self . __connectedNodes else 0 status [ 'readonly_nodes_count' ] = len ( self . __readonlyNodes ) for node in self . __readonlyNodes : status [ 'readonly_node_status_server_' + node . id ] = 2 if node in self . __connectedNodes else 0 status [ 'log_len' ] = len ( self . __raftLog ) status [ 'last_applied' ] = self . __raftLastApplied status [ 'commit_idx' ] = self . __raftCommitIndex status [ 'raft_term' ] = self . __raftCurrentTerm status [ 'next_node_idx_count' ] = len ( self . __raftNextIndex ) for node , idx in iteritems ( self . __raftNextIndex ) : status [ 'next_node_idx_server_' + node . id ] = idx status [ 'match_idx_count' ] = len ( self . __raftMatchIndex ) for node , idx in iteritems ( self . __raftMatchIndex ) : status [ 'match_idx_server_' + node . id ] = idx status [ 'leader_commit_idx' ] = self . __leaderCommitIndex status [ 'uptime' ] = int ( time . time ( ) - self . __startTime ) status [ 'self_code_version' ] = self . __selfCodeVersion status [ 'enabled_code_version' ] = self . __enabledCodeVersion return status


def printStatus(self):
        """Dumps different debug info about cluster to default logger"""
        status = self.getStatus()
        for k, v in iteritems(status):
            logging.info('%s: %s' % (str(k), str(v)))

def
printStatus
(
self
)
:
status
=
self
.
getStatus
(
)
for
k
,
v
in
iteritems
(
status
)
:
logging
.
info
(
'%s: %s'
%
(
str
(
k
)
,
str
(
v
)
)
)


def _connToNode(self, conn):
        """
        Find the node to which a connection belongs.

        :param conn: connection object
        :type conn: TcpConnection
        :returns corresponding node or None if the node cannot be found
        :rtype Node or None
        """

        for node in self._connections:
            if self._connections[node] is conn:
                return node
        return None

def _connToNode ( self , conn ) : for node in self . _connections : if self . _connections [ node ] is conn : return node return None

Find the node to which a connection belongs .


def _maybeBind(self):
        """
        Bind the server unless it is already bound, this is a read-only node, or the last attempt was too recently.

        :raises TransportNotReadyError if the bind attempt fails
        """

        if self._ready or self._selfIsReadonlyNode or time.time() < self._lastBindAttemptTime + self._syncObj.conf.bindRetryTime:
            return
        self._lastBindAttemptTime = time.time()
        try:
            self._server.bind()
        except Exception as e:
            self._bindAttempts += 1
            if self._syncObj.conf.maxBindRetries and self._bindAttempts >= self._syncObj.conf.maxBindRetries:
                self._bindOverEvent.set()
                raise TransportNotReadyError
        else:
            self._ready = True
            self._bindOverEvent.set()

def _maybeBind ( self ) : if self . _ready or self . _selfIsReadonlyNode or time . time ( ) < self . _lastBindAttemptTime + self . _syncObj . conf . bindRetryTime : return self . _lastBindAttemptTime = time . time ( ) try : self . _server . bind ( ) except Exception as e : self . _bindAttempts += 1 if self . _syncObj . conf . maxBindRetries and self . _bindAttempts >= self . _syncObj . conf . maxBindRetries : self . _bindOverEvent . set ( ) raise TransportNotReadyError else : self . _ready = True self . _bindOverEvent . set ( )

Bind the server unless it is already bound this is a read - only node or the last attempt was too recently .


def _onNewIncomingConnection(self, conn):
        """
        Callback for connections initiated by the other side

        :param conn: connection object
        :type conn: TcpConnection
        """

        self._unknownConnections.add(conn)
        encryptor = self._syncObj.encryptor
        if encryptor:
            conn.encryptor = encryptor
        conn.setOnMessageReceivedCallback(functools.partial(self._onIncomingMessageReceived, conn))
        conn.setOnDisconnectedCallback(functools.partial(self._onDisconnected, conn))

def _onNewIncomingConnection ( self , conn ) : self . _unknownConnections . add ( conn ) encryptor = self . _syncObj . encryptor if encryptor : conn . encryptor = encryptor conn . setOnMessageReceivedCallback ( functools . partial ( self . _onIncomingMessageReceived , conn ) ) conn . setOnDisconnectedCallback ( functools . partial ( self . _onDisconnected , conn ) )


def _onIncomingMessageReceived(self, conn, message):
        """
        Callback for initial messages on incoming connections. Handles encryption, utility messages, and association of the connection with a Node.
        Once this initial setup is done, the relevant connected callback is executed, and further messages are deferred to the onMessageReceived callback.

        :param conn: connection object
        :type conn: TcpConnection
        :param message: received message
        :type message: any
        """

        if self._syncObj.encryptor and not conn.sendRandKey:
            conn.sendRandKey = message
            conn.recvRandKey = os.urandom(32)
            conn.send(conn.recvRandKey)
            return

        # Utility messages
        if isinstance(message, list):
            done = False
            try:
                if message[0] == 'status':
                    conn.send(self._syncObj.getStatus())
                    done = True
                elif message[0] == 'add':
                    self._syncObj.addNodeToCluster(message[1], callback = functools.partial(self._utilityCallback, conn = conn, cmd = 'ADD', arg = message[1]))
                    done = True
                elif message[0] == 'remove':
                    if message[1] == self._selfNode.address:
                        conn.send('FAIL REMOVE ' + message[1])
                    else:
                        self._syncObj.removeNodeFromCluster(message[1], callback = functools.partial(self._utilityCallback, conn = conn, cmd = 'REMOVE', arg = message[1]))
                    done = True
                elif message[0] == 'set_version':
                    self._syncObj.setCodeVersion(message[1], callback = functools.partial(self._utilityCallback, conn = conn, cmd = 'SET_VERSION', arg = str(message[1])))
                    done = True
            except Exception as e:
                conn.send(str(e))
                done = True
            if done:
                return

        # At this point, message should be either a node ID (i.e. address) or 'readonly'
        node = self._nodeAddrToNode[message] if message in self._nodeAddrToNode else None

        if node is None and message != 'readonly':
            conn.disconnect()
            self._unknownConnections.discard(conn)
            return

        readonly = node is None
        if readonly:
            nodeId = str(self._readonlyNodesCounter)
            node = Node(nodeId)
            self._readonlyNodes.add(node)
            self._readonlyNodesCounter += 1

        self._unknownConnections.discard(conn)
        self._connections[node] = conn
        conn.setOnMessageReceivedCallback(functools.partial(self._onMessageReceived, node))
        if not readonly:
            self._onNodeConnected(node)
        else:
            self._onReadonlyNodeConnected(node)

Callback for initial messages on incoming connections . Handles encryption utility messages and association of the connection with a Node . Once this initial setup is done the relevant connected callback is executed and further messages are deferred to the onMessageReceived callback .


def _utilityCallback(self, res, err, conn, cmd, arg):
        """
        Callback for the utility messages

        :param res: result of the command
        :param err: error code (one of pysyncobj.config.FAIL_REASON)
        :param conn: utility connection
        :param cmd: command
        :param arg: command arguments
        """

        cmdResult = 'FAIL'
        if err == FAIL_REASON.SUCCESS:
            cmdResult = 'SUCCESS'
        conn.send(cmdResult + ' ' + cmd + ' ' + arg)

def
_utilityCallback
(
self
,
res
,
err
,
conn
,
cmd
,
arg
)
:
cmdResult
=
'FAIL'
if
err
==
FAIL_REASON
.
SUCCESS
:
cmdResult
=
'SUCCESS'
conn
.
send
(
cmdResult
+
' '
+
cmd
+
' '
+
arg
)


def _shouldConnect(self, node):
        """
        Check whether this node should initiate a connection to another node

        :param node: the other node
        :type node: Node
        """

        return isinstance(node, TCPNode) and node not in self._preventConnectNodes and (self._selfIsReadonlyNode or self._selfNode.address > node.address)

def _shouldConnect ( self , node ) : return isinstance ( node , TCPNode ) and node not in self . _preventConnectNodes and ( self . _selfIsReadonlyNode or self . _selfNode . address > node . address )


def _connectIfNecessarySingle(self, node):
        """
        Connect to a node if necessary.

        :param node: node to connect to
        :type node: Node
        """

        if node in self._connections and self._connections[node].state != CONNECTION_STATE.DISCONNECTED:
            return True
        if not self._shouldConnect(node):
            return False
        assert node in self._connections # Since we "should connect" to this node, there should always be a connection object already in place.
        if node in self._lastConnectAttempt and time.time() - self._lastConnectAttempt[node] < self._syncObj.conf.connectionRetryTime:
            return False
        self._lastConnectAttempt[node] = time.time()
        return self._connections[node].connect(node.ip, node.port)


def _onOutgoingConnected(self, conn):
        """
        Callback for when a new connection from this to another node is established. Handles encryption and informs the other node which node this is.
        If encryption is disabled, this triggers the onNodeConnected callback and messages are deferred to the onMessageReceived callback.
        If encryption is enabled, the first message is handled by _onOutgoingMessageReceived.

        :param conn: connection object
        :type conn: TcpConnection
        """

        if self._syncObj.encryptor:
            conn.setOnMessageReceivedCallback(functools.partial(self._onOutgoingMessageReceived, conn)) # So we can process the sendRandKey
            conn.recvRandKey = os.urandom(32)
            conn.send(conn.recvRandKey)
        else:
            # The onMessageReceived callback is configured in addNode already.
            if not self._selfIsReadonlyNode:
                conn.send(self._selfNode.address)
            else:
                conn.send('readonly')
            self._onNodeConnected(self._connToNode(conn))

Callback for when a new connection from this to another node is established . Handles encryption and informs the other node which node this is . If encryption is disabled this triggers the onNodeConnected callback and messages are deferred to the onMessageReceived callback . If encryption is enabled the first message is handled by _onOutgoingMessageReceived .


def _onOutgoingMessageReceived(self, conn, message):
        """
        Callback for receiving a message on a new outgoing connection. Used only if encryption is enabled to exchange the random keys.
        Once the key exchange is done, this triggers the onNodeConnected callback, and further messages are deferred to the onMessageReceived callback.

        :param conn: connection object
        :type conn: TcpConnection
        :param message: received message
        :type message: any
        """

        if not conn.sendRandKey:
            conn.sendRandKey = message
            conn.send(self._selfNode.address)

        node = self._connToNode(conn)
        conn.setOnMessageReceivedCallback(functools.partial(self._onMessageReceived, node))
        self._onNodeConnected(node)

def _onOutgoingMessageReceived ( self , conn , message ) : if not conn . sendRandKey : conn . sendRandKey = message conn . send ( self . _selfNode . address ) node = self . _connToNode ( conn ) conn . setOnMessageReceivedCallback ( functools . partial ( self . _onMessageReceived , node ) ) self . _onNodeConnected ( node )

Callback for receiving a message on a new outgoing connection . Used only if encryption is enabled to exchange the random keys . Once the key exchange is done this triggers the onNodeConnected callback and further messages are deferred to the onMessageReceived callback .


def _onDisconnected(self, conn):
        """
        Callback for when a connection is terminated or considered dead. Initiates a reconnect if necessary.

        :param conn: connection object
        :type conn: TcpConnection
        """

        self._unknownConnections.discard(conn)
        node = self._connToNode(conn)
        if node is not None:
            if node in self._nodes:
                self._onNodeDisconnected(node)
                self._connectIfNecessarySingle(node)
            else:
                self._readonlyNodes.discard(node)
                self._onReadonlyNodeDisconnected(node)

def _onDisconnected ( self , conn ) : self . _unknownConnections . discard ( conn ) node = self . _connToNode ( conn ) if node is not None : if node in self . _nodes : self . _onNodeDisconnected ( node ) self . _connectIfNecessarySingle ( node ) else : self . _readonlyNodes . discard ( node ) self . _onReadonlyNodeDisconnected ( node )

Callback for when a connection is terminated or considered dead . Initiates a reconnect if necessary .