def _fit(self, df):
        """Private fit method of the Estimator, which trains the model.
        """
        simple_rdd = df_to_simple_rdd(df, categorical=self.get_categorical_labels(), nb_classes=self.get_nb_classes(),
                                      features_col=self.getFeaturesCol(), label_col=self.getLabelCol())
        simple_rdd = simple_rdd.repartition(self.get_num_workers())
        keras_model = model_from_yaml(self.get_keras_model_config())
        metrics = self.get_metrics()
        loss = self.get_loss()
        optimizer = get_optimizer(self.get_optimizer_config())
        keras_model.compile(loss=loss, optimizer=optimizer, metrics=metrics)

        spark_model = SparkModel(model=keras_model,
                                 mode=self.get_mode(),
                                 frequency=self.get_frequency(),
                                 num_workers=self.get_num_workers())
        spark_model.fit(simple_rdd,
                        epochs=self.get_epochs(),
                        batch_size=self.get_batch_size(),
                        verbose=self.get_verbosity(),
                        validation_split=self.get_validation_split())

        model_weights = spark_model.master_network.get_weights()
        weights = simple_rdd.ctx.broadcast(model_weights)
        return ElephasTransformer(labelCol=self.getLabelCol(),
                                  outputCol='prediction',
                                  keras_model_config=spark_model.master_network.to_yaml(),
                                  weights=weights)
def _fit ( self , df ) : simple_rdd = df_to_simple_rdd ( df , categorical = self . get_categorical_labels ( ) , nb_classes = self . get_nb_classes ( ) , features_col = self . getFeaturesCol ( ) , label_col = self . getLabelCol ( ) ) simple_rdd = simple_rdd . repartition ( self . get_num_workers ( ) ) keras_model = model_from_yaml ( self . get_keras_model_config ( ) ) metrics = self . get_metrics ( ) loss = self . get_loss ( ) optimizer = get_optimizer ( self . get_optimizer_config ( ) ) keras_model . compile ( loss = loss , optimizer = optimizer , metrics = metrics ) spark_model = SparkModel ( model = keras_model , mode = self . get_mode ( ) , frequency = self . get_frequency ( ) , num_workers = self . get_num_workers ( ) ) spark_model . fit ( simple_rdd , epochs = self . get_epochs ( ) , batch_size = self . get_batch_size ( ) , verbose = self . get_verbosity ( ) , validation_split = self . get_validation_split ( ) ) model_weights = spark_model . master_network . get_weights ( ) weights = simple_rdd . ctx . broadcast ( model_weights ) return ElephasTransformer ( labelCol = self . getLabelCol ( ) , outputCol = 'prediction' , keras_model_config = spark_model . master_network . to_yaml ( ) , weights = weights )
Private fit method of the Estimator, which trains the model.