def df_to_simple_rdd(df, categorical=False, nb_classes=None, features_col='features', label_col='label'):
    """Convert DataFrame into RDD of pairs
    """
    sql_context = df.sql_ctx
    sql_context.registerDataFrameAsTable(df, "temp_table")
    selected_df = sql_context.sql(
        "SELECT {0} AS features, {1} as label from temp_table".format(features_col, label_col))
    if isinstance(selected_df.first().features, MLLibVector):
        lp_rdd = selected_df.rdd.map(
            lambda row: LabeledPoint(row.label, row.features))
    else:
        lp_rdd = selected_df.rdd.map(lambda row: LabeledPoint(
            row.label, MLLibVectors.fromML(row.features)))
    rdd = lp_to_simple_rdd(lp_rdd, categorical, nb_classes)
    return rdd
def df_to_simple_rdd ( df , categorical = False , nb_classes = None , features_col = 'features' , label_col = 'label' ) : sql_context = df . sql_ctx sql_context . registerDataFrameAsTable ( df , "temp_table" ) selected_df = sql_context . sql ( "SELECT {0} AS features, {1} as label from temp_table" . format ( features_col , label_col ) ) if isinstance ( selected_df . first ( ) . features , MLLibVector ) : lp_rdd = selected_df . rdd . map ( lambda row : LabeledPoint ( row . label , row . features ) ) else : lp_rdd = selected_df . rdd . map ( lambda row : LabeledPoint ( row . label , MLLibVectors . fromML ( row . features ) ) ) rdd = lp_to_simple_rdd ( lp_rdd , categorical , nb_classes ) return rdd
Convert DataFrame into RDD of pairs