from pyspark.ml.classification import NaiveBayes
from pyspark.ml.evaluation import MulticlassClassificationEvaluator
from pyspark.sql import SparkSession
#from pyspark.mllib.linalg import VectorUDT
from pyspark.ml.linalg import VectorUDT

if __name__ == '__main__':

    #initialize spark session
    spark = SparkSession\
            .builder\
            .appName("Test")\
            .getOrCreate()
    sc = spark.sparkContext

    #reading the train dataframes
    trainingDF = spark.read.load("../data/train_small.parquet")

    #train = trainingDF.withColumn('features',trainingDF.features.cast(VectorUDT()))     
    
    # Split the data into train and test
    splits = trainingDF.randomSplit([0.6, 0.4], 1234)
    train = splits[0]
    test = splits[1]

    # create the trainer and set its parameters
    nb = NaiveBayes(smoothing=1.0, modelType="multinomial")

    # train the model
    model = nb.fit(train)
    # compute precision on the test set
    result = model.transform(test)
    predictionAndLabels = result.select("prediction", "label")
    evaluator = MulticlassClassificationEvaluator(metricName="precision")
    print("Precision:" + str(evaluator.evaluate(predictionAndLabels)))
    
from pyspark.ml.classification import NaiveBayes
from pyspark.ml.evaluation import MulticlassClassificationEvaluator
from pyspark.sql import SparkSession
#from pyspark.mllib.linalg import VectorUDT
from pyspark.ml.linalg import VectorUDT

if __name__ == '__main__':

    #initialize spark session
    spark = SparkSession\
            .builder\
            .appName("Test")\
            .getOrCreate()
    sc = spark.sparkContext

    #reading the train dataframes
    trainingDF = spark.read.load("../data/train_small.parquet")

    #train = trainingDF.withColumn('features',trainingDF.features.cast(VectorUDT()))     
    
    # Split the data into train and test
    splits = trainingDF.randomSplit([0.6, 0.4], 1234)
    train = splits[0]
    test = splits[1]

    # create the trainer and set its parameters
    nb = NaiveBayes(smoothing=1.0, modelType="multinomial")

    # train the model
    model = nb.fit(train)
    # compute precision on the test set
    result = model.transform(test)
    predictionAndLabels = result.select("prediction", "label")
    evaluator = MulticlassClassificationEvaluator(metricName="precision")
    print("Precision:" + str(evaluator.evaluate(predictionAndLabels)))
    
#from pyspark.mllib.linalg import VectorUDT #initialize spark session #reading the train dataframes #train = trainingDF.withColumn('features',trainingDF.features.cast(VectorUDT())) # Split the data into train and test # create the trainer and set its parameters # train the model # compute precision on the test set