<filename>defunct/naiveBayes/naiveBayes.py
from pyspark.mllib.classification import NaiveBayes, NaiveBayesModel
from pyspark.mllib.util import MLUtils
from pyspark.mllib.linalg import Vectors
from pyspark.mllib.regression import LabeledPoint
from pyspark.sql import SparkSession
from tempfile import NamedTemporaryFile
from fileinput import input
import os
import shutil as sh


if __name__ == '__main__':

#-------------------------------------------------------------------------------
# Initialize Spark
#-------------------------------------------------------------------------------
    #initialize spark session
    spark = SparkSession\
            .builder\
            .appName("Test")\
            .getOrCreate()
    #.config('spark.sql.warehouse.dir', 'file:///C:/')\        
    sc = spark.sparkContext
    #sc.setLogLevel("ERROR")

#-------------------------------------------------------------------------------
# Read the training data and build the model
#-------------------------------------------------------------------------------
    #reading the train dataframes
    trainingDF = spark.read.load("../data/train_small.parquet")     
    
    #convert every row to LabeledPoint
    transformedTrainingRDD = (trainingDF.rdd
                             .map(lambda row: LabeledPoint(row.label,row.features)))
    
    # Save the RDD in LibSVM format, as Naive Bayes reads in the same format
    # Remove libsvmfile if exists
    #if os.path.isdir("./trainingLibsvmfile"):
    #    sh.rmtree("./trainingLibsvmfile")
    
    MLUtils.saveAsLibSVMFile(transformedTrainingRDD,"trainingLibsvmfile")
    trainingData = MLUtils.loadLibSVMFile(sc, "trainingLibsvmfile/*")
    print "trainingLibsvmfile created!!"
    
    # Train a naive Bayes model.
    model = NaiveBayes.train(trainingData, 1.0)
    print "Model built!!"


#-------------------------------------------------------------------------------
# Read the testing data and predict
#-------------------------------------------------------------------------------
    #reading the test dataframes
    testingDF = spark.read.load("../data/test_small.parquet")     
    
    #convert every row to LabeledPoint
    transformedTestRDD = (testingDF.rdd
                             .map(lambda row: LabeledPoint(row.label,row.features)))
    
    # Save the RDD in LibSVM format, as Naive Bayes reads in the same format
    # Remove libsvmfile if exists
    #if os.path.isdir("./testingLibsvmfile"):
    #    sh.rmtree("./testingLibsvmfile")
    
    MLUtils.saveAsLibSVMFile(transformedTestRDD,"testingLibsvmfile")
    testingData = MLUtils.loadLibSVMFile(sc, "testingLibsvmfile/*")
    print "testingLibsvmfile created!!"
    
   
    print "Predicting...."
    # Make prediction and test accuracy.
    predictionAndLabel = testingData.map(lambda p: (model.predict(p.features), p.label))
    testIncorrect = predictionAndLabel.filter(lambda (v, p): v != p).count()
    testCorrect = predictionAndLabel.filter(lambda (v, p): v == p).count()
    #accuracy = 1.0 * predictionAndLabel.filter(lambda (x, v): x == v).count() / testingData.count()
    print('Total Count: '+str(testingData.count()))
    print('Correct Count: '+str(testCorrect))
    print('Incorrect Count: '+str(testIncorrect))
    print('Learned classification Naive Bayes model:')
    print(model.toDebugString())

   
<filename>defunct/naiveBayes/naiveBayes.py
from pyspark.mllib.classification import NaiveBayes, NaiveBayesModel
from pyspark.mllib.util import MLUtils
from pyspark.mllib.linalg import Vectors
from pyspark.mllib.regression import LabeledPoint
from pyspark.sql import SparkSession
from tempfile import NamedTemporaryFile
from fileinput import input
import os
import shutil as sh


if __name__ == '__main__':

#-------------------------------------------------------------------------------
# Initialize Spark
#-------------------------------------------------------------------------------
    #initialize spark session
    spark = SparkSession\
            .builder\
            .appName("Test")\
            .getOrCreate()
    #.config('spark.sql.warehouse.dir', 'file:///C:/')\        
    sc = spark.sparkContext
    #sc.setLogLevel("ERROR")

#-------------------------------------------------------------------------------
# Read the training data and build the model
#-------------------------------------------------------------------------------
    #reading the train dataframes
    trainingDF = spark.read.load("../data/train_small.parquet")     
    
    #convert every row to LabeledPoint
    transformedTrainingRDD = (trainingDF.rdd
                             .map(lambda row: LabeledPoint(row.label,row.features)))
    
    # Save the RDD in LibSVM format, as Naive Bayes reads in the same format
    # Remove libsvmfile if exists
    #if os.path.isdir("./trainingLibsvmfile"):
    #    sh.rmtree("./trainingLibsvmfile")
    
    MLUtils.saveAsLibSVMFile(transformedTrainingRDD,"trainingLibsvmfile")
    trainingData = MLUtils.loadLibSVMFile(sc, "trainingLibsvmfile/*")
    print "trainingLibsvmfile created!!"
    
    # Train a naive Bayes model.
    model = NaiveBayes.train(trainingData, 1.0)
    print "Model built!!"


#-------------------------------------------------------------------------------
# Read the testing data and predict
#-------------------------------------------------------------------------------
    #reading the test dataframes
    testingDF = spark.read.load("../data/test_small.parquet")     
    
    #convert every row to LabeledPoint
    transformedTestRDD = (testingDF.rdd
                             .map(lambda row: LabeledPoint(row.label,row.features)))
    
    # Save the RDD in LibSVM format, as Naive Bayes reads in the same format
    # Remove libsvmfile if exists
    #if os.path.isdir("./testingLibsvmfile"):
    #    sh.rmtree("./testingLibsvmfile")
    
    MLUtils.saveAsLibSVMFile(transformedTestRDD,"testingLibsvmfile")
    testingData = MLUtils.loadLibSVMFile(sc, "testingLibsvmfile/*")
    print "testingLibsvmfile created!!"
    
   
    print "Predicting...."
    # Make prediction and test accuracy.
    predictionAndLabel = testingData.map(lambda p: (model.predict(p.features), p.label))
    testIncorrect = predictionAndLabel.filter(lambda (v, p): v != p).count()
    testCorrect = predictionAndLabel.filter(lambda (v, p): v == p).count()
    #accuracy = 1.0 * predictionAndLabel.filter(lambda (x, v): x == v).count() / testingData.count()
    print('Total Count: '+str(testingData.count()))
    print('Correct Count: '+str(testCorrect))
    print('Incorrect Count: '+str(testIncorrect))
    print('Learned classification Naive Bayes model:')
    print(model.toDebugString())

   
#------------------------------------------------------------------------------- # Initialize Spark #------------------------------------------------------------------------------- #initialize spark session #.config('spark.sql.warehouse.dir', 'file:///C:/')\ #sc.setLogLevel("ERROR") #------------------------------------------------------------------------------- # Read the training data and build the model #------------------------------------------------------------------------------- #reading the train dataframes #convert every row to LabeledPoint # Save the RDD in LibSVM format, as Naive Bayes reads in the same format # Remove libsvmfile if exists #if os.path.isdir("./trainingLibsvmfile"): #    sh.rmtree("./trainingLibsvmfile") # Train a naive Bayes model. #------------------------------------------------------------------------------- # Read the testing data and predict #------------------------------------------------------------------------------- #reading the test dataframes #convert every row to LabeledPoint # Save the RDD in LibSVM format, as Naive Bayes reads in the same format # Remove libsvmfile if exists #if os.path.isdir("./testingLibsvmfile"): #    sh.rmtree("./testingLibsvmfile") # Make prediction and test accuracy. #accuracy = 1.0 * predictionAndLabel.filter(lambda (x, v): x == v).count() / testingData.count()