<filename>src/min-temperatures.py
from pyspark import SparkConf, SparkContext

conf = SparkConf().setMaster("local").setAppName("MinTemperatures")
sc = SparkContext.getOrCreate()

def parseLine(line):
    fields = line.split(',')
    stationID = fields[0]
    entryType = fields[2]
    temperature = float(fields[3]) * 0.1 * (9.0 / 5.0) + 32.0
    return (stationID, entryType, temperature)
def minTemperature(t1,t2): #Return the minimum temperature
    return min(t1,t2)

lines = sc.textFile("/home/bjit-542/Big Data/SparkCourse/1800.csv")
parsedLines = lines.map(parseLine)
minTemps = parsedLines.filter(lambda x: "TMIN" in x[1])
stationTemps = minTemps.map(lambda x: (x[0],x[2]))
minTemps = stationTemps.reduceByKey(minTemperature)
results = minTemps.collect();

for result in results:
    print(result[0] + "\t{:.2f}F".format(result[1]))
#minTemps.saveAsTextFile("/home/bjit-542/Big Data/SparkCourse/minTempResult.txt")

<filename>src/min-temperatures.py
from pyspark import SparkConf, SparkContext

conf = SparkConf().setMaster("local").setAppName("MinTemperatures")
sc = SparkContext.getOrCreate()

def parseLine(line):
    fields = line.split(',')
    stationID = fields[0]
    entryType = fields[2]
    temperature = float(fields[3]) * 0.1 * (9.0 / 5.0) + 32.0
    return (stationID, entryType, temperature)
def minTemperature(t1,t2): #Return the minimum temperature
    return min(t1,t2)

lines = sc.textFile("/home/bjit-542/Big Data/SparkCourse/1800.csv")
parsedLines = lines.map(parseLine)
minTemps = parsedLines.filter(lambda x: "TMIN" in x[1])
stationTemps = minTemps.map(lambda x: (x[0],x[2]))
minTemps = stationTemps.reduceByKey(minTemperature)
results = minTemps.collect();

for result in results:
    print(result[0] + "\t{:.2f}F".format(result[1]))
#minTemps.saveAsTextFile("/home/bjit-542/Big Data/SparkCourse/minTempResult.txt")

#Return the minimum temperature #minTemps.saveAsTextFile("/home/bjit-542/Big Data/SparkCourse/minTempResult.txt")