from pyspark.sql import SparkSession
from pyspark.sql import functions as func

spark = SparkSession.builder.appName("WordCount").getOrCreate()

# Read each line of my book into a dataframe
inputDF = spark.read.text("file:///home/sambiase/courses/SparkCourse/book.txt")

# Split using a regular expression that extracts words
# Explode is similar to flatMap in RDD. It takes each row and makes new rows of each word
# \\W+  --> this means Split the results into Words
words = inputDF.select(func.explode(func.split(inputDF.value, "\\W+")).alias("word"))
words.filter(words.word != "")

# Normalize everything to lowercase
lowercaseWords = words.select(func.lower(words.word).alias("word"))

# Count up the occurrences of each word
wordCounts = lowercaseWords.groupBy("word").count()

# Sort by counts
wordCountsSorted = wordCounts.sort("count")

# Show the results.
wordCountsSorted.show(wordCountsSorted.count())

from pyspark.sql import SparkSession
from pyspark.sql import functions as func

spark = SparkSession.builder.appName("WordCount").getOrCreate()

# Read each line of my book into a dataframe
inputDF = spark.read.text("file:///home/sambiase/courses/SparkCourse/book.txt")

# Split using a regular expression that extracts words
# Explode is similar to flatMap in RDD. It takes each row and makes new rows of each word
# \\W+  --> this means Split the results into Words
words = inputDF.select(func.explode(func.split(inputDF.value, "\\W+")).alias("word"))
words.filter(words.word != "")

# Normalize everything to lowercase
lowercaseWords = words.select(func.lower(words.word).alias("word"))

# Count up the occurrences of each word
wordCounts = lowercaseWords.groupBy("word").count()

# Sort by counts
wordCountsSorted = wordCounts.sort("count")

# Show the results.
wordCountsSorted.show(wordCountsSorted.count())

# Read each line of my book into a dataframe # Split using a regular expression that extracts words # Explode is similar to flatMap in RDD. It takes each row and makes new rows of each word # \\W+  --> this means Split the results into Words # Normalize everything to lowercase # Count up the occurrences of each word # Sort by counts # Show the results.