<reponame>carlosbpy/pyspark-3.1.1-pgsql
import os
from pyspark.sql import SparkSession

if __name__ == '__main__':
    
    spark = SparkSession \
    .builder \
    .appName("Job - Consumer-zone") \
    .getOrCreate()

    consumer_df = spark.read \
    .format("parquet") \
    .load("processing-zone/*.parquet")

    consumer_df.createOrReplaceTempView("tb_user")

    spark.sql("SELECT count(*) FROM tb_user").show()

    def processing_count_gender_per_user():
        return spark.sql("""
                    SELECT 
                        count(id) as qtd, 
                        gender 
                    FROM
                        tb_user
                    GROUP BY 2
                    """)

    metrics_gender = processing_count_gender_per_user()

    metrics_gender.write.mode("overwrite") \
    .partitionBy("gender") \
    .format("parquet") \
    .save("consumer-zone/")

import os
from pyspark.sql import SparkSession

if __name__ == '__main__':
    
    spark = SparkSession \
    .builder \
    .appName("Job - Consumer-zone") \
    .getOrCreate()

    consumer_df = spark.read \
    .format("parquet") \
    .load("processing-zone/*.parquet")

    consumer_df.createOrReplaceTempView("tb_user")

    spark.sql("SELECT count(*) FROM tb_user").show()

    def processing_count_gender_per_user():
        return spark.sql("""
                    SELECT 
                        count(id) as qtd, 
                        gender 
                    FROM
                        tb_user
                    GROUP BY 2
                    """)

    metrics_gender = processing_count_gender_per_user()

    metrics_gender.write.mode("overwrite") \
    .partitionBy("gender") \
    .format("parquet") \
    .save("consumer-zone/")
SELECT                          count(id) as qtd,                          gender                      FROM                         tb_user                     GROUP BY 2