<reponame>yaelgat/hw5<filename>hw5.py
import json
import pathlib
from typing import Union, Tuple
import numpy as np
import re
import pandas as pd
from statistics import mean
import math
import matplotlib as plt


class QuestionnaireAnalysis:
    """
    Reads and analyzes data generated by the questionnaire experiment.
    Should be able to accept strings and pathlib.Path objects.
    """

    def __init__(self, data_fname: Union[pathlib.Path, str]):
        self.data_fname = pathlib.Path(data_fname)
        if self.data_fname.exists() == False:
            raise ValueError()

    def read_data(self):
        """Reads the json data located in self.data_fname into memory, to
        the attribute self.data.
        """
        with open(self.data_fname, 'r') as j:
            # try:
            self.data = json.loads(j.read())
            # except ValueError as e:
                # print('invalid json: %s' % e)
                # return None # or: raise
                # raise Exception('ValueError')
                # raise ValueError()

    def show_age_distrib(self) -> Tuple[np.ndarray, np.ndarray]:
        self.ages = []
        for s in range(len(self.data)):
            self.ages.append(self.data[s]["age"])
        self.ages = np.array(self.ages).astype(np.float)
        nan_array = np.isnan(self.ages)
        not_nan_array = ~ nan_array
        self.age = self.ages[not_nan_array]
        # self.bins=np.ndarray(shape = (1,10), buffer = np.array([[0,10],[10,20],[20,30],[30,40],[40,50],[50,60],[60,70],[70,80],[80,90],[90,100]]))
        self.histogram = np.histogram(self.age, bins= 10, range = (0,100))
        return self.histogram

    def remove_rows_without_mail(self) -> pd.DataFrame:
        # regex = '^(\w|\.|\_|\-)+@(\w|\_|\-|\.)+\.\w{2,3}$'
        regex = '^(\w|\.|\_|\-)+[@](\w|\_|\-|\.)+[.]\w+$'
        # print (self.data)
        # print ('here')
        x = []
        for entry in self.data:
            if re.search(regex, entry["email"]):
                # continue
                x.append(entry)
            # else:
                # print(entry["email"])
                # x.remove(entry)
        self.data_mail = pd.DataFrame(x)
        # print (self.data_mail)
        # print (type(self.dataframe))
        return self.data_mail
    
    def fill_na_with_mean(self) -> Tuple[pd.DataFrame, np.ndarray]:
        grades = {"q1":[], "q2":[], "q3":[], "q4":[], "q5":[],}
        for entry in self.data:
            for i in range(1,6):
                if entry["q" + str(i)] != 'nan':
                    grades["q" + str(i)].append(entry["q" + str(i)])
        means = []
        for i in range(1,6):
            means.append(mean(grades["q" + str(i)]))
        bad_students = []
        for k, entry in enumerate(self.data):
            for i in range(1,6):
                if entry["q" + str(i)] == 'nan':
                    if k not in bad_students:    
                        bad_students.append(k)
                        entry["q" + str(i)] = means[i-1]
        self.data_mean = pd.DataFrame(self.data)
        # print (len(self.data_mean))
        bad_students = np.array(bad_students)
        # print (bad_students)
        self.my_tuple= (self.data_mean, bad_students)
        # print (self.my_tuple)
        return self.my_tuple

    def score_subjects(self, maximal_nans_per_sub: int = 1) -> pd.DataFrame:
        for k, entry in enumerate(self.data):
            counter = 0
            valid_grades = []
            for i in range(1, 6):
                if entry["q" + str(i)] == 'nan':
                    counter += 1
                else:
                    valid_grades.append(entry["q" + str(i)])
            if counter > maximal_nans_per_sub:
                entry['score'] = None
            else:
                entry['score'] = math.floor(mean(valid_grades))
                # entry['score'] = np.array([math.floor(mean(valid_grades))], dtype=np.uint8)
        print (type(entry['score']))
        self.data_score = pd.DataFrame(self.data)
        self.data_score = self.data_score.astype({"score" : "UInt8"})
        return self.data_score

    def correlate_gender_age(self) -> pd.DataFrame:
        self.data_score = pd.DataFrame(self.data)
        ind = list(range(0,len(pd.Series.tolist(self.data_score['age']))))
        arrays = [ind, pd.Series.tolist(self.data_score['age']), pd.Series.tolist(self.data_score['gender'])]
        self.data_score.index = pd.MultiIndex.from_arrays(arrays)
        self.data_grouped = pd.DataFrame(self.data)
        result=[]
        for k, entry in enumerate(self.data):
            if self.data_grouped['age'][k] == 'nan': 
                result.append(float("NaN"))
            else:
                if self.data_grouped['age'][k] > 40:
                    result.append(True)
                else:
                    result.append(False)
        self.data_grouped['age'] = result
        # print (self.data_grouped)
        # self.data_plot = self.data_grouped.groupby(['gender','fourty']).mean('q1')
        temp = pd.DataFrame(self.data_grouped, columns = ['gender', 'age','q1','q2','q3','q4','q5'])
        temp['q1']=pd.to_numeric(temp['q1'], errors='coerce')
        temp['q2']=pd.to_numeric(temp['q2'], errors='coerce')
        temp['q3']=pd.to_numeric(temp['q3'], errors='coerce')
        temp['q4']=pd.to_numeric(temp['q4'], errors='coerce')
        temp['q5']=pd.to_numeric(temp['q5'], errors='coerce')
        self.data_plot = temp.groupby(['gender','age']).mean()
        print (self.data_plot)
        # print(temp.dtypes)
        return self.data_plot


 
# data_fname = r'data.json'
# ques = QuestionnaireAnalysis(data_fname)
#q1
# ques.read_data()
#q2
# ques.show_age_distrib()
#q3
# ques.remove_rows_without_mail()
#q4
# ques.fill_na_with_mean()
#q5
# ques.score_subjects()
#bonus
# ques.correlate_gender_age()
import json
import pathlib
from typing import Union, Tuple
import numpy as np
import re
import pandas as pd
from statistics import mean
import math
import matplotlib as plt


class QuestionnaireAnalysis:
    """
    Reads and analyzes data generated by the questionnaire experiment.
    Should be able to accept strings and pathlib.Path objects.
    """

    def __init__(self, data_fname: Union[pathlib.Path, str]):
        self.data_fname = pathlib.Path(data_fname)
        if self.data_fname.exists() == False:
            raise ValueError()

    def read_data(self):
        """Reads the json data located in self.data_fname into memory, to
        the attribute self.data.
        """
        with open(self.data_fname, 'r') as j:
            # try:
            self.data = json.loads(j.read())
            # except ValueError as e:
                # print('invalid json: %s' % e)
                # return None # or: raise
                # raise Exception('ValueError')
                # raise ValueError()

    def show_age_distrib(self) -> Tuple[np.ndarray, np.ndarray]:
        self.ages = []
        for s in range(len(self.data)):
            self.ages.append(self.data[s]["age"])
        self.ages = np.array(self.ages).astype(np.float)
        nan_array = np.isnan(self.ages)
        not_nan_array = ~ nan_array
        self.age = self.ages[not_nan_array]
        # self.bins=np.ndarray(shape = (1,10), buffer = np.array([[0,10],[10,20],[20,30],[30,40],[40,50],[50,60],[60,70],[70,80],[80,90],[90,100]]))
        self.histogram = np.histogram(self.age, bins= 10, range = (0,100))
        return self.histogram

    def remove_rows_without_mail(self) -> pd.DataFrame:
        # regex = '^(\w|\.|\_|\-)+@(\w|\_|\-|\.)+\.\w{2,3}$'
        regex = '^(\w|\.|\_|\-)+[@](\w|\_|\-|\.)+[.]\w+$'
        # print (self.data)
        # print ('here')
        x = []
        for entry in self.data:
            if re.search(regex, entry["email"]):
                # continue
                x.append(entry)
            # else:
                # print(entry["email"])
                # x.remove(entry)
        self.data_mail = pd.DataFrame(x)
        # print (self.data_mail)
        # print (type(self.dataframe))
        return self.data_mail
    
    def fill_na_with_mean(self) -> Tuple[pd.DataFrame, np.ndarray]:
        grades = {"q1":[], "q2":[], "q3":[], "q4":[], "q5":[],}
        for entry in self.data:
            for i in range(1,6):
                if entry["q" + str(i)] != 'nan':
                    grades["q" + str(i)].append(entry["q" + str(i)])
        means = []
        for i in range(1,6):
            means.append(mean(grades["q" + str(i)]))
        bad_students = []
        for k, entry in enumerate(self.data):
            for i in range(1,6):
                if entry["q" + str(i)] == 'nan':
                    if k not in bad_students:    
                        bad_students.append(k)
                        entry["q" + str(i)] = means[i-1]
        self.data_mean = pd.DataFrame(self.data)
        # print (len(self.data_mean))
        bad_students = np.array(bad_students)
        # print (bad_students)
        self.my_tuple= (self.data_mean, bad_students)
        # print (self.my_tuple)
        return self.my_tuple

    def score_subjects(self, maximal_nans_per_sub: int = 1) -> pd.DataFrame:
        for k, entry in enumerate(self.data):
            counter = 0
            valid_grades = []
            for i in range(1, 6):
                if entry["q" + str(i)] == 'nan':
                    counter += 1
                else:
                    valid_grades.append(entry["q" + str(i)])
            if counter > maximal_nans_per_sub:
                entry['score'] = None
            else:
                entry['score'] = math.floor(mean(valid_grades))
                # entry['score'] = np.array([math.floor(mean(valid_grades))], dtype=np.uint8)
        print (type(entry['score']))
        self.data_score = pd.DataFrame(self.data)
        self.data_score = self.data_score.astype({"score" : "UInt8"})
        return self.data_score

    def correlate_gender_age(self) -> pd.DataFrame:
        self.data_score = pd.DataFrame(self.data)
        ind = list(range(0,len(pd.Series.tolist(self.data_score['age']))))
        arrays = [ind, pd.Series.tolist(self.data_score['age']), pd.Series.tolist(self.data_score['gender'])]
        self.data_score.index = pd.MultiIndex.from_arrays(arrays)
        self.data_grouped = pd.DataFrame(self.data)
        result=[]
        for k, entry in enumerate(self.data):
            if self.data_grouped['age'][k] == 'nan': 
                result.append(float("NaN"))
            else:
                if self.data_grouped['age'][k] > 40:
                    result.append(True)
                else:
                    result.append(False)
        self.data_grouped['age'] = result
        # print (self.data_grouped)
        # self.data_plot = self.data_grouped.groupby(['gender','fourty']).mean('q1')
        temp = pd.DataFrame(self.data_grouped, columns = ['gender', 'age','q1','q2','q3','q4','q5'])
        temp['q1']=pd.to_numeric(temp['q1'], errors='coerce')
        temp['q2']=pd.to_numeric(temp['q2'], errors='coerce')
        temp['q3']=pd.to_numeric(temp['q3'], errors='coerce')
        temp['q4']=pd.to_numeric(temp['q4'], errors='coerce')
        temp['q5']=pd.to_numeric(temp['q5'], errors='coerce')
        self.data_plot = temp.groupby(['gender','age']).mean()
        print (self.data_plot)
        # print(temp.dtypes)
        return self.data_plot


 
# data_fname = r'data.json'
# ques = QuestionnaireAnalysis(data_fname)
#q1
# ques.read_data()
#q2
# ques.show_age_distrib()
#q3
# ques.remove_rows_without_mail()
#q4
# ques.fill_na_with_mean()
#q5
# ques.score_subjects()
#bonus
# ques.correlate_gender_age()
Reads and analyzes data generated by the questionnaire experiment.     Should be able to accept strings and pathlib.Path objects. Reads the json data located in self.data_fname into memory, to         the attribute self.data. # try: # except ValueError as e: # print('invalid json: %s' % e) # return None # or: raise # raise Exception('ValueError') # raise ValueError() # self.bins=np.ndarray(shape = (1,10), buffer = np.array([[0,10],[10,20],[20,30],[30,40],[40,50],[50,60],[60,70],[70,80],[80,90],[90,100]])) # regex = '^(\w|\.|\_|\-)+@(\w|\_|\-|\.)+\.\w{2,3}$' # print (self.data) # print ('here') # continue # else: # print(entry["email"]) # x.remove(entry) # print (self.data_mail) # print (type(self.dataframe)) # print (len(self.data_mean)) # print (bad_students) # print (self.my_tuple) # entry['score'] = np.array([math.floor(mean(valid_grades))], dtype=np.uint8) # print (self.data_grouped) # self.data_plot = self.data_grouped.groupby(['gender','fourty']).mean('q1') # print(temp.dtypes) # data_fname = r'data.json' # ques = QuestionnaireAnalysis(data_fname) #q1 # ques.read_data() #q2 # ques.show_age_distrib() #q3 # ques.remove_rows_without_mail() #q4 # ques.fill_na_with_mean() #q5 # ques.score_subjects() #bonus # ques.correlate_gender_age()