import pandas as pd
import json
from jsonschema import validate
from os import path
import sys
import PySimpleGUI as sg


raw_data = None
class_list = None
DayInfo = None
delta_max = None
UUID_to_email = None


def set_data_vars(data_path, json_path):
    """
    Sets global data variables to be used in ``ElementCollection.py`` and ``Main.py``, including

    * raw_data: the dataframe read from the raw data's csv
    * class_list: list of unique emails from raw_data (we assume this is acceptable as a list of everyone in the class)
    * DayInfo: the JSON data imported into the program, if the user uses a JSON input

    :param data_path: path to the raw_data csv
    :param json_path: path to the JSON file, or -1 if the user isn't using a JSON file to provide H5P IDs
    """
    global raw_data
    global class_list
    global DayInfo
    global delta_max
    global UUID_to_email

    raw_data = pd.read_csv(data_path)

    # Uncomment below to print the number of bytes the dataframe takes in memory
    # print("raw data: " + str(raw_data.memory_usage(index=True, deep=True).sum()))
    UUID_to_email = find_emails(raw_data)

    # Only keep around columns we care about
    raw_data = raw_data[["Name", "Verb", "object id", "Question/Slide", "Timestamp", "Duration", "Response"]]

    # Convert the Timestamp column to datetime objects
    raw_data["Timestamp"] = pd.to_datetime(raw_data["Timestamp"], errors='coerce')
    # Drop all rows where the datetime conversion failed or where email doesn't exist, b/c that means they're bad data
    rows_count = len(raw_data.index)
    raw_data = raw_data.dropna(subset=["Timestamp"])
    rows_dropped_timestamp = rows_count - len(raw_data.index)

    # Drop all columns with an NaN email, b/c that's bad data
    rows_count = len(raw_data.index)
    raw_data = raw_data.dropna(subset=["Name"])
    rows_dropped_name_nan = rows_count - len(raw_data.index)

    # Drop all columns that don't have a valid email URL, b/c that means they're bad data
    # Commented out Spring 2021 when reverting to using UUIDs
    # rows_count = len(raw_data.index)
    # raw_data = raw_data[raw_data["Email"].str.slice(start=0, stop=7).str.fullmatch("mailto:", case=False)]
    # rows_dropped_bad_email = rows_count - len(raw_data.index)

    # Reformat email column to remove the "mailto:"
    # raw_data["Email"] = raw_data["Email"].str.slice(start=7)

    # Give notice popup about dropped rows
    if rows_dropped_timestamp != 0 or rows_dropped_name_nan != 0:
        sg.Popup("Some data was dropped because of improper formatting:\nBad timestamp: " + str(rows_dropped_timestamp)
                 + " entries\nNo UUID associated: " + str(rows_dropped_name_nan) + " entries",
                 title="Info: Data Dropped")

    # Drop all "consumed" verbs b/c they seem to be pretty useless
    raw_data = raw_data[raw_data["Verb"] != 'consumed']

    # Parse the actual object ID from the "object id" column
    url_list = raw_data["object id"].to_list()
    id_list = [int(s[s.index("id=") + 3: len(s) if s.find("?", s.index("id=")) == -1 else s.find("?", s.index("id="))])
               for s in url_list]
    raw_data["object id"] = id_list

    # Sort the timestamp column to ensure times are not out of order
    raw_data = raw_data.sort_values(by=['Timestamp'], ascending=False)

    class_list = set(raw_data["Name"])

    # import data in DayElement.json (Error handling done in Main.py)
    with open(json_path) as f:
        DayInfo = json.load(f)
        # Find path to DayElementSchema
        bundle_dir = getattr(sys, '_MEIPASS', path.abspath(path.dirname(__file__)))
        validation_path = path.abspath(path.join(bundle_dir, 'DayElementSchema.json'))
        with open(validation_path) as v:
            schema = json.load(v)
            # Perform Validation
            validate(instance=DayInfo, schema=schema)

        # Drop data from non-student emails (using the filter emails list)
        # Commented out Spring 2021 upon switch to UUIDs
        # raw_data = raw_data[~raw_data["Email"].isin(DayInfo["Filter_Emails"])]
        # class_list = class_list - set(DayInfo["Filter_Emails"])

        delta_max = DayInfo["Time_Delta"]


def find_emails(df):
    # Define an email regex and find responses that are emails
    email_regex = '([a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+)'
    email_rows = df.Response.str.extract(email_regex, expand=False)

    # Transform this into a new df, then into a dictionary mapping name->email
    email_df = pd.DataFrame({'Name': df.Name, 'Response': email_rows}).dropna().set_index("Name")
    dictionary = email_df.T.to_dict('list')  # T = transposition, some wacky stuff from stack overflow
    # Remove lists
    for key in dictionary:
        dictionary[key] = dictionary[key][0]

    return dictionary

import pandas as pd
import json
from jsonschema import validate
from os import path
import sys
import PySimpleGUI as sg


raw_data = None
class_list = None
DayInfo = None
delta_max = None
UUID_to_email = None


def set_data_vars(data_path, json_path):
    """
    Sets global data variables to be used in ``ElementCollection.py`` and ``Main.py``, including

    * raw_data: the dataframe read from the raw data's csv
    * class_list: list of unique emails from raw_data (we assume this is acceptable as a list of everyone in the class)
    * DayInfo: the JSON data imported into the program, if the user uses a JSON input

    :param data_path: path to the raw_data csv
    :param json_path: path to the JSON file, or -1 if the user isn't using a JSON file to provide H5P IDs
    """
    global raw_data
    global class_list
    global DayInfo
    global delta_max
    global UUID_to_email

    raw_data = pd.read_csv(data_path)

    # Uncomment below to print the number of bytes the dataframe takes in memory
    # print("raw data: " + str(raw_data.memory_usage(index=True, deep=True).sum()))
    UUID_to_email = find_emails(raw_data)

    # Only keep around columns we care about
    raw_data = raw_data[["Name", "Verb", "object id", "Question/Slide", "Timestamp", "Duration", "Response"]]

    # Convert the Timestamp column to datetime objects
    raw_data["Timestamp"] = pd.to_datetime(raw_data["Timestamp"], errors='coerce')
    # Drop all rows where the datetime conversion failed or where email doesn't exist, b/c that means they're bad data
    rows_count = len(raw_data.index)
    raw_data = raw_data.dropna(subset=["Timestamp"])
    rows_dropped_timestamp = rows_count - len(raw_data.index)

    # Drop all columns with an NaN email, b/c that's bad data
    rows_count = len(raw_data.index)
    raw_data = raw_data.dropna(subset=["Name"])
    rows_dropped_name_nan = rows_count - len(raw_data.index)

    # Drop all columns that don't have a valid email URL, b/c that means they're bad data
    # Commented out Spring 2021 when reverting to using UUIDs
    # rows_count = len(raw_data.index)
    # raw_data = raw_data[raw_data["Email"].str.slice(start=0, stop=7).str.fullmatch("mailto:", case=False)]
    # rows_dropped_bad_email = rows_count - len(raw_data.index)

    # Reformat email column to remove the "mailto:"
    # raw_data["Email"] = raw_data["Email"].str.slice(start=7)

    # Give notice popup about dropped rows
    if rows_dropped_timestamp != 0 or rows_dropped_name_nan != 0:
        sg.Popup("Some data was dropped because of improper formatting:\nBad timestamp: " + str(rows_dropped_timestamp)
                 + " entries\nNo UUID associated: " + str(rows_dropped_name_nan) + " entries",
                 title="Info: Data Dropped")

    # Drop all "consumed" verbs b/c they seem to be pretty useless
    raw_data = raw_data[raw_data["Verb"] != 'consumed']

    # Parse the actual object ID from the "object id" column
    url_list = raw_data["object id"].to_list()
    id_list = [int(s[s.index("id=") + 3: len(s) if s.find("?", s.index("id=")) == -1 else s.find("?", s.index("id="))])
               for s in url_list]
    raw_data["object id"] = id_list

    # Sort the timestamp column to ensure times are not out of order
    raw_data = raw_data.sort_values(by=['Timestamp'], ascending=False)

    class_list = set(raw_data["Name"])

    # import data in DayElement.json (Error handling done in Main.py)
    with open(json_path) as f:
        DayInfo = json.load(f)
        # Find path to DayElementSchema
        bundle_dir = getattr(sys, '_MEIPASS', path.abspath(path.dirname(__file__)))
        validation_path = path.abspath(path.join(bundle_dir, 'DayElementSchema.json'))
        with open(validation_path) as v:
            schema = json.load(v)
            # Perform Validation
            validate(instance=DayInfo, schema=schema)

        # Drop data from non-student emails (using the filter emails list)
        # Commented out Spring 2021 upon switch to UUIDs
        # raw_data = raw_data[~raw_data["Email"].isin(DayInfo["Filter_Emails"])]
        # class_list = class_list - set(DayInfo["Filter_Emails"])

        delta_max = DayInfo["Time_Delta"]


def find_emails(df):
    # Define an email regex and find responses that are emails
    email_regex = '([a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+)'
    email_rows = df.Response.str.extract(email_regex, expand=False)

    # Transform this into a new df, then into a dictionary mapping name->email
    email_df = pd.DataFrame({'Name': df.Name, 'Response': email_rows}).dropna().set_index("Name")
    dictionary = email_df.T.to_dict('list')  # T = transposition, some wacky stuff from stack overflow
    # Remove lists
    for key in dictionary:
        dictionary[key] = dictionary[key][0]

    return dictionary

Sets global data variables to be used in ``ElementCollection.py`` and ``Main.py``, including      * raw_data: the dataframe read from the raw data's csv     * class_list: list of unique emails from raw_data (we assume this is acceptable as a list of everyone in the class)     * DayInfo: the JSON data imported into the program, if the user uses a JSON input      :param data_path: path to the raw_data csv     :param json_path: path to the JSON file, or -1 if the user isn't using a JSON file to provide H5P IDs # Uncomment below to print the number of bytes the dataframe takes in memory # print("raw data: " + str(raw_data.memory_usage(index=True, deep=True).sum())) # Only keep around columns we care about # Convert the Timestamp column to datetime objects # Drop all rows where the datetime conversion failed or where email doesn't exist, b/c that means they're bad data # Drop all columns with an NaN email, b/c that's bad data # Drop all columns that don't have a valid email URL, b/c that means they're bad data # Commented out Spring 2021 when reverting to using UUIDs # rows_count = len(raw_data.index) # raw_data = raw_data[raw_data["Email"].str.slice(start=0, stop=7).str.fullmatch("mailto:", case=False)] # rows_dropped_bad_email = rows_count - len(raw_data.index) # Reformat email column to remove the "mailto:" # raw_data["Email"] = raw_data["Email"].str.slice(start=7) # Give notice popup about dropped rows # Drop all "consumed" verbs b/c they seem to be pretty useless # Parse the actual object ID from the "object id" column # Sort the timestamp column to ensure times are not out of order # import data in DayElement.json (Error handling done in Main.py) # Find path to DayElementSchema # Perform Validation # Drop data from non-student emails (using the filter emails list) # Commented out Spring 2021 upon switch to UUIDs # raw_data = raw_data[~raw_data["Email"].isin(DayInfo["Filter_Emails"])] # class_list = class_list - set(DayInfo["Filter_Emails"]) # Define an email regex and find responses that are emails # Transform this into a new df, then into a dictionary mapping name->email # T = transposition, some wacky stuff from stack overflow # Remove lists