import gzip
import os
import time
import pandas as pd


def parse(path):
    g = gzip.open(path, 'rb')
    for e in g:
        yield eval(e)


def load_json_gz(path):
    i = 0
    df = {}
    for d in parse(path):
        df[i] = d
        i += 1

    return pd.DataFrame.from_dict(df, orient='index')


"""
only choose latest 20000 records for this study
"""


def yelp_rating_data():
    p = '../dataset/yelp/yelp_academic_dataset_review.csv'
    data = pd.read_csv(p)
    data = data[['user_id', 'business_id', 'stars', 'date']]
    data['date'] = data.date.apply(lambda x: int(
        time.mktime(time.strptime(x, "%Y-%m-%d"))))
    data.rename(index=str, columns={
                'business_id': 'item_id', 'stars': 'rating', 'date': 'timestamp'}, inplace=True)
    data = data.sort_values(
        by='timestamp').reset_index(drop=True).iloc[-20000:, :]
    if not os.path.isfile('./exp_data/yelp_exp.csv'):
        data.to_csv('./exp_data/yelp_exp.csv', index=False)

    return data


def amazon_rating_data():
    p = '../dataset/amazon/ratings_Digital_Music.csv'
    data = pd.read_csv(
        p, names=['user_id', 'item_id', 'rating', 'timestamp'], encoding='utf-8')
    data = data.sort_values(
        by='timestamp').reset_index(drop=True).iloc[-20000:, :]
    if not os.path.isfile('./exp_data/amazon_exp.csv'):
        data.to_csv('./exp_data/amazon_exp.csv', index=False)

    return data


a_data = amazon_rating_data()
y_data = yelp_rating_data()

# split via timestamp 0.75--0.25
max_time, min_time = a_data.timestamp.max(), a_data.timestamp.min()
threshold = min_time + int((max_time - min_time) * 0.75)
a_data_1, a_data_2 = a_data[a_data.timestamp <= threshold].reset_index(drop=True), a_data[a_data.timestamp > threshold].reset_index(drop=True)
a_data_1.to_csv('./exp_data/amazon_exp_before.csv', index=False)
a_data_2.to_csv('./exp_data/amazon_exp_after.csv', index=False)

max_time, min_time = y_data.timestamp.max(), y_data.timestamp.min()
threshold = min_time + int((max_time - min_time) * 0.75)
y_data_1, y_data_2 = y_data[y_data.timestamp <= threshold].reset_index(drop=True), y_data[y_data.timestamp > threshold].reset_index(drop=True)

y_data_1.to_csv('./exp_data/yelp_exp_before.csv', index=False)
y_data_2.to_csv('./exp_data/yelp_exp_after.csv', index=False)

import gzip
import os
import time
import pandas as pd


def parse(path):
    g = gzip.open(path, 'rb')
    for e in g:
        yield eval(e)


def load_json_gz(path):
    i = 0
    df = {}
    for d in parse(path):
        df[i] = d
        i += 1

    return pd.DataFrame.from_dict(df, orient='index')


"""
only choose latest 20000 records for this study
"""


def yelp_rating_data():
    p = '../dataset/yelp/yelp_academic_dataset_review.csv'
    data = pd.read_csv(p)
    data = data[['user_id', 'business_id', 'stars', 'date']]
    data['date'] = data.date.apply(lambda x: int(
        time.mktime(time.strptime(x, "%Y-%m-%d"))))
    data.rename(index=str, columns={
                'business_id': 'item_id', 'stars': 'rating', 'date': 'timestamp'}, inplace=True)
    data = data.sort_values(
        by='timestamp').reset_index(drop=True).iloc[-20000:, :]
    if not os.path.isfile('./exp_data/yelp_exp.csv'):
        data.to_csv('./exp_data/yelp_exp.csv', index=False)

    return data


def amazon_rating_data():
    p = '../dataset/amazon/ratings_Digital_Music.csv'
    data = pd.read_csv(
        p, names=['user_id', 'item_id', 'rating', 'timestamp'], encoding='utf-8')
    data = data.sort_values(
        by='timestamp').reset_index(drop=True).iloc[-20000:, :]
    if not os.path.isfile('./exp_data/amazon_exp.csv'):
        data.to_csv('./exp_data/amazon_exp.csv', index=False)

    return data


a_data = amazon_rating_data()
y_data = yelp_rating_data()

# split via timestamp 0.75--0.25
max_time, min_time = a_data.timestamp.max(), a_data.timestamp.min()
threshold = min_time + int((max_time - min_time) * 0.75)
a_data_1, a_data_2 = a_data[a_data.timestamp <= threshold].reset_index(drop=True), a_data[a_data.timestamp > threshold].reset_index(drop=True)
a_data_1.to_csv('./exp_data/amazon_exp_before.csv', index=False)
a_data_2.to_csv('./exp_data/amazon_exp_after.csv', index=False)

max_time, min_time = y_data.timestamp.max(), y_data.timestamp.min()
threshold = min_time + int((max_time - min_time) * 0.75)
y_data_1, y_data_2 = y_data[y_data.timestamp <= threshold].reset_index(drop=True), y_data[y_data.timestamp > threshold].reset_index(drop=True)

y_data_1.to_csv('./exp_data/yelp_exp_before.csv', index=False)
y_data_2.to_csv('./exp_data/yelp_exp_after.csv', index=False)

only choose latest 20000 records for this study # split via timestamp 0.75--0.25