"""
Proof of concept for a recommendation system implemented using Logistic Regression and feature hashing.  You'll need to download one of the MovieLens datasets from http://grouplens.org/datasets/movielens/.

Copyright 2016 <NAME>

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import argparse
from collections import defaultdict
from datetime import date
import random

from sklearn.feature_extraction import FeatureHasher
from sklearn.linear_model import SGDClassifier
from sklearn.metrics import roc_curve
from sklearn.metrics import roc_auc_score
from sklearn.metrics import confusion_matrix

import numpy as np
import scipy.sparse as sp

from common import read_ratings
from common import plot_aucs_nnzs

def create_training_sets(ratings):
    print "Creating user movie-interaction lists"
    user_movies = defaultdict(set)
    max_movie_id = 0
    for r in ratings:
        user_movies[r.user_id].add(r.movie_id)
        max_movie_id = max(max_movie_id, r.movie_id)


    training_set = []
    test_set = []
    for user_id, movie_ids in user_movies.iteritems():
        if random.random() >= 0.5:
            training_set.append((user_id, movie_ids))
        else:
            test_set.append((user_id, movie_ids))

    return max_movie_id, training_set, test_set

def generate_features(max_movie_id, seen_movies):
    all_movie_ids = set(range(0, max_movie_id + 1))
    seen_pairs = []
    unseen_pairs = []

    # positive examples
    for movie_id1 in seen_movies:
        movie_pairs = dict()
        for movie_id2 in seen_movies:
            # product with itself will always be 1 when the label is 1
            # and so will result in model overfitting
            if movie_id1 != movie_id2:
                movie_pairs["%s_%s" % (movie_id1, movie_id2)] = 1.
        seen_pairs.append(movie_pairs)

    # negative_examples
    unseen_movies = all_movie_ids - seen_movies
    for movie_id1 in random.sample(unseen_movies, len(seen_movies)):
        movie_pairs = dict()
        for movie_id2 in seen_movies:
            movie_pairs["%s_%s" % (movie_id1, movie_id2)] = 1.
        unseen_pairs.append(movie_pairs)

    labels = np.hstack([np.ones(len(seen_pairs)), np.zeros(len(unseen_pairs))])

    return labels, (seen_pairs, unseen_pairs)
    

def train_and_score(max_movie_id, training, testset, model_sizes):
    extractors = dict()
    models = dict()
    
    print "Creating models"
    for model_size in model_sizes:
        extractors[model_size] = FeatureHasher(n_features=2**model_size)
        models[model_size] = SGDClassifier(loss="log", penalty="L2")

    print "Training"
    for i, (user_id, seen_movies) in enumerate(training):
        print "Training on user", i, user_id
        labels, (seen_pairs, unseen_pairs) = generate_features(max_movie_id, seen_movies)
        for model_size, extractor in extractors.iteritems():
            seen_features = extractor.transform(seen_pairs)
            unseen_features = extractor.transform(unseen_pairs)
            features = sp.vstack([seen_features, unseen_features])
            model = models[model_size]
            model.partial_fit(features, labels, classes=[0, 1])
                
    print "Testing"
    all_labels = []
    all_predicted_labels = defaultdict(list)
    all_predicted_prob = defaultdict(list)
    for i, (user_id, seen_movies) in enumerate(testset):
        print "Testing on user", i, user_id
        labels, (seen_pairs, unseen_pairs) = generate_features(max_movie_id, seen_movies)
        all_labels.extend(labels)
        
        for model_size, extractor in extractors.iteritems():
            seen_features = extractor.transform(seen_pairs)
            unseen_features = extractor.transform(unseen_pairs)
            features = sp.vstack([seen_features, unseen_features])
            
            model = models[model_size]
            predicted_labels = model.predict(features)
            predicted_prob = model.predict_proba(features)
            all_predicted_labels[model_size].extend(predicted_labels)
            # Probabilities for positive class
            all_predicted_prob[model_size].extend(predicted_prob[:, 1])

    print "Scoring"
    aucs = []
    nnz_features = []
    for model_size, model in models.iteritems():
        pred_log_prob = all_predicted_prob[model_size]
        auc = roc_auc_score(all_labels, pred_log_prob)
        cm = confusion_matrix(all_labels, all_predicted_labels[model_size])
        print "Model size", model_size, "auc", auc
        print cm
        print
        aucs.append(auc)
        nnz_features.append(np.count_nonzero(model.coef_))

    return aucs, nnz_features
        

def parseargs():
    parser = argparse.ArgumentParser()

    parser.add_argument("--ratings-fl",
                        type=str,
                        required=True,
                        help="Ratings file")

    parser.add_argument("--figures-dir",
                        type=str,
                        required=True,
                        help="Directory for outputting figures")

    parser.add_argument("--model-bits",
                        type=int,
                        nargs="+",
                        help="Model sizes in terms of bits")

    return parser.parse_args()


if __name__ == "__main__":
    args = parseargs()

    ratings = read_ratings(args.ratings_fl)
    max_movie_id, training, test = create_training_sets(ratings)

    training = random.sample(training, 1000)
    test = random.sample(test, 100)

    aucs, nnzs = train_and_score(max_movie_id, training, test, args.model_bits)
    
    plot_aucs_nnzs(args.figures_dir + "/lr_hashing_auc_nnzs.png",
                   args.model_bits,
                   aucs,
                   nnzs)

"""
Proof of concept for a recommendation system implemented using Logistic Regression and feature hashing.  You'll need to download one of the MovieLens datasets from http://grouplens.org/datasets/movielens/.

Copyright 2016 <NAME>

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import argparse
from collections import defaultdict
from datetime import date
import random

from sklearn.feature_extraction import FeatureHasher
from sklearn.linear_model import SGDClassifier
from sklearn.metrics import roc_curve
from sklearn.metrics import roc_auc_score
from sklearn.metrics import confusion_matrix

import numpy as np
import scipy.sparse as sp

from common import read_ratings
from common import plot_aucs_nnzs

def create_training_sets(ratings):
    print "Creating user movie-interaction lists"
    user_movies = defaultdict(set)
    max_movie_id = 0
    for r in ratings:
        user_movies[r.user_id].add(r.movie_id)
        max_movie_id = max(max_movie_id, r.movie_id)


    training_set = []
    test_set = []
    for user_id, movie_ids in user_movies.iteritems():
        if random.random() >= 0.5:
            training_set.append((user_id, movie_ids))
        else:
            test_set.append((user_id, movie_ids))

    return max_movie_id, training_set, test_set

def generate_features(max_movie_id, seen_movies):
    all_movie_ids = set(range(0, max_movie_id + 1))
    seen_pairs = []
    unseen_pairs = []

    # positive examples
    for movie_id1 in seen_movies:
        movie_pairs = dict()
        for movie_id2 in seen_movies:
            # product with itself will always be 1 when the label is 1
            # and so will result in model overfitting
            if movie_id1 != movie_id2:
                movie_pairs["%s_%s" % (movie_id1, movie_id2)] = 1.
        seen_pairs.append(movie_pairs)

    # negative_examples
    unseen_movies = all_movie_ids - seen_movies
    for movie_id1 in random.sample(unseen_movies, len(seen_movies)):
        movie_pairs = dict()
        for movie_id2 in seen_movies:
            movie_pairs["%s_%s" % (movie_id1, movie_id2)] = 1.
        unseen_pairs.append(movie_pairs)

    labels = np.hstack([np.ones(len(seen_pairs)), np.zeros(len(unseen_pairs))])

    return labels, (seen_pairs, unseen_pairs)
    

def train_and_score(max_movie_id, training, testset, model_sizes):
    extractors = dict()
    models = dict()
    
    print "Creating models"
    for model_size in model_sizes:
        extractors[model_size] = FeatureHasher(n_features=2**model_size)
        models[model_size] = SGDClassifier(loss="log", penalty="L2")

    print "Training"
    for i, (user_id, seen_movies) in enumerate(training):
        print "Training on user", i, user_id
        labels, (seen_pairs, unseen_pairs) = generate_features(max_movie_id, seen_movies)
        for model_size, extractor in extractors.iteritems():
            seen_features = extractor.transform(seen_pairs)
            unseen_features = extractor.transform(unseen_pairs)
            features = sp.vstack([seen_features, unseen_features])
            model = models[model_size]
            model.partial_fit(features, labels, classes=[0, 1])
                
    print "Testing"
    all_labels = []
    all_predicted_labels = defaultdict(list)
    all_predicted_prob = defaultdict(list)
    for i, (user_id, seen_movies) in enumerate(testset):
        print "Testing on user", i, user_id
        labels, (seen_pairs, unseen_pairs) = generate_features(max_movie_id, seen_movies)
        all_labels.extend(labels)
        
        for model_size, extractor in extractors.iteritems():
            seen_features = extractor.transform(seen_pairs)
            unseen_features = extractor.transform(unseen_pairs)
            features = sp.vstack([seen_features, unseen_features])
            
            model = models[model_size]
            predicted_labels = model.predict(features)
            predicted_prob = model.predict_proba(features)
            all_predicted_labels[model_size].extend(predicted_labels)
            # Probabilities for positive class
            all_predicted_prob[model_size].extend(predicted_prob[:, 1])

    print "Scoring"
    aucs = []
    nnz_features = []
    for model_size, model in models.iteritems():
        pred_log_prob = all_predicted_prob[model_size]
        auc = roc_auc_score(all_labels, pred_log_prob)
        cm = confusion_matrix(all_labels, all_predicted_labels[model_size])
        print "Model size", model_size, "auc", auc
        print cm
        print
        aucs.append(auc)
        nnz_features.append(np.count_nonzero(model.coef_))

    return aucs, nnz_features
        

def parseargs():
    parser = argparse.ArgumentParser()

    parser.add_argument("--ratings-fl",
                        type=str,
                        required=True,
                        help="Ratings file")

    parser.add_argument("--figures-dir",
                        type=str,
                        required=True,
                        help="Directory for outputting figures")

    parser.add_argument("--model-bits",
                        type=int,
                        nargs="+",
                        help="Model sizes in terms of bits")

    return parser.parse_args()


if __name__ == "__main__":
    args = parseargs()

    ratings = read_ratings(args.ratings_fl)
    max_movie_id, training, test = create_training_sets(ratings)

    training = random.sample(training, 1000)
    test = random.sample(test, 100)

    aucs, nnzs = train_and_score(max_movie_id, training, test, args.model_bits)
    
    plot_aucs_nnzs(args.figures_dir + "/lr_hashing_auc_nnzs.png",
                   args.model_bits,
                   aucs,
                   nnzs)

Proof of concept for a recommendation system implemented using Logistic Regression and feature hashing.  You'll need to download one of the MovieLens datasets from http://grouplens.org/datasets/movielens/.  Copyright 2016 <NAME>  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at      http://www.apache.org/licenses/LICENSE-2.0  Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. # positive examples # product with itself will always be 1 when the label is 1 # and so will result in model overfitting # negative_examples # Probabilities for positive class