<reponame>CieloStrive/ML-IN-CYBER-SECURITY<filename>Spam_Filter/mlcs_lab1.py
# -*- coding: utf-8 -*-
"""MLCS_LAB1.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1KnN9oHgEbNbMZg8DooJNOqvXTWeBTj92

Upload dataset and unzip the file
"""

!tar xzf lingspam_public.tar.gz

"""Import library"""

from decimal import *
import numpy as np
import matplotlib.pyplot as plt
import math
import os
from collections import Counter
from collections import OrderedDict
from sklearn import svm

"""Load words in emails into a word list."""

train_path = './lingspam_public/lemm_stop'  # parent directory of dataset we use
paths = [os.path.join(train_path, folders) for folders in os.listdir(train_path)]  
all_words = [] 
labels = []  # 1 for spam, 0 for legit
test_path = ""
for path in paths:
    if(path[-2:] == '10'):  # path[-2:] check if this is part10 which is used for testing, skip 
        test_path = path
        continue
    files = [os.path.join(path, f) for f in os.listdir(path)]  # get file names of this part and generate path to files
    for email in files:
        if(email.find('spmsg') != -1): #check filenames to check if it is spam 
            labels.append(1)
        else:
            labels.append(0)
        
        with open(email) as m:
            for i, content in enumerate(m):  # i: line number , content: content of each line
                if(i == 2):  # open txt and find content is in the 3rd line
                    words = content.split()
                    all_words += words

word_dict = Counter(all_words)  

# remove symbols which have no big relationship with spam or legit
for word in list(word_dict.keys()):
    if(word.isalpha() == False):
        del word_dict[word]

# distinct words
dist_words = list(word_dict.keys())

"""# Compute Information Gain(IG)

1. compute H(C), C is RV that determins if a document is spam or legit
"""

spam = 0
legit = 0
for label in labels:
  if label == 1:
    spam += 1
  else:
    legit += 1  

print(f"spam email: {spam}, legit email: {legit}, total email {spam + legit}")
p_spam = spam / (spam + legit)

H_c = -1 * ( p_spam*np.log2(p_spam) + (1-p_spam)*np.log2(1-p_spam) )
print('H(C): ', H_c)

"""2. Compute H(C|Xi), need to create dictionary for word:#spam and word:#legit to help computing H(C|Xi)"""

word_spam = {}
word_legit = {}
for word in dist_words:
    word_spam[word] = 0
    word_legit[word] = 0

for path in paths:
    if(path[-2:] == '10'):  
        continue
    files = [os.path.join(path, f) for f in os.listdir(path)]  
    for email in files:
        if(email.find('spmsg') != -1):  
            is_spam = True
        else:
            is_spam = False
        dist_set = set() # there could be repeated word, we only need  occurance, use set
        with open(email) as m:
            for i, content in enumerate(m):  
                if(i == 2):  
                    words = content.split()
                    for word in words:
                        if(word.isalpha() == False): # skip symbols
                            continue
                        else:
                            dist_set.add(word)
        for e in dist_set:
            if(is_spam == True):
                word_spam[e] += 1
            else:
                word_legit[e] += 1

"""Compute H(C|Xi) = "combination" of P(X=x,C=c)log(P(C=c|X=x))"""

H_cx = {}
for word in dist_words:
    H_cx[word] = 0 
    # c = legit, x = 0 
    if ((legit - word_legit[word]) != 0 and (spam - word_spam[word] + legit - word_legit[word]) != 0): #sum only when non-zero probability
        H_cx[word] += -1 * (legit - word_legit[word]) / (spam + legit) * np.log2( (legit - word_legit[word]) / (spam - word_spam[word] + legit - word_legit[word]) )
    # c = legit, x = 1 
    if ((word_legit[word]) != 0 and (word_spam[word] + word_legit[word]) != 0):
        H_cx[word] += -1 * (word_legit[word]) / (spam + legit) * np.log2( word_legit[word] / (word_spam[word] + word_legit[word]) )
    # c = spam, x = 0 
    if (spam-word_spam[word] != 0 and (spam - word_spam[word] + legit - word_legit[word]) != 0):
        H_cx[word] += -1 * (spam - word_spam[word]) / (spam + legit) * np.log2( (spam - word_spam[word] ) / (spam - word_spam[word] + legit - word_legit[word]) ) 
    # c = spam, x = 1 
    if (word_spam[word] != 0 and word_spam[word] + word_legit[word] != 0):
        H_cx[word] += -1 * (word_spam[word]) / (spam + legit) * np.log2( word_spam[word] / (word_spam[word] + word_legit[word]) )

"""3. Compute IG"""

IG = {}
for word in list(H_cx.keys()):
    IG[word] = H_c - H_cx[word]

"""# Get top N features"""

# sort by value
# sorted can be applied to iterable containers
# sorted return a new result rather than sort in place
# sorted return a list object not a sorted dictionary!!!!!!!
# sorted_IG = sorted(IG.items(), key=lambda x : x[1], reverse=True)
sorted_IG = OrderedDict(sorted(IG.items(), key=lambda x : x[1], reverse=True))
f10 = list(sorted_IG.keys())[:10]  # top 10
f100 = list(sorted_IG.keys())[:100]  # top 100
f1000 = list(sorted_IG.keys())[:1000]  # top 1000

"""# Part1 report: top-10 words:"""

print("Top-10 words:")
c=0
for i in sorted_IG.items():
  c+=1
  if c > 10:
    break
  print(i[0],":",i[1])

"""# Bernoulli NB classifier with binary feature

*Load test data*
"""

test_label = []  
f10_test_bbf = []  # test emails' feature matrix for top 10
f100_test_bbf = []  # test emails' feature matrix for top 100
f1000_test_bbf = []  # test emails' feature matrix for top 1000
emails = [os.path.join(test_path, file_name) for file_name in os.listdir(test_path)]  
for email in emails:
    # spam: 1, legit: 0
    if email.find('spmsg')!=-1:
        test_label.append(1)
    else:
        test_label.append(0)
    
    with open(email) as m:
        temp_f10_bf = []
        temp_f100_bf = []
        temp_f1000_bf = []

        for i, content in enumerate(m):  
            if i==2:  # content at 3rd line
                words_in_email = set(content.split())
                for w in f10:
                    if w in words_in_email:
                        temp_f10_bf.append(1)
                    else:
                        temp_f10_bf.append(0)
                for w in f100:
                    if w in words_in_email:
                        temp_f100_bf.append(1)
                    else:
                        temp_f100_bf.append(0)
                for w in f1000:
                    if w in words_in_email:
                        temp_f1000_bf.append(1)
                    else:
                        temp_f1000_bf.append(0)

        f10_test_bbf.append(temp_f10_bf)
        f100_test_bbf.append(temp_f100_bf)
        f1000_test_bbf.append(temp_f1000_bf)

"""**1. top 10 feature**"""

N = 10

"""1.1 Compute probability"""

# x is conbination of words
# compute P(x|spam)
p_x_spam_10 = []
for case in f10_test_bbf:
    p = 1
    for i in range(N):
        if case[i] == 1:
            p *= (word_spam[f10[i]] + 1) / (spam + 2) # with laplacian smoothing
        else:
            p *= (1 - (word_spam[f10[i]] + 1) / (spam + 2)) # with laplacian smoothing
    p_x_spam_10.append(p)    

# compute P(x|legit)
p_x_legit_10 = []
for case in f10_test_bbf:
    p = 1
    for i in range(N):
        if case[i] == 1:
            p *= (word_legit[f10[i]] + 1) / (legit + 2) # with laplacian smoothing
        else:
            p *= 1 - (word_legit[f10[i]] + 1) / (legit + 2) # with laplacian smoothing
    p_x_legit_10.append(p)    

# compute P(spam)
p_spam = spam/(spam+legit)

# compute P(legit)
p_legit = legit/(spam+legit) 

# compute P(x)
p_x_10 = []
for i in range(len(f10_test_bbf)):
    p = p_spam * p_x_spam_10[i] + p_legit * p_x_legit_10[i]
    p_x_10.append(p)

"""1.2 Classify by computing P(spam|X) and P(legit|X) and compare"""

f10_pre = []
p_spam_x_10 = []
p_legit_x_10 = []
for i in range(len(f10_test_bbf)):
    psx = p_x_spam_10[i] * p_spam / p_x_10[i]
    p_spam_x_10.append(psx)
    plx = p_x_legit_10[i] * p_legit / p_x_10[i]
    p_legit_x_10.append(plx)
    if psx >= plx :
        f10_pre.append(1)
    else:
        f10_pre.append(0)

"""1.3 Report result"""

test_label[:10]

f10_pre[:10]

#p_spam_x_10[:10]

#p_legit_x_10[:10]

correct = 0;
total = len(test_label)
for i in range(total):
    if test_label[i] == f10_pre[i]:
        correct+=1
correct_rate = correct / total
print("correct rate: ",correct_rate)

# spam precision is defined as the fraction of true spam e-mails among all e-mails predicted as spam
total_pre_spam = 0
true_spam_in_spam = 0
for i in range(total):
    if f10_pre[i] == 1:
        total_pre_spam += 1
        if test_label[i] == 1:
            true_spam_in_spam += 1
spam_precision_bbf_10 = true_spam_in_spam / total_pre_spam   
print("spam precision: ",spam_precision_bbf_10)

# spam recall is defined as fraction of true spam e-mails predicted as spam.
pre_spam_in_true = 0
total_true_spam = 0
for i in range(total):
    if test_label[i] == 1:
        total_true_spam += 1
        if f10_pre[i] == 1:
            pre_spam_in_true += 1
        
spam_recall_bbf_10 = pre_spam_in_true / total_true_spam  
print("spam_recall: ",spam_recall_bbf_10)

"""**2. top 100 feature**"""

N = 100

"""2.1 Compute probability"""

# x is conbination of words
# compute P(x|spam)
p_x_spam_100 = []
for case in f100_test_bbf:
    p = 1
    for i in range(N):
        if case[i] == 1:
            p *= (word_spam[f100[i]] + 1) / (spam + 2) # with laplacian smoothing
        else:
            p *= (1 - (word_spam[f100[i]] + 1) / (spam + 2)) # with laplacian smoothing
    p_x_spam_100.append(p)    

# compute P(x|legit)
p_x_legit_100 = []
for case in f100_test_bbf:
    p = 1
    for i in range(N):
        if case[i] == 1:
            p *= (word_legit[f100[i]] + 1) / (legit + 2) # with laplacian smoothing
        else:
            p *= 1 - (word_legit[f100[i]] + 1) / (legit + 2) # with laplacian smoothing
    p_x_legit_100.append(p)    

# compute P(spam)
p_spam = spam/(spam+legit)

# compute P(legit)
p_legit = legit/(spam+legit) 

# compute P(x)
p_x_100 = []
for i in range(len(f100_test_bbf)):
    p = p_spam * p_x_spam_100[i] + p_legit * p_x_legit_100[i]
    p_x_100.append(p)
 
#print(len(p_x_spam_100))
#print(len(p_x_100))

"""2.2 Classify by computing P(spam|X) and P(legit|X) and compare"""

f100_pre = []
p_spam_x_100 = []
p_legit_x_100 = []
for i in range(len(f100_test_bbf)):
    psx = p_x_spam_100[i] * p_spam / p_x_100[i]
    p_spam_x_100.append(psx)
    plx = p_x_legit_100[i] * p_legit / p_x_100[i]
    p_legit_x_100.append(plx)
    if psx >= plx :
        f100_pre.append(1)
    else:
        f100_pre.append(0)

"""2.3 Report result"""

test_label[:10]

f100_pre[:10]

correct = 0;
total = len(test_label)
for i in range(total):
    if test_label[i] == f100_pre[i]:
        correct+=1
correct_rate = correct / total
print("correct rate: ",correct_rate)

# spam precision is defined as the fraction of true spam e-mails among all e-mails predicted as spam
total_pre_spam = 0
true_spam_in_spam = 0
for i in range(total):
    if f100_pre[i] == 1:
        total_pre_spam += 1
        if test_label[i] == 1:
            true_spam_in_spam += 1
spam_precision_bbf_100 = true_spam_in_spam / total_pre_spam   
print("spam precision: ",spam_precision_bbf_100)

# spam recall is defined as fraction of true spam e-mails predicted as spam.
pre_spam_in_true = 0
total_true_spam = 0
for i in range(total):
    if test_label[i] == 1:
        total_true_spam += 1
        if f100_pre[i] == 1:
            pre_spam_in_true += 1
        
spam_recall_bbf_100 = pre_spam_in_true / total_true_spam  
print("spam_recall: ",spam_recall_bbf_100)

"""3. top 100 feature"""

N = 1000

"""**3.1 Compute probability**"""

# x is conbination of words
# compute P(x|spam)
p_x_spam_1000 = []
for case in f1000_test_bbf:
    p = 1
    for i in range(N):
        if case[i] == 1:
            p *= (word_spam[f1000[i]] + 1) / (spam + 2) # with laplacian smoothing
        else:
            p *= (1 - (word_spam[f1000[i]] + 1) / (spam + 2)) # with laplacian smoothing
    p_x_spam_1000.append(p)    

# compute P(x|legit)
p_x_legit_1000 = []
for case in f1000_test_bbf:
    p = 1
    for i in range(N):
        if case[i] == 1:
            p *= (word_legit[f1000[i]] + 1) / (legit + 2) # with laplacian smoothing
        else:
            p *= 1 - (word_legit[f1000[i]] + 1) / (legit + 2) # with laplacian smoothing
    p_x_legit_1000.append(p)    

# compute P(spam)
p_spam = spam/(spam+legit)

# compute P(legit)
p_legit = legit/(spam+legit) 

# compute P(x)
p_x_1000 = []
for i in range(len(f1000_test_bbf)):
    p = p_spam * p_x_spam_1000[i] + p_legit * p_x_legit_1000[i]
    p_x_1000.append(p)
 
#print(len(p_x_spam_1000))
#print(len(p_x_1000))

"""3.2 Classify by computing P(spam|X) and P(legit|X) and compare"""

f1000_pre = []
p_spam_x_1000 = []
p_legit_x_1000 = []
for i in range(len(f1000_test_bbf)):
    psx = p_x_spam_1000[i] * p_spam / p_x_1000[i]
    p_spam_x_1000.append(psx)
    plx = p_x_legit_1000[i] * p_legit / p_x_1000[i]
    p_legit_x_1000.append(plx)
    if psx >= plx :
        f1000_pre.append(1)
    else:
        f1000_pre.append(0)

"""3.3 Report result"""

test_label[:10]

f1000_pre[:10]

correct = 0;
total = len(test_label)
for i in range(total):
    if test_label[i] == f1000_pre[i]:
        correct+=1
correct_rate = correct / total
print("correct rate: ",correct_rate)

# spam precision is defined as the fraction of true spam e-mails among all e-mails predicted as spam
total_pre_spam = 0
true_spam_in_spam = 0
for i in range(total):
    if f1000_pre[i] == 1:
        total_pre_spam += 1
        if test_label[i] == 1:
            true_spam_in_spam += 1
spam_precision_bbf_1000 = true_spam_in_spam / total_pre_spam   
print("spam precision: ",spam_precision_bbf_1000)

# spam recall is defined as fraction of true spam e-mails predicted as spam.
pre_spam_in_true = 0
total_true_spam = 0
for i in range(total):
    if test_label[i] == 1:
        total_true_spam += 1
        if f1000_pre[i] == 1:
            pre_spam_in_true += 1
        
spam_recall_bbf_1000 = pre_spam_in_true / total_true_spam  
print("spam_recall: ",spam_recall_bbf_1000)

"""***It seems that spam precision goes up while spam recall goes down with more feature selected***

# Multinomial NB with binary features

*Load test data*
"""

test_label = []  
f10_test_mbf = []  # test emails' feature matrix for top 10
f100_test_mbf = []  # test emails' feature matrix for top 100
f1000_test_mbf = []  # test emails' feature matrix for top 1000
emails = [os.path.join(test_path, file_name) for file_name in os.listdir(test_path)]  
for email in emails:
    # spam: 1, legit: 0
    if email.find('spmsg')!=-1:
        test_label.append(1)
    else:
        test_label.append(0)
    
    with open(email) as m:
        temp_f10_bf = []
        temp_f100_bf = []
        temp_f1000_bf = []

        for i, content in enumerate(m):  
            if i==2:  # content at 3rd line
                words_in_email = set(content.split())
                for w in f10:
                    if w in words_in_email:
                        temp_f10_bf.append(1)
                    else:
                        temp_f10_bf.append(0)
                for w in f100:
                    if w in words_in_email:
                        temp_f100_bf.append(1)
                    else:
                        temp_f100_bf.append(0)
                for w in f1000:
                    if w in words_in_email:
                        temp_f1000_bf.append(1)
                    else:
                        temp_f1000_bf.append(0)

        f10_test_mbf.append(temp_f10_bf)
        f100_test_mbf.append(temp_f100_bf)
        f1000_test_mbf.append(temp_f1000_bf)

"""**1. top 10 feature**"""

N = 10

"""1.1 Compute probability"""

# compute P(x|spam)
p_x_spam_10 = []
for case in f10_test_mbf:
    p = 1
    for i in range(N):
        if case[i] == 1:
            p *= (word_spam[f10[i]] + 1) / (spam + 2) # with laplacian smoothing
        else:
            p *= 1 # different from bernoulli: ignore negative evidence
    p_x_spam_10.append(p)    

# compute P(x|legit)
p_x_legit_10 = []
for case in f10_test_mbf:
    p = 1
    for i in range(N):
        if case[i] == 1:
            p *= (word_legit[f10[i]] + 1) / (legit + 2) # with laplacian smoothing
        else:
            p *= 1 # different from bernoulli: ignore negative evidence
    p_x_legit_10.append(p)    

# compute P(spam)
p_spam = spam/(spam+legit)

# compute P(legit)
p_legit = legit/(spam+legit) 

# compute P(x)
p_x_10 = []
for i in range(len(f10_test_mbf)):
    p = p_spam * p_x_spam_10[i] + p_legit * p_x_legit_10[i]
    p_x_10.append(p)
 
#print(len(p_x_spam_10))
#print(len(p_x_10))

"""1.2 Classify by computing P(spam|X) and P(legit|X) and compare"""

f10_pre = []
p_spam_x_10 = []
p_legit_x_10 = []
for i in range(len(f10_test_mbf)):
    psx = p_x_spam_10[i] * p_spam / p_x_10[i]
    p_spam_x_10.append(psx)
    plx = p_x_legit_10[i] * p_legit / p_x_10[i]
    p_legit_x_10.append(plx)
    if psx >= plx :
        f10_pre.append(1)
    else:
        f10_pre.append(0)

"""1.3 report result"""

correct = 0;
total = len(test_label)
for i in range(total):
    if test_label[i] == f10_pre[i]:
        correct+=1
correct_rate = correct / total
print("correct rate: ",correct_rate)

# spam precision is defined as the fraction of true spam e-mails among all e-mails predicted as spam
total_pre_spam = 0
true_spam_in_spam = 0
for i in range(total):
    if f10_pre[i] == 1:
        total_pre_spam += 1
        if test_label[i] == 1:
            true_spam_in_spam += 1
spam_precision_mbf_10 = true_spam_in_spam / total_pre_spam   
print("spam precision: ",spam_precision_mbf_10)

# spam recall is defined as fraction of true spam e-mails predicted as spam.
pre_spam_in_true = 0
total_true_spam = 0
for i in range(total):
    if test_label[i] == 1:
        total_true_spam += 1
        if f10_pre[i] == 1:
            pre_spam_in_true += 1
        
spam_recall_mbf_10 = pre_spam_in_true / total_true_spam  
print("spam_recall: ",spam_recall_mbf_10)

"""**2. top 100 feature**"""

N = 100

"""2.1 Compute probability"""

# compute P(x|spam)
p_x_spam_100 = []
for case in f100_test_mbf:
    p = 1
    for i in range(N):
        if case[i] == 1:
            p *= (word_spam[f100[i]] + 1) / (spam + 2) # with laplacian smoothing
        else:
            p *= 1 # different from bernoulli: ignore negative evidence 
    p_x_spam_100.append(p)    

# compute P(x|legit)
p_x_legit_100 = []
for case in f100_test_mbf:
    p = 1
    for i in range(N):
        if case[i] == 1:
            p *= (word_legit[f100[i]] + 1) / (legit + 2) # with laplacian smoothing
        else:
            p *= 1 # different from bernoulli: ignore negative evidence
    p_x_legit_100.append(p)    

# compute P(spam)
p_spam = spam/(spam+legit)

# compute P(legit)
p_legit = legit/(spam+legit) 

# compute P(x)
p_x_100 = []
for i in range(len(f100_test_mbf)):
    p = p_spam * p_x_spam_100[i] + p_legit * p_x_legit_100[i]
    p_x_100.append(p)
 
#print(len(p_x_spam_100))
#print(len(p_x_100))

"""2.2 Classify by computing P(spam|X) and P(legit|X) and compare"""

f100_pre = []
p_spam_x_100 = []
p_legit_x_100 = []
for i in range(len(f100_test_mbf)):
    psx = p_x_spam_100[i] * p_spam / p_x_100[i]
    p_spam_x_10.append(psx)
    plx = p_x_legit_100[i] * p_legit / p_x_100[i]
    p_legit_x_100.append(plx)
    if psx >= plx :
        f100_pre.append(1)
    else:
        f100_pre.append(0)

"""2.3 Report result"""

correct = 0;
total = len(test_label)
for i in range(total):
    if test_label[i] == f100_pre[i]:
        correct+=1
correct_rate = correct / total
print("correct rate: ",correct_rate)

# spam precision is defined as the fraction of true spam e-mails among all e-mails predicted as spam
total_pre_spam = 0
true_spam_in_spam = 0
for i in range(total):
    if f100_pre[i] == 1:
        total_pre_spam += 1
        if test_label[i] == 1:
            true_spam_in_spam += 1
spam_precision_mbf_100 = true_spam_in_spam / total_pre_spam   
print("spam precision: ",spam_precision_mbf_100)

# spam recall is defined as fraction of true spam e-mails predicted as spam.
pre_spam_in_true = 0
total_true_spam = 0
for i in range(total):
    if test_label[i] == 1:
        total_true_spam += 1
        if f100_pre[i] == 1:
            pre_spam_in_true += 1
        
spam_recall_mbf_100 = pre_spam_in_true / total_true_spam  
print("spam_recall: ",spam_recall_mbf_100)

"""**3. top 1000 feature**"""

N = 1000

"""3.1 Compute probability"""

# compute P(x|spam)
p_x_spam_1000 = []
for case in f1000_test_mbf:
    p = 1
    for i in range(N):
        if case[i] == 1:
            p *= (word_spam[f1000[i]] + 1) / (spam + 2) # with laplacian smoothing
        else:
            p *= 1 # different from bernoulli: ignore negative evidence 
    p_x_spam_1000.append(p)    

# compute P(x|legit)
p_x_legit_1000 = []
for case in f1000_test_mbf:
    p = 1
    for i in range(N):
        if case[i] == 1:
            p *= (word_legit[f1000[i]] + 1) / (legit + 2) # with laplacian smoothing
        else:
            p *= 1 # different from bernoulli: ignore negative evidence
    p_x_legit_1000.append(p)    

# compute P(spam)
p_spam = spam/(spam+legit)

# compute P(legit)
p_legit = legit/(spam+legit) 

# compute P(x)
p_x_1000 = []
for i in range(len(f1000_test_mbf)):
    p = p_spam * p_x_spam_1000[i] + p_legit * p_x_legit_1000[i]
    p_x_1000.append(p)
 
#print(len(p_x_spam_1000))
#print(len(p_x_1000))

"""3.2 Classify by computing P(spam|X) and P(legit|X) and compare"""

f1000_pre = []
p_spam_x_1000 = []
p_legit_x_1000 = []
for i in range(len(f1000_test_mbf)):
    psx = p_x_spam_1000[i] * p_spam / p_x_1000[i]
    p_spam_x_1000.append(psx)
    plx = p_x_legit_1000[i] * p_legit / p_x_1000[i]
    p_legit_x_1000.append(plx)
    if psx >= plx :
        f1000_pre.append(1)
    else:
        f1000_pre.append(0)

"""3.3 Report result"""

correct = 0;
total = len(test_label)
for i in range(total):
    if test_label[i] == f1000_pre[i]:
        correct+=1
correct_rate = correct / total
print("correct rate: ",correct_rate)

# spam precision is defined as the fraction of true spam e-mails among all e-mails predicted as spam
total_pre_spam = 0
true_spam_in_spam = 0
for i in range(total):
    if f1000_pre[i] == 1:
        total_pre_spam += 1
        if test_label[i] == 1:
            true_spam_in_spam += 1
spam_precision_mbf_1000 = true_spam_in_spam / total_pre_spam   
print("spam precision: ",spam_precision_mbf_1000)

# spam recall is defined as fraction of true spam e-mails predicted as spam.
pre_spam_in_true = 0
total_true_spam = 0
for i in range(total):
    if test_label[i] == 1:
        total_true_spam += 1
        if f1000_pre[i] == 1:
            pre_spam_in_true += 1
        
spam_recall_mbf_1000 = pre_spam_in_true / total_true_spam  
print("spam_recall: ",spam_recall_mbf_1000)

"""***Adding feature seems helps multinomial NB with binary feature positively***

# Multinomial NB with term frequency (TF) features

*Load test data*
"""

test_label = []  
f10_test_mtf = []  # test emails' feature matrix for top 10
f100_test_mtf = []  # test emails' feature matrix for top 100
f1000_test_mtf = []  # test emails' feature matrix for top 1000
emails = [os.path.join(test_path, file_name) for file_name in os.listdir(test_path)]  
for email in emails:
    # spam: 1, legit: 0
    if email.find('spmsg')!=-1:
        test_label.append(1)
    else:
        test_label.append(0)
    
    with open(email) as m:
        temp_f10_tf = []
        temp_f100_tf = []
        temp_f1000_tf = []

        for i, content in enumerate(m):  
            if i==2:  # content at 3rd line
                words_in_email = set(content.split())
                for w in f10:
                    if w in words_in_email:
                        temp_f10_tf.append(1)
                    else:
                        temp_f10_tf.append(0)
                for w in f100:
                    if w in words_in_email:
                        temp_f100_tf.append(1)
                    else:
                        temp_f100_tf.append(0)
                for w in f1000:
                    if w in words_in_email:
                        temp_f1000_tf.append(1)
                    else:
                        temp_f1000_tf.append(0)

        f10_test_mtf.append(temp_f10_tf)
        f100_test_mtf.append(temp_f100_tf)
        f1000_test_mtf.append(temp_f1000_tf)

"""*For Term Frequency we need new dictionary for words*"""

# occurence
word_occ_spam = {}
word_occ_legit = {}
for word in dist_words:
    word_occ_spam[word] = 0
    word_occ_legit[word] = 0

for path in paths:
    if(path[-2:] == '10'):  
        continue
    files = [os.path.join(path, f) for f in os.listdir(path)]  
    for email in files:
        if(email.find('spmsg') != -1):  
            is_spam = True
        else:
            is_spam = False
        # dist_set = set() # there could be repeated word, we only need  occurance, use set
        with open(email) as m:
            for i, content in enumerate(m):  
                if(i == 2):  
                    words = content.split()
                    for word in words:
                        if(word.isalpha() == False): # skip symbols
                            continue
                        else:
                            if(is_spam == True):
                                word_occ_spam[word] += 1
                            else:
                                word_occ_legit[word] += 1

# total #occurence of word in spam
total_occ_spam = 0
total_occ_legit = 0
for i in word_occ_spam.values():
    total_occ_spam += i
for i in word_occ_legit.values():
    total_occ_legit += i
print(f'total words\' occurence in spam is {total_occ_spam}, total occurence in legit is {total_occ_legit}')

"""**1. Top 10 feature**"""

N = 10

"""1.1 compute probability

*In TF computation, it requires a lot in precision so we need more decimal: import decimal library*
"""

# In TF computation, it requires a lot in precision so we need more decimal
from decimal import *
getcontext().prec = 32
# M = #distinct_term
M = len(dist_words)
# compute P(x|spam)
p_x_spam_10 = []
for case in f10_test_mtf:
    p = 1
    for i in range(N):
        if case[i] == 1:
            p *= Decimal(word_occ_spam[f10[i]] + 1) / Decimal(total_occ_spam + M) # with laplacian smoothing
            # if p == 0 : 
            #     print(word_occ_spam[f10[i]] + 1, " ", total_occ_spam + M)
        else:
            p *= 1 # different from bernoulli: ignore negative evidence 
    p_x_spam_10.append(p)    

# compute P(x|legit)
p_x_legit_10 = []
for case in f10_test_mtf:
    p = 1
    for i in range(N):
        if case[i] == 1:
            p *= Decimal(word_occ_legit[f10[i]] + 1) / Decimal(total_occ_legit + M) # with laplacian smoothing
        else:
            p *= 1 # different from bernoulli: ignore negative evidence
    p_x_legit_10.append(p)    

# compute P(spam)
p_spam = Decimal(spam)/Decimal((spam+legit))

# compute P(legit)
p_legit = Decimal(legit)/Decimal((spam+legit)) 

# compute P(x)
p_x_10 = []
for i in range(len(f10_test_mtf)):
    p = p_spam * p_x_spam_10[i] + p_legit * p_x_legit_10[i]
    # if p == 0:
    #     print(p_spam, " ",p_x_spam_10[i]," ", p_legit," ", p_x_legit_10[i])
    p_x_10.append(p)
 
#print(len(p_x_spam_10))
#print(len(p_x_10))

"""1.2 Classify"""

f10_pre = []
p_spam_x_10 = []
p_legit_x_10 = []
for i in range(len(f10_test_mbf)):
    psx = p_x_spam_10[i] * p_spam / p_x_10[i]
    p_spam_x_10.append(psx)
    plx = p_x_legit_10[i] * p_legit / p_x_10[i]
    p_legit_x_10.append(plx)
    if psx >= plx :
        f10_pre.append(1)
    else:
        f10_pre.append(0)

print("samples of P(X|spam) with Decimal: ")
print(p_spam_x_10[:10])

"""1.3 Report result"""

correct = 0;
total = len(test_label)
for i in range(total):
    if test_label[i] == f10_pre[i]:
        correct+=1
correct_rate = correct / total
print("correct rate: ",correct_rate)

# spam precision is defined as the fraction of true spam e-mails among all e-mails predicted as spam
total_pre_spam = 0
true_spam_in_spam = 0
for i in range(total):
    if f10_pre[i] == 1:
        total_pre_spam += 1
        if test_label[i] == 1:
            true_spam_in_spam += 1
spam_precision_mtf_10 = true_spam_in_spam / total_pre_spam   
print("spam precision: ",spam_precision_mtf_10)

# spam recall is defined as fraction of true spam e-mails predicted as spam.
pre_spam_in_true = 0
total_true_spam = 0
for i in range(total):
    if test_label[i] == 1:
        total_true_spam += 1
        if f10_pre[i] == 1:
            pre_spam_in_true += 1
        
spam_recall_mtf_10 = pre_spam_in_true / total_true_spam  
print("spam_recall: ",spam_recall_mtf_10)

"""**2. Top 100 feature**"""

N = 100

"""2.1 Compute probability"""

# In TF computation, it requires a lot in precision so we need more decimal
from decimal import *
getcontext().prec = 32
# M = #distinct_term
M = len(dist_words)
# compute P(x|spam)
p_x_spam_100 = []
for case in f100_test_mtf:
    p = 1
    for i in range(N):
        if case[i] == 1:
            p *= Decimal(word_occ_spam[f100[i]] + 1) / Decimal(total_occ_spam + M) # with laplacian smoothing
            # if p == 0 : 
            #     print(word_occ_spam[f100[i]] + 1, " ", total_occ_spam + M)
        else:
            p *= 1 # different from bernoulli: ignore negative evidence 
    p_x_spam_100.append(p)    

# compute P(x|legit)
p_x_legit_100 = []
for case in f100_test_mtf:
    p = 1
    for i in range(N):
        if case[i] == 1:
            p *= Decimal(word_occ_legit[f100[i]] + 1) / Decimal(total_occ_legit + M) # with laplacian smoothing
        else:
            p *= 1 # different from bernoulli: ignore negative evidence
    p_x_legit_100.append(p)    

# compute P(spam)
p_spam = Decimal(spam)/Decimal((spam+legit))

# compute P(legit)
p_legit = Decimal(legit)/Decimal((spam+legit)) 

# compute P(x)
p_x_100 = []
for i in range(len(f100_test_mtf)):
    p = p_spam * p_x_spam_100[i] + p_legit * p_x_legit_100[i]
    # if p == 0:
    #     print(p_spam, " ",p_x_spam_100[i]," ", p_legit," ", p_x_legit_100[i])
    p_x_100.append(p)
 
#print(len(p_x_spam_100))
#print(len(p_x_100))

"""2.2 Classify"""

f100_pre = []
p_spam_x_100 = []
p_legit_x_100 = []
for i in range(len(f100_test_mbf)):
    psx = p_x_spam_100[i] * p_spam / p_x_100[i]
    p_spam_x_100.append(psx)
    plx = p_x_legit_100[i] * p_legit / p_x_100[i]
    p_legit_x_100.append(plx)
    if psx >= plx :
        f100_pre.append(1)
    else:
        f100_pre.append(0)

print("samples of P(X|spam) with Decimal: ")
print(p_spam_x_100[:10])

"""2.3 Report result"""

correct = 0;
total = len(test_label)
for i in range(total):
    if test_label[i] == f100_pre[i]:
        correct+=1
correct_rate = correct / total
print("correct rate: ",correct_rate)

# spam precision is defined as the fraction of true spam e-mails among all e-mails predicted as spam
total_pre_spam = 0
true_spam_in_spam = 0
for i in range(total):
    if f100_pre[i] == 1:
        total_pre_spam += 1
        if test_label[i] == 1:
            true_spam_in_spam += 1
spam_precision_mtf_100 = true_spam_in_spam / total_pre_spam   
print("spam precision: ",spam_precision_mtf_100)

# spam recall is defined as fraction of true spam e-mails predicted as spam.
pre_spam_in_true = 0
total_true_spam = 0
for i in range(total):
    if test_label[i] == 1:
        total_true_spam += 1
        if f100_pre[i] == 1:
            pre_spam_in_true += 1
        
spam_recall_mtf_100 = pre_spam_in_true / total_true_spam  
print("spam_recall: ",spam_recall_mtf_100)

"""**3. Top 1000 feature**"""

N = 1000

"""3.1 Compute probability"""

# In TF computation, it requires a lot in precision so we need more decimal
from decimal import *
getcontext().prec = 32
# M = #distinct_term
M = len(dist_words)
# compute P(x|spam)
p_x_spam_1000 = []
for case in f1000_test_mtf:
    p = 1
    for i in range(N):
        if case[i] == 1:
            p *= Decimal(word_occ_spam[f1000[i]] + 1) / Decimal(total_occ_spam + M) # with laplacian smoothing
            # if p == 0 : 
            #     print(word_occ_spam[f1000[i]] + 1, " ", total_occ_spam + M)
        else:
            p *= 1 # different from bernoulli: ignore negative evidence 
    p_x_spam_1000.append(p)    

# compute P(x|legit)
p_x_legit_1000 = []
for case in f1000_test_mtf:
    p = 1
    for i in range(N):
        if case[i] == 1:
            p *= Decimal(word_occ_legit[f1000[i]] + 1) / Decimal(total_occ_legit + M) # with laplacian smoothing
        else:
            p *= 1 # different from bernoulli: ignore negative evidence
    p_x_legit_1000.append(p)    

# compute P(spam)
p_spam = Decimal(spam)/Decimal((spam+legit))

# compute P(legit)
p_legit = Decimal(legit)/Decimal((spam+legit)) 

# compute P(x)
p_x_1000 = []
for i in range(len(f1000_test_mtf)):
    p = p_spam * p_x_spam_1000[i] + p_legit * p_x_legit_1000[i]
    # if p == 0:
    #     print(p_spam, " ",p_x_spam_1000[i]," ", p_legit," ", p_x_legit_1000[i])
    p_x_1000.append(p)
 
#print(len(p_x_spam_1000))
#print(len(p_x_1000))

"""3.2 Classify"""

f1000_pre = []
p_spam_x_1000 = []
p_legit_x_1000 = []
for i in range(len(f1000_test_mbf)):
    psx = p_x_spam_1000[i] * p_spam / p_x_1000[i]
    p_spam_x_1000.append(psx)
    plx = p_x_legit_1000[i] * p_legit / p_x_1000[i]
    p_legit_x_1000.append(plx)
    if psx >= plx :
        f1000_pre.append(1)
    else:
        f1000_pre.append(0)

print("samples of P(X|spam) with Decimal: ")
print(p_spam_x_1000[:10])

"""3.3 Report"""

correct = 0;
total = len(test_label)
for i in range(total):
    if test_label[i] == f1000_pre[i]:
        correct+=1
correct_rate = correct / total
print("correct rate: ",correct_rate)

# spam precision is defined as the fraction of true spam e-mails among all e-mails predicted as spam
total_pre_spam = 0
true_spam_in_spam = 0
for i in range(total):
    if f1000_pre[i] == 1:
        total_pre_spam += 1
        if test_label[i] == 1:
            true_spam_in_spam += 1
spam_precision_mtf_1000 = true_spam_in_spam / total_pre_spam   
print("spam precision: ",spam_precision_mtf_1000)

# spam recall is defined as fraction of true spam e-mails predicted as spam.
pre_spam_in_true = 0
total_true_spam = 0
for i in range(total):
    if test_label[i] == 1:
        total_true_spam += 1
        if f1000_pre[i] == 1:
            pre_spam_in_true += 1
        
spam_recall_mtf_1000 = pre_spam_in_true / total_true_spam  
print("spam_recall: ",spam_recall_mtf_1000)

"""# Part2 report: spam precision and spam recall of combinations of NB filters and top-N feature"""

print(f"Bernoulli NB with binary feature and top-10: precesion = {spam_precision_bbf_10}, recall = {spam_recall_bbf_10}")
print(f"Bernoulli NB with binary feature and top-100: precesion = {spam_precision_bbf_100}, recall = {spam_recall_bbf_100}")
print(f"Bernoulli NB with binary feature and top-1000: precesion = {spam_precision_bbf_1000}, recall = {spam_recall_bbf_1000}")
print(f"Multinomial NB with binary feature and top-10: precesion = {spam_precision_mbf_10}, recall = {spam_recall_mbf_10}")
print(f"Multinomial NB with binary feature and top-100: precesion = {spam_precision_mbf_100}, recall = {spam_recall_mbf_100}")
print(f"Multinomial NB with binary feature and top-1000: precesion = {spam_precision_mbf_1000}, recall = {spam_recall_mbf_1000}")
print(f"Multinomial NB with term frequency and top-10: precesion = {spam_precision_mtf_10}, recall = {spam_recall_mtf_10}")
print(f"Multinomial NB with term frequency and top-100: precesion = {spam_precision_mtf_100}, recall = {spam_recall_mtf_100}")
print(f"Multinomial NB with term frequency and top-1000: precesion = {spam_precision_mtf_1000}, recall = {spam_recall_mtf_1000}")

"""# SVM based filter

load data, generate top-N feature matrix with IG
"""

train_path = './lingspam_public/lemm_stop'  # parent directory of dataset we use
paths = [os.path.join(train_path, folders) for folders in os.listdir(train_path)]  
all_words = [] 

train_data = [] # all feature
train_data_10 = []
train_data_100 = []
train_data_1000 = []
train_label = []

test_data = [] # all feature
test_data_10 = []
test_data_100 = []
test_data_1000 = []
test_label = []



for path in paths:
    files = [os.path.join(path, f) for f in os.listdir(path)]  # get file names of this part and generate path to files
    for email in files:
        if email.find('spmsg') != -1 : #check filenames to check if it is spam 
            if path[-2:] == '10' : # if it is test data
                test_label.append(1)
            else:
                train_label.append(1)
        else:
            if path[-2:] == '10' :
                test_label.append(0)
            else:
                train_label.append(0)
        
        with open(email) as m:
            for i, content in enumerate(m):  # i: line number , content: content of each line
                if(i == 2):  # open txt and find content is in the 3rd line
                    words_in_email = set(content.split())
                    temp_train_10 = []
                    temp_train_100 = []
                    temp_train_1000 = []
                    temp_test_10 = []
                    temp_test_100 = []
                    temp_test_1000 = []

                    temp_train = []
                    temp_test = []
                    for w in dist_words:
                        if w in words_in_email:
                            if path[-2:] == '10' :
                                temp_test.append(1)
                            else:
                                temp_train.append(1)
                        else:
                            if path[-2:] == '10' :
                                temp_test.append(0)
                            else:
                                temp_train.append(0)
                    for w in f10:
                        if w in words_in_email:
                            if path[-2:] == '10' :
                                temp_test_10.append(1)
                            else:
                                temp_train_10.append(1)
                        else:
                            if path[-2:] == '10' :
                                temp_test_10.append(0)
                            else:
                                temp_train_10.append(0)
                    for w in f100:
                        if w in words_in_email:
                            if path[-2:] == '10' :
                                temp_test_100.append(1)
                            else:
                                temp_train_100.append(1)
                        else:
                            if path[-2:] == '10' :
                                temp_test_100.append(0)
                            else:
                                temp_train_100.append(0)
                    for w in f1000:
                        if w in words_in_email:
                            if path[-2:] == '10' :
                                temp_test_1000.append(1)
                            else:
                                temp_train_1000.append(1)
                        else:
                            if path[-2:] == '10' :
                                temp_test_1000.append(0)
                            else:
                                temp_train_1000.append(0)

        if path[-2:] == '10':
            test_data.append(np.array(temp_test))
            test_data_10.append(np.array(temp_test_10))
            test_data_100.append(np.array(temp_test_100))
            test_data_1000.append(np.array(temp_test_1000))
        else :
            train_data.append(np.array(temp_train))
            train_data_10.append(np.array(temp_train_10))
            train_data_100.append(np.array(temp_train_100))
            train_data_1000.append(np.array(temp_train_1000))

"""*Memo for an exception : when I try to fit for SVM, it says "setting an array element with a sequence" which indicates my matrix is not a generalized "rectangular", then I found I mistakenly wrote temp_train_10.append() in a top-1000 situation(part10).*

**ALL feature**

*Memo: Be carefull with strength of regularization,  make it soft because what I have is 0-1 feature!*
"""

# Commented out IPython magic to ensure Python compatibility.
from sklearn import svm
svm_ = svm.SVC(kernel = 'rbf',C=1000,probability=False) # because it is 0-1 feature, no need for strength on regularization
# %time svm_.fit(train_data,train_label)

pre_svm = svm_.predict(test_data)
print(pre_svm)

total = len(test_label)
# correct rate
score = svm_.score(test_data, test_label)
print("correct rate: ",score)

# spam precision is defined as the fraction of true spam e-mails among all e-mails predicted as spam
total_pre_spam = 0
true_spam_in_spam = 0
for i in range(total):
    if pre_svm[i] == 1:
        total_pre_spam += 1
        if test_label[i] == 1:
            true_spam_in_spam += 1
spam_precision_svm_af = true_spam_in_spam / (total_pre_spam) 
print("spam precision: ",spam_precision_svm_af)

# spam recall is defined as fraction of true spam e-mails predicted as spam.
pre_spam_in_true = 0
total_true_spam = 0
for i in range(total):
    if test_label[i] == 1:
        total_true_spam += 1
        if pre_svm[i] == 1:
            pre_spam_in_true += 1
        
spam_recall_svm_af = pre_spam_in_true / total_true_spam  
print("spam_recall: ",spam_recall_svm_af)

"""**1. Top 10 feature**"""

# Commented out IPython magic to ensure Python compatibility.
from sklearn import svm
#svm_10 = svm.SVC(kernel = 'linear',C=1000,probability=False)
svm_10 = svm.SVC(kernel = 'rbf',C=1000,probability=False)
# %time svm_10.fit(train_data_10,train_label)

pre_svm_10 = svm_10.predict(test_data_10)
print(pre_svm_10)

total = len(test_label)
# correct rate
score_10 = svm_10.score(test_data_10, test_label)
print("correct rate: ",score_10)

# spam precision is defined as the fraction of true spam e-mails among all e-mails predicted as spam
total_pre_spam = 0
true_spam_in_spam = 0
for i in range(total):
    if pre_svm_10[i] == 1:
        total_pre_spam += 1
        if test_label[i] == 1:
            true_spam_in_spam += 1
spam_precision_svm_10 = true_spam_in_spam / (total_pre_spam) 
print("spam precision: ",spam_precision_svm_10)

# spam recall is defined as fraction of true spam e-mails predicted as spam.
pre_spam_in_true = 0
total_true_spam = 0
for i in range(total):
    if test_label[i] == 1:
        total_true_spam += 1
        if pre_svm_10[i] == 1:
            pre_spam_in_true += 1
        
spam_recall_svm_10 = pre_spam_in_true / total_true_spam  
print("spam_recall: ",spam_recall_svm_10)

"""**2. Top 100 feature**"""

# Commented out IPython magic to ensure Python compatibility.
from sklearn import svm
#svm_100 = svm.SVC(kernel = 'linear',C=1000,probability=False)
svm_100 = svm.SVC(kernel = 'rbf',C=1000,probability=False)
# %time svm_100.fit(train_data_100,train_label)

pre_svm_100 = svm_100.predict(test_data_100)
print(pre_svm_100)

total = len(test_label)
# correct rate
score_100 = svm_100.score(test_data_100, test_label)
print("correct rate: ",score_100)

# spam precision is defined as the fraction of true spam e-mails among all e-mails predicted as spam
total_pre_spam = 0
true_spam_in_spam = 0
for i in range(total):
    if pre_svm_100[i] == 1:
        total_pre_spam += 1
        if test_label[i] == 1:
            true_spam_in_spam += 1
spam_precision_svm_100 = true_spam_in_spam / (total_pre_spam) 
print("spam precision: ",spam_precision_svm_100)

# spam recall is defined as fraction of true spam e-mails predicted as spam.
pre_spam_in_true = 0
total_true_spam = 0
for i in range(total):
    if test_label[i] == 1:
        total_true_spam += 1
        if pre_svm_100[i] == 1:
            pre_spam_in_true += 1
        
spam_recall_svm_100 = pre_spam_in_true / total_true_spam  
print("spam_recall: ",spam_recall_svm_100)

"""**3. Top 1000 feature**"""

# Commented out IPython magic to ensure Python compatibility.
from sklearn import svm
#svm_1000 = svm.SVC(kernel = 'linear',C=1000,probability=False)
svm_1000 = svm.SVC(kernel = 'rbf',C=1000,probability=False)
# %time svm_1000.fit(train_data_1000,train_label)

pre_svm_1000 = svm_1000.predict(test_data_1000)
print(pre_svm_1000)

total = len(test_label)
# correct rate
score_1000 = svm_1000.score(test_data_1000, test_label)
print("correct rate: ",score_1000)

# spam precision is defined as the fraction of true spam e-mails among all e-mails predicted as spam
total_pre_spam = 0
true_spam_in_spam = 0
for i in range(total):
    if pre_svm_1000[i] == 1:
        total_pre_spam += 1
        if test_label[i] == 1:
            true_spam_in_spam += 1
spam_precision_svm_1000 = true_spam_in_spam / (total_pre_spam) 
print("spam precision: ",spam_precision_svm_1000)

# spam recall is defined as fraction of true spam e-mails predicted as spam.
pre_spam_in_true = 0
total_true_spam = 0
for i in range(total):
    if test_label[i] == 1:
        total_true_spam += 1
        if pre_svm_1000[i] == 1:
            pre_spam_in_true += 1
        
spam_recall_svm_1000 = pre_spam_in_true / total_true_spam  
print("spam_recall: ",spam_recall_svm_1000)

"""# Part4 Report: SVM spam precision and spam recall

I used SVM with RBF kernel, it perform obviously better on recall than linear kernel. And I set regularization very soft because I am using binary feature, it only has 0 or 1. 

For features, I used binary word feature, and select them by IG used before, I tried all feature and top-10, top-100, top-1000 from IG.

**It results into top-1000 with RBF kernel performs best in general, because its precision and recall rate are both high.**
"""

print(f"RBF kernal with all feature: precision = {spam_precision_svm_af}, recall = {spam_recall_svm_af}")
print(f"RBF kernal with top-10 feature: precision = {spam_precision_svm_10}, recall = {spam_recall_svm_10}")
print(f"RBF kernal with top-100 feature: precision = {spam_precision_svm_100}, recall = {spam_recall_svm_100}")
print(f"RBF kernal with top-1000 feature: precision = {spam_precision_svm_1000}, recall = {spam_recall_svm_1000}")

"""Some extra statistics when using linear kernel:

linear & top-10: precision = 0.88372, recall = 0.775510

linear & top-100: precision = 0.9743, recall = 0.775510

linear & top-1000: precision = 1.0, recall = 0.85714

# Adversarial Classification

baseline multinomial NB with binary feature and top-10 feature

**1. Before attacker's modification**
"""

N = 10
test_label = []  
f10_test_mbf = []  # test emails' feature matrix for top 10

emails = [os.path.join(test_path, file_name) for file_name in os.listdir(test_path)]  
for email in emails:
    # spam: 1, legit: 0
    if email.find('spmsg')!=-1:
        test_label.append(1)
    else:
        test_label.append(0)
    
    with open(email) as m:
        temp_f10_bf = []
        for i, content in enumerate(m):  
            if i==2:  # content at 3rd line
                words_in_email = set(content.split())
                for w in f10:
                    if w in words_in_email:
                        temp_f10_bf.append(1)
                    else:
                        temp_f10_bf.append(0)

        f10_test_mbf.append(temp_f10_bf)
        
# compute P(x|spam)
p_x_spam_10 = []
for case in f10_test_mbf:
    p = 1
    for i in range(N):
        if case[i] == 1:
            p *= (word_spam[f10[i]] + 1) / (spam + 2) 
        else:
            p *= 1 
    p_x_spam_10.append(p)    

# compute P(x|legit)
p_x_legit_10 = []
for case in f10_test_mbf:
    p = 1
    for i in range(N):
        if case[i] == 1:
            p *= (word_legit[f10[i]] + 1) / (legit + 2) 
        else:
            p *= 1 
    p_x_legit_10.append(p)    

# compute P(spam)
p_spam = spam/(spam+legit)

# compute P(legit)
p_legit = legit/(spam+legit) 

# compute P(x)
p_x_10 = []
for i in range(len(f10_test_mbf)):
    p = p_spam * p_x_spam_10[i] + p_legit * p_x_legit_10[i]
    p_x_10.append(p)
 
#print(len(p_x_spam_10))
#print(len(p_x_10))

f10_pre = []
p_spam_x_10 = []
p_legit_x_10 = []
for i in range(len(f10_test_mbf)):
    psx = p_x_spam_10[i] * p_spam / p_x_10[i]
    p_spam_x_10.append(psx)
    plx = p_x_legit_10[i] * p_legit / p_x_10[i]
    p_legit_x_10.append(plx)
    if psx >= plx :
        f10_pre.append(1)
    else:
        f10_pre.append(0)

# false negative
fn_before = 0
true_positive = 0
for i in range(total):
    if test_label[i] == 1 and f10_pre[i] == 1 :
        true_positive +=1
    if test_label[i] == 1 and f10_pre[i] == 0:
        fn_before+= 1
        
fnr_before = n_fn_before / (true_positive + n_fn_before)  
print("Flase negative rate before attacker's modification: ",fnr_before)

"""**2. After attacker's modification**

Compute P(Xi=xi|C=c)
"""

# compute P(x|spam)

p_1_spam_10 = []
p_0_spam_10 = []

for i in range(N):
    # 1_spam
    p = (word_spam[f10[i]] + 1) / (spam + 2) # with laplacian smoothing
    p_1_spam_10.append(p)
    # 0_spam
    p = 1 - (word_spam[f10[i]] + 1) / (spam + 2)   
    p_0_spam_10.append(p) 

# compute P(x|legit)
p_1_legit_10 = []
p_0_legit_10 = []
p = 1
for i in range(N):
    # 1_spam
    p = (word_legit[f10[i]] + 1) / (legit + 2) # with laplacian smoothing
    p_1_legit_10.append(p)
    # 0_spam
    p = 1 - (word_legit[f10[i]] + 1) / (legit + 2)
    p_0_legit_10.append(p)

"""Compute desired reduction log( P(spam | x) / P(legit | x) ) =  log( P(x | spam) / P(x | legit) ) + log(P(spam)/P(legit))"""

desired_delta = []
for case in f10_test_mbf:
    p = 1
    for i in range(N):
        if case[i] == 1:
            p *= p_1_spam_10[i] / p_1_legit_10[i]
        else:
            p *= p_0_spam_10[i] / p_0_legit_10[i]
    d = np.log2(p) + np.log2(spam/legit)
    desired_delta.append(d)

print("desired delta:",desired_delta[5:15])
print("corresponding label:",test_label[5:15])

"""compute  delta_LO_ix' = LOc(xi) - LOc(xi') for top-10 features

LO(xi) = log( P(xi | spam) / P(xi | legit) )

As assignment states, the adversary only use **ADD_WORDS** strategy, so I set reduction for term_i to 0 when each email has this term

Only consider adding word for reduction of each feature
"""

delta_add_word = []
for i in range(10):
    d = np.log2(p_0_spam_10[i] / p_0_legit_10[i]) - np.log2(p_1_spam_10[i] / p_1_legit_10[i]) # delta_LO_ix' = LOc(xi) - LOc(xi') old - new
    d = max(0,d)
    delta_add_word.append(d)

"""determine reduction acquired and **sort them** by value, now we can see top-10words in this sorted order"""

sorted_delta = []
for case in f10_test_mbf:
    delta_i = {}
    for i in range(10):
        if case[i] == 1:
            delta_i[f10[i]] = 0
        else:
            delta_i[f10[i]] = delta_add_word[i]
    sorted_delta_i = OrderedDict(sorted(delta_i.items(), key=lambda x : x[1], reverse=True))
    sorted_delta.append(sorted_delta_i)

#print("check f_index helper: index of remove",f_index["remove"])
print("orgin f10 sequence: ", f10)
#print("case 0:", f10_test_mbf[0])
print("desired delta of case 0: ",desired_delta[0])
for (w,d) in sorted_delta[0].items():
    print(w,":", d)

"""Because adding a term incur an unit cost, we can pick term with larger reduction first (*being greedy*) for minimum cost tring to meet desired reduction!

**Start modify**
"""

# feature index helper
f_index = {}
for i in range(10):
    f_index[f10[i]]=i

avg_cost = 0
test_spam = 0
f10_test_mbf_mod = []
for i in f10_test_mbf:
    f10_test_mbf_mod.append(i.copy())

for i in range(len(sorted_delta)):
    if test_label[i] == 0: # do not modify legit email
        continue
    else:
        test_spam+=1
    d = sorted_delta[i]
    sum = 0
    for (k,v) in d.items():
        if v <= 0: #if value reach 0 in sorted reduction values, can't reach desired reduction, break
            break
        else: # reduction > 0
            f10_test_mbf_mod[i][f_index[k]] = 1 # add word
            avg_cost+=1
            sum += v
            if desired_delta[i] <= sum: # meet desired reduction 
                break; 
avg_cost /= test_spam
print("average cost of attacker: ", avg_cost)

print(f10)
print("origin:",f10_test_mbf[0])
print("modify:",f10_test_mbf_mod[0])

"""perform classification after attacker's modification on test data"""

# compute P(x|spam)
p_x_spam_10 = []
for case in f10_test_mbf_mod:
    p = 1
    for i in range(N):
        if case[i] == 1:
            p *= (word_spam[f10[i]] + 1) / (spam + 2) # with laplacian smoothing
        else:
            p *= 1 
    p_x_spam_10.append(p)    

# compute P(x|legit)
p_x_legit_10 = []
for case in f10_test_mbf_mod:
    p = 1
    for i in range(N):
        if case[i] == 1:
            p *= (word_legit[f10[i]] + 1) / (legit + 2) # with laplacian smoothing
        else:
            p *= 1 
    p_x_legit_10.append(p)    

# compute P(x)
p_x_10 = []
for i in range(len(f10_test_mbf_mod)):
    p = p_spam * p_x_spam_10[i] + p_legit * p_x_legit_10[i]
    p_x_10.append(p)

f10_mod_pre = []
p_spam_x_10 = []
p_legit_x_10 = []
for i in range(len(f10_test_mbf_mod)):
    psx = p_x_spam_10[i] * p_spam / p_x_10[i]
    p_spam_x_10.append(psx)
    plx = p_x_legit_10[i] * p_legit / p_x_10[i]
    p_legit_x_10.append(plx)
    if psx >= plx :
        f10_mod_pre.append(1)
    else:
        f10_mod_pre.append(0)

# false negative
fn_after = 0
true_positive = 0
for i in range(total):
    if test_label[i] == 1 and f10_mod_pre[i] == 1 :
        true_positive +=1
    if test_label[i] == 1 and f10_mod_pre[i] == 0:
        fn_after+= 1
        
fnr_after = fn_after / (true_positive + fn_after)  
print("Flase negative rate after attacker's modification: ",fnr_after)

"""**3. After Update calssifier**

PA{spam|x'}/PA{legit|x'} = PA{x'|spam}*PA{SPAM} / (PA{x'|legit} * PA{legit})
"""

# # compute PA(x'|spam), x' can be generated by any possible x (ADD WORDS)so it is sum of P(x|spam)
# p_x_spam_10 = []
# for CASE in f10_test_mbf_mod: # facing attack
#     sum = 0
#     for case in f10_test_mbf_mod:
#         # can contain it self I think, or there might be zero availble x
#         if case == CASE:
#             continue
#         candidate = True
#         count_add = 0
#         for i in range(len(case)):
#             # because we assume classifier knows adversary will add word in a optimal way, so according to average cost of attack, I set limit to at most 1 word to add
#             if count_add > 1:  
#                 candidate = False
#             if CASE[i] == 0 and case[i] == 1:
#                 candidate = False
#             else: # can ADD WORDS to case to get CASE
#                 if CASE[i] == 1 and case[i] == 0:
#                     count_add+=1
#         p = 0
#         if candidate == True:
#             p = 1
#             for i in range(N):
#                 if case[i] == 1:
#                     p *= (word_spam[f10[i]] + 1) / (spam + 2) 
#                 else:
#                     p *= 1   
#         sum += p
#     p_x_spam_10.append(sum)    

# # compute PA(x'|legit)
# p_x_legit_10 = []
# for case in f10_test_mbf_mod:
#     p = 1
#     for i in range(N):
#         if case[i] == 1:
#             p *= (word_legit[f10[i]] + 1) / (legit + 2) 
#         else:
#             p *= 1
#     p_x_legit_10.append(p)    



# f10_update_pre = []

# for i in range(len(f10_test_mbf_mod)):
#     psx = p_x_spam_10[i] * p_spam 
#     p_spam_x_10.append(psx)
#     plx = p_x_legit_10[i] * p_legit 
#     p_legit_x_10.append(plx)
#     if psx / plx >=1 :
#         f10_update_pre.append(1)
#     else:
#         f10_update_pre.append(0)

# compute PA(x'|legit)
p_x_legit_10 = []
for case in f10_test_mbf_mod:
    p = 1
    for i in range(N):
        if case[i] == 1:
            p *= (word_legit[f10[i]] + 1) / (legit + 2) 
        else:
            p *= 1
    p_x_legit_10.append(p) 

# assume classifier know attack strategy: always add words with minimum cost
# compute PA(x'|spam), x' can be generated by any possible x (ADD WORDS), so it is sum of P(x|spam)
p_x_spam_10 = []
for i in range(len(f10_test_mbf_mod)):
    case = f10_test_mbf_mod[i] 
    sum = 0
    for w in sorted_delta[i].keys(): # according to sorted delta(reduction), inversely find optimal original x from x' by remove word with larger delta (greedy)
        if case[f_index[w]] == 1: # can remove word
            case[f_index[w]] = 0  # remove
            psx = 1
            plx = 1
            for i in range(N):
                if case[i] == 1:
                    psx *= (word_spam[f10[i]] + 1) / (spam + 2) 
                else:
                    psx *= 1
                if case[i] == 1:
                    plx *= (word_legit[f10[i]] + 1) / (legit + 2) 
                else:
                    plx *= 1
            if psx*p_spam / (plx*p_legit) >= 1: # the original one inversly recover from modified case is spam! we find attacker's original case to modify
                sum+=psx
                break
            else:
                 continue
    p_x_spam_10.append(sum)

f10_update_pre = []

for i in range(len(f10_test_mbf_mod)):
    psx = p_x_spam_10[i] * p_spam 
    p_spam_x_10.append(psx)
    plx = p_x_legit_10[i] * p_legit 
    p_legit_x_10.append(plx)
    if psx / plx >=1 :
        f10_update_pre.append(1)
    else:
        f10_update_pre.append(0)

# false negative
fn_update = 0
true_positive = 0
for i in range(total):
    if test_label[i] == 1 and f10_update_pre[i] == 1 :
        true_positive +=1
    if test_label[i] == 1 and f10_update_pre[i] == 0:
        fn_update+= 1
        
fnr_update = fn_update / (true_positive + fn_update)  
print("Flase negative rate after update classifier: ",fnr_update) 

# false positive
fp_update = 0
true_negative = 0
for i in range(total):
    if test_label[i] == 0 and f10_update_pre[i] == 0 :
        true_negative +=1
    if test_label[i] == 0 and f10_update_pre[i] == 1:
        fp_update+= 1
        
fpr_update = fp_update / (true_negative + fp_update)  
print("Flase positive rate after update classifier: ",fpr_update)

"""# Part4 report: adversarial classification"""

print("Flase negative rate before attacker's modification: ",fnr_before) 
print("Flase negative rate after attacker's modification: ",fnr_after)
print("average cost of attacker: ", avg_cost)  
print("Flase negative rate after update classifier: ",fnr_update)   
print("Flase positive rate after update classifier: ",fpr_update)
# -*- coding: utf-8 -*-
"""MLCS_LAB1.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1KnN9oHgEbNbMZg8DooJNOqvXTWeBTj92

Upload dataset and unzip the file
"""

!tar xzf lingspam_public.tar.gz

"""Import library"""

from decimal import *
import numpy as np
import matplotlib.pyplot as plt
import math
import os
from collections import Counter
from collections import OrderedDict
from sklearn import svm

"""Load words in emails into a word list."""

train_path = './lingspam_public/lemm_stop'  # parent directory of dataset we use
paths = [os.path.join(train_path, folders) for folders in os.listdir(train_path)]  
all_words = [] 
labels = []  # 1 for spam, 0 for legit
test_path = ""
for path in paths:
    if(path[-2:] == '10'):  # path[-2:] check if this is part10 which is used for testing, skip 
        test_path = path
        continue
    files = [os.path.join(path, f) for f in os.listdir(path)]  # get file names of this part and generate path to files
    for email in files:
        if(email.find('spmsg') != -1): #check filenames to check if it is spam 
            labels.append(1)
        else:
            labels.append(0)
        
        with open(email) as m:
            for i, content in enumerate(m):  # i: line number , content: content of each line
                if(i == 2):  # open txt and find content is in the 3rd line
                    words = content.split()
                    all_words += words

word_dict = Counter(all_words)  

# remove symbols which have no big relationship with spam or legit
for word in list(word_dict.keys()):
    if(word.isalpha() == False):
        del word_dict[word]

# distinct words
dist_words = list(word_dict.keys())

"""# Compute Information Gain(IG)

1. compute H(C), C is RV that determins if a document is spam or legit
"""

spam = 0
legit = 0
for label in labels:
  if label == 1:
    spam += 1
  else:
    legit += 1  

print(f"spam email: {spam}, legit email: {legit}, total email {spam + legit}")
p_spam = spam / (spam + legit)

H_c = -1 * ( p_spam*np.log2(p_spam) + (1-p_spam)*np.log2(1-p_spam) )
print('H(C): ', H_c)

"""2. Compute H(C|Xi), need to create dictionary for word:#spam and word:#legit to help computing H(C|Xi)"""

word_spam = {}
word_legit = {}
for word in dist_words:
    word_spam[word] = 0
    word_legit[word] = 0

for path in paths:
    if(path[-2:] == '10'):  
        continue
    files = [os.path.join(path, f) for f in os.listdir(path)]  
    for email in files:
        if(email.find('spmsg') != -1):  
            is_spam = True
        else:
            is_spam = False
        dist_set = set() # there could be repeated word, we only need  occurance, use set
        with open(email) as m:
            for i, content in enumerate(m):  
                if(i == 2):  
                    words = content.split()
                    for word in words:
                        if(word.isalpha() == False): # skip symbols
                            continue
                        else:
                            dist_set.add(word)
        for e in dist_set:
            if(is_spam == True):
                word_spam[e] += 1
            else:
                word_legit[e] += 1

"""Compute H(C|Xi) = "combination" of P(X=x,C=c)log(P(C=c|X=x))"""

H_cx = {}
for word in dist_words:
    H_cx[word] = 0 
    # c = legit, x = 0 
    if ((legit - word_legit[word]) != 0 and (spam - word_spam[word] + legit - word_legit[word]) != 0): #sum only when non-zero probability
        H_cx[word] += -1 * (legit - word_legit[word]) / (spam + legit) * np.log2( (legit - word_legit[word]) / (spam - word_spam[word] + legit - word_legit[word]) )
    # c = legit, x = 1 
    if ((word_legit[word]) != 0 and (word_spam[word] + word_legit[word]) != 0):
        H_cx[word] += -1 * (word_legit[word]) / (spam + legit) * np.log2( word_legit[word] / (word_spam[word] + word_legit[word]) )
    # c = spam, x = 0 
    if (spam-word_spam[word] != 0 and (spam - word_spam[word] + legit - word_legit[word]) != 0):
        H_cx[word] += -1 * (spam - word_spam[word]) / (spam + legit) * np.log2( (spam - word_spam[word] ) / (spam - word_spam[word] + legit - word_legit[word]) ) 
    # c = spam, x = 1 
    if (word_spam[word] != 0 and word_spam[word] + word_legit[word] != 0):
        H_cx[word] += -1 * (word_spam[word]) / (spam + legit) * np.log2( word_spam[word] / (word_spam[word] + word_legit[word]) )

"""3. Compute IG"""

IG = {}
for word in list(H_cx.keys()):
    IG[word] = H_c - H_cx[word]

"""# Get top N features"""

# sort by value
# sorted can be applied to iterable containers
# sorted return a new result rather than sort in place
# sorted return a list object not a sorted dictionary!!!!!!!
# sorted_IG = sorted(IG.items(), key=lambda x : x[1], reverse=True)
sorted_IG = OrderedDict(sorted(IG.items(), key=lambda x : x[1], reverse=True))
f10 = list(sorted_IG.keys())[:10]  # top 10
f100 = list(sorted_IG.keys())[:100]  # top 100
f1000 = list(sorted_IG.keys())[:1000]  # top 1000

"""# Part1 report: top-10 words:"""

print("Top-10 words:")
c=0
for i in sorted_IG.items():
  c+=1
  if c > 10:
    break
  print(i[0],":",i[1])

"""# Bernoulli NB classifier with binary feature

*Load test data*
"""

test_label = []  
f10_test_bbf = []  # test emails' feature matrix for top 10
f100_test_bbf = []  # test emails' feature matrix for top 100
f1000_test_bbf = []  # test emails' feature matrix for top 1000
emails = [os.path.join(test_path, file_name) for file_name in os.listdir(test_path)]  
for email in emails:
    # spam: 1, legit: 0
    if email.find('spmsg')!=-1:
        test_label.append(1)
    else:
        test_label.append(0)
    
    with open(email) as m:
        temp_f10_bf = []
        temp_f100_bf = []
        temp_f1000_bf = []

        for i, content in enumerate(m):  
            if i==2:  # content at 3rd line
                words_in_email = set(content.split())
                for w in f10:
                    if w in words_in_email:
                        temp_f10_bf.append(1)
                    else:
                        temp_f10_bf.append(0)
                for w in f100:
                    if w in words_in_email:
                        temp_f100_bf.append(1)
                    else:
                        temp_f100_bf.append(0)
                for w in f1000:
                    if w in words_in_email:
                        temp_f1000_bf.append(1)
                    else:
                        temp_f1000_bf.append(0)

        f10_test_bbf.append(temp_f10_bf)
        f100_test_bbf.append(temp_f100_bf)
        f1000_test_bbf.append(temp_f1000_bf)

"""**1. top 10 feature**"""

N = 10

"""1.1 Compute probability"""

# x is conbination of words
# compute P(x|spam)
p_x_spam_10 = []
for case in f10_test_bbf:
    p = 1
    for i in range(N):
        if case[i] == 1:
            p *= (word_spam[f10[i]] + 1) / (spam + 2) # with laplacian smoothing
        else:
            p *= (1 - (word_spam[f10[i]] + 1) / (spam + 2)) # with laplacian smoothing
    p_x_spam_10.append(p)    

# compute P(x|legit)
p_x_legit_10 = []
for case in f10_test_bbf:
    p = 1
    for i in range(N):
        if case[i] == 1:
            p *= (word_legit[f10[i]] + 1) / (legit + 2) # with laplacian smoothing
        else:
            p *= 1 - (word_legit[f10[i]] + 1) / (legit + 2) # with laplacian smoothing
    p_x_legit_10.append(p)    

# compute P(spam)
p_spam = spam/(spam+legit)

# compute P(legit)
p_legit = legit/(spam+legit) 

# compute P(x)
p_x_10 = []
for i in range(len(f10_test_bbf)):
    p = p_spam * p_x_spam_10[i] + p_legit * p_x_legit_10[i]
    p_x_10.append(p)

"""1.2 Classify by computing P(spam|X) and P(legit|X) and compare"""

f10_pre = []
p_spam_x_10 = []
p_legit_x_10 = []
for i in range(len(f10_test_bbf)):
    psx = p_x_spam_10[i] * p_spam / p_x_10[i]
    p_spam_x_10.append(psx)
    plx = p_x_legit_10[i] * p_legit / p_x_10[i]
    p_legit_x_10.append(plx)
    if psx >= plx :
        f10_pre.append(1)
    else:
        f10_pre.append(0)

"""1.3 Report result"""

test_label[:10]

f10_pre[:10]

#p_spam_x_10[:10]

#p_legit_x_10[:10]

correct = 0;
total = len(test_label)
for i in range(total):
    if test_label[i] == f10_pre[i]:
        correct+=1
correct_rate = correct / total
print("correct rate: ",correct_rate)

# spam precision is defined as the fraction of true spam e-mails among all e-mails predicted as spam
total_pre_spam = 0
true_spam_in_spam = 0
for i in range(total):
    if f10_pre[i] == 1:
        total_pre_spam += 1
        if test_label[i] == 1:
            true_spam_in_spam += 1
spam_precision_bbf_10 = true_spam_in_spam / total_pre_spam   
print("spam precision: ",spam_precision_bbf_10)

# spam recall is defined as fraction of true spam e-mails predicted as spam.
pre_spam_in_true = 0
total_true_spam = 0
for i in range(total):
    if test_label[i] == 1:
        total_true_spam += 1
        if f10_pre[i] == 1:
            pre_spam_in_true += 1
        
spam_recall_bbf_10 = pre_spam_in_true / total_true_spam  
print("spam_recall: ",spam_recall_bbf_10)

"""**2. top 100 feature**"""

N = 100

"""2.1 Compute probability"""

# x is conbination of words
# compute P(x|spam)
p_x_spam_100 = []
for case in f100_test_bbf:
    p = 1
    for i in range(N):
        if case[i] == 1:
            p *= (word_spam[f100[i]] + 1) / (spam + 2) # with laplacian smoothing
        else:
            p *= (1 - (word_spam[f100[i]] + 1) / (spam + 2)) # with laplacian smoothing
    p_x_spam_100.append(p)    

# compute P(x|legit)
p_x_legit_100 = []
for case in f100_test_bbf:
    p = 1
    for i in range(N):
        if case[i] == 1:
            p *= (word_legit[f100[i]] + 1) / (legit + 2) # with laplacian smoothing
        else:
            p *= 1 - (word_legit[f100[i]] + 1) / (legit + 2) # with laplacian smoothing
    p_x_legit_100.append(p)    

# compute P(spam)
p_spam = spam/(spam+legit)

# compute P(legit)
p_legit = legit/(spam+legit) 

# compute P(x)
p_x_100 = []
for i in range(len(f100_test_bbf)):
    p = p_spam * p_x_spam_100[i] + p_legit * p_x_legit_100[i]
    p_x_100.append(p)
 
#print(len(p_x_spam_100))
#print(len(p_x_100))

"""2.2 Classify by computing P(spam|X) and P(legit|X) and compare"""

f100_pre = []
p_spam_x_100 = []
p_legit_x_100 = []
for i in range(len(f100_test_bbf)):
    psx = p_x_spam_100[i] * p_spam / p_x_100[i]
    p_spam_x_100.append(psx)
    plx = p_x_legit_100[i] * p_legit / p_x_100[i]
    p_legit_x_100.append(plx)
    if psx >= plx :
        f100_pre.append(1)
    else:
        f100_pre.append(0)

"""2.3 Report result"""

test_label[:10]

f100_pre[:10]

correct = 0;
total = len(test_label)
for i in range(total):
    if test_label[i] == f100_pre[i]:
        correct+=1
correct_rate = correct / total
print("correct rate: ",correct_rate)

# spam precision is defined as the fraction of true spam e-mails among all e-mails predicted as spam
total_pre_spam = 0
true_spam_in_spam = 0
for i in range(total):
    if f100_pre[i] == 1:
        total_pre_spam += 1
        if test_label[i] == 1:
            true_spam_in_spam += 1
spam_precision_bbf_100 = true_spam_in_spam / total_pre_spam   
print("spam precision: ",spam_precision_bbf_100)

# spam recall is defined as fraction of true spam e-mails predicted as spam.
pre_spam_in_true = 0
total_true_spam = 0
for i in range(total):
    if test_label[i] == 1:
        total_true_spam += 1
        if f100_pre[i] == 1:
            pre_spam_in_true += 1
        
spam_recall_bbf_100 = pre_spam_in_true / total_true_spam  
print("spam_recall: ",spam_recall_bbf_100)

"""3. top 100 feature"""

N = 1000

"""**3.1 Compute probability**"""

# x is conbination of words
# compute P(x|spam)
p_x_spam_1000 = []
for case in f1000_test_bbf:
    p = 1
    for i in range(N):
        if case[i] == 1:
            p *= (word_spam[f1000[i]] + 1) / (spam + 2) # with laplacian smoothing
        else:
            p *= (1 - (word_spam[f1000[i]] + 1) / (spam + 2)) # with laplacian smoothing
    p_x_spam_1000.append(p)    

# compute P(x|legit)
p_x_legit_1000 = []
for case in f1000_test_bbf:
    p = 1
    for i in range(N):
        if case[i] == 1:
            p *= (word_legit[f1000[i]] + 1) / (legit + 2) # with laplacian smoothing
        else:
            p *= 1 - (word_legit[f1000[i]] + 1) / (legit + 2) # with laplacian smoothing
    p_x_legit_1000.append(p)    

# compute P(spam)
p_spam = spam/(spam+legit)

# compute P(legit)
p_legit = legit/(spam+legit) 

# compute P(x)
p_x_1000 = []
for i in range(len(f1000_test_bbf)):
    p = p_spam * p_x_spam_1000[i] + p_legit * p_x_legit_1000[i]
    p_x_1000.append(p)
 
#print(len(p_x_spam_1000))
#print(len(p_x_1000))

"""3.2 Classify by computing P(spam|X) and P(legit|X) and compare"""

f1000_pre = []
p_spam_x_1000 = []
p_legit_x_1000 = []
for i in range(len(f1000_test_bbf)):
    psx = p_x_spam_1000[i] * p_spam / p_x_1000[i]
    p_spam_x_1000.append(psx)
    plx = p_x_legit_1000[i] * p_legit / p_x_1000[i]
    p_legit_x_1000.append(plx)
    if psx >= plx :
        f1000_pre.append(1)
    else:
        f1000_pre.append(0)

"""3.3 Report result"""

test_label[:10]

f1000_pre[:10]

correct = 0;
total = len(test_label)
for i in range(total):
    if test_label[i] == f1000_pre[i]:
        correct+=1
correct_rate = correct / total
print("correct rate: ",correct_rate)

# spam precision is defined as the fraction of true spam e-mails among all e-mails predicted as spam
total_pre_spam = 0
true_spam_in_spam = 0
for i in range(total):
    if f1000_pre[i] == 1:
        total_pre_spam += 1
        if test_label[i] == 1:
            true_spam_in_spam += 1
spam_precision_bbf_1000 = true_spam_in_spam / total_pre_spam   
print("spam precision: ",spam_precision_bbf_1000)

# spam recall is defined as fraction of true spam e-mails predicted as spam.
pre_spam_in_true = 0
total_true_spam = 0
for i in range(total):
    if test_label[i] == 1:
        total_true_spam += 1
        if f1000_pre[i] == 1:
            pre_spam_in_true += 1
        
spam_recall_bbf_1000 = pre_spam_in_true / total_true_spam  
print("spam_recall: ",spam_recall_bbf_1000)

"""***It seems that spam precision goes up while spam recall goes down with more feature selected***

# Multinomial NB with binary features

*Load test data*
"""

test_label = []  
f10_test_mbf = []  # test emails' feature matrix for top 10
f100_test_mbf = []  # test emails' feature matrix for top 100
f1000_test_mbf = []  # test emails' feature matrix for top 1000
emails = [os.path.join(test_path, file_name) for file_name in os.listdir(test_path)]  
for email in emails:
    # spam: 1, legit: 0
    if email.find('spmsg')!=-1:
        test_label.append(1)
    else:
        test_label.append(0)
    
    with open(email) as m:
        temp_f10_bf = []
        temp_f100_bf = []
        temp_f1000_bf = []

        for i, content in enumerate(m):  
            if i==2:  # content at 3rd line
                words_in_email = set(content.split())
                for w in f10:
                    if w in words_in_email:
                        temp_f10_bf.append(1)
                    else:
                        temp_f10_bf.append(0)
                for w in f100:
                    if w in words_in_email:
                        temp_f100_bf.append(1)
                    else:
                        temp_f100_bf.append(0)
                for w in f1000:
                    if w in words_in_email:
                        temp_f1000_bf.append(1)
                    else:
                        temp_f1000_bf.append(0)

        f10_test_mbf.append(temp_f10_bf)
        f100_test_mbf.append(temp_f100_bf)
        f1000_test_mbf.append(temp_f1000_bf)

"""**1. top 10 feature**"""

N = 10

"""1.1 Compute probability"""

# compute P(x|spam)
p_x_spam_10 = []
for case in f10_test_mbf:
    p = 1
    for i in range(N):
        if case[i] == 1:
            p *= (word_spam[f10[i]] + 1) / (spam + 2) # with laplacian smoothing
        else:
            p *= 1 # different from bernoulli: ignore negative evidence
    p_x_spam_10.append(p)    

# compute P(x|legit)
p_x_legit_10 = []
for case in f10_test_mbf:
    p = 1
    for i in range(N):
        if case[i] == 1:
            p *= (word_legit[f10[i]] + 1) / (legit + 2) # with laplacian smoothing
        else:
            p *= 1 # different from bernoulli: ignore negative evidence
    p_x_legit_10.append(p)    

# compute P(spam)
p_spam = spam/(spam+legit)

# compute P(legit)
p_legit = legit/(spam+legit) 

# compute P(x)
p_x_10 = []
for i in range(len(f10_test_mbf)):
    p = p_spam * p_x_spam_10[i] + p_legit * p_x_legit_10[i]
    p_x_10.append(p)
 
#print(len(p_x_spam_10))
#print(len(p_x_10))

"""1.2 Classify by computing P(spam|X) and P(legit|X) and compare"""

f10_pre = []
p_spam_x_10 = []
p_legit_x_10 = []
for i in range(len(f10_test_mbf)):
    psx = p_x_spam_10[i] * p_spam / p_x_10[i]
    p_spam_x_10.append(psx)
    plx = p_x_legit_10[i] * p_legit / p_x_10[i]
    p_legit_x_10.append(plx)
    if psx >= plx :
        f10_pre.append(1)
    else:
        f10_pre.append(0)

"""1.3 report result"""

correct = 0;
total = len(test_label)
for i in range(total):
    if test_label[i] == f10_pre[i]:
        correct+=1
correct_rate = correct / total
print("correct rate: ",correct_rate)

# spam precision is defined as the fraction of true spam e-mails among all e-mails predicted as spam
total_pre_spam = 0
true_spam_in_spam = 0
for i in range(total):
    if f10_pre[i] == 1:
        total_pre_spam += 1
        if test_label[i] == 1:
            true_spam_in_spam += 1
spam_precision_mbf_10 = true_spam_in_spam / total_pre_spam   
print("spam precision: ",spam_precision_mbf_10)

# spam recall is defined as fraction of true spam e-mails predicted as spam.
pre_spam_in_true = 0
total_true_spam = 0
for i in range(total):
    if test_label[i] == 1:
        total_true_spam += 1
        if f10_pre[i] == 1:
            pre_spam_in_true += 1
        
spam_recall_mbf_10 = pre_spam_in_true / total_true_spam  
print("spam_recall: ",spam_recall_mbf_10)

"""**2. top 100 feature**"""

N = 100

"""2.1 Compute probability"""

# compute P(x|spam)
p_x_spam_100 = []
for case in f100_test_mbf:
    p = 1
    for i in range(N):
        if case[i] == 1:
            p *= (word_spam[f100[i]] + 1) / (spam + 2) # with laplacian smoothing
        else:
            p *= 1 # different from bernoulli: ignore negative evidence 
    p_x_spam_100.append(p)    

# compute P(x|legit)
p_x_legit_100 = []
for case in f100_test_mbf:
    p = 1
    for i in range(N):
        if case[i] == 1:
            p *= (word_legit[f100[i]] + 1) / (legit + 2) # with laplacian smoothing
        else:
            p *= 1 # different from bernoulli: ignore negative evidence
    p_x_legit_100.append(p)    

# compute P(spam)
p_spam = spam/(spam+legit)

# compute P(legit)
p_legit = legit/(spam+legit) 

# compute P(x)
p_x_100 = []
for i in range(len(f100_test_mbf)):
    p = p_spam * p_x_spam_100[i] + p_legit * p_x_legit_100[i]
    p_x_100.append(p)
 
#print(len(p_x_spam_100))
#print(len(p_x_100))

"""2.2 Classify by computing P(spam|X) and P(legit|X) and compare"""

f100_pre = []
p_spam_x_100 = []
p_legit_x_100 = []
for i in range(len(f100_test_mbf)):
    psx = p_x_spam_100[i] * p_spam / p_x_100[i]
    p_spam_x_10.append(psx)
    plx = p_x_legit_100[i] * p_legit / p_x_100[i]
    p_legit_x_100.append(plx)
    if psx >= plx :
        f100_pre.append(1)
    else:
        f100_pre.append(0)

"""2.3 Report result"""

correct = 0;
total = len(test_label)
for i in range(total):
    if test_label[i] == f100_pre[i]:
        correct+=1
correct_rate = correct / total
print("correct rate: ",correct_rate)

# spam precision is defined as the fraction of true spam e-mails among all e-mails predicted as spam
total_pre_spam = 0
true_spam_in_spam = 0
for i in range(total):
    if f100_pre[i] == 1:
        total_pre_spam += 1
        if test_label[i] == 1:
            true_spam_in_spam += 1
spam_precision_mbf_100 = true_spam_in_spam / total_pre_spam   
print("spam precision: ",spam_precision_mbf_100)

# spam recall is defined as fraction of true spam e-mails predicted as spam.
pre_spam_in_true = 0
total_true_spam = 0
for i in range(total):
    if test_label[i] == 1:
        total_true_spam += 1
        if f100_pre[i] == 1:
            pre_spam_in_true += 1
        
spam_recall_mbf_100 = pre_spam_in_true / total_true_spam  
print("spam_recall: ",spam_recall_mbf_100)

"""**3. top 1000 feature**"""

N = 1000

"""3.1 Compute probability"""

# compute P(x|spam)
p_x_spam_1000 = []
for case in f1000_test_mbf:
    p = 1
    for i in range(N):
        if case[i] == 1:
            p *= (word_spam[f1000[i]] + 1) / (spam + 2) # with laplacian smoothing
        else:
            p *= 1 # different from bernoulli: ignore negative evidence 
    p_x_spam_1000.append(p)    

# compute P(x|legit)
p_x_legit_1000 = []
for case in f1000_test_mbf:
    p = 1
    for i in range(N):
        if case[i] == 1:
            p *= (word_legit[f1000[i]] + 1) / (legit + 2) # with laplacian smoothing
        else:
            p *= 1 # different from bernoulli: ignore negative evidence
    p_x_legit_1000.append(p)    

# compute P(spam)
p_spam = spam/(spam+legit)

# compute P(legit)
p_legit = legit/(spam+legit) 

# compute P(x)
p_x_1000 = []
for i in range(len(f1000_test_mbf)):
    p = p_spam * p_x_spam_1000[i] + p_legit * p_x_legit_1000[i]
    p_x_1000.append(p)
 
#print(len(p_x_spam_1000))
#print(len(p_x_1000))

"""3.2 Classify by computing P(spam|X) and P(legit|X) and compare"""

f1000_pre = []
p_spam_x_1000 = []
p_legit_x_1000 = []
for i in range(len(f1000_test_mbf)):
    psx = p_x_spam_1000[i] * p_spam / p_x_1000[i]
    p_spam_x_1000.append(psx)
    plx = p_x_legit_1000[i] * p_legit / p_x_1000[i]
    p_legit_x_1000.append(plx)
    if psx >= plx :
        f1000_pre.append(1)
    else:
        f1000_pre.append(0)

"""3.3 Report result"""

correct = 0;
total = len(test_label)
for i in range(total):
    if test_label[i] == f1000_pre[i]:
        correct+=1
correct_rate = correct / total
print("correct rate: ",correct_rate)

# spam precision is defined as the fraction of true spam e-mails among all e-mails predicted as spam
total_pre_spam = 0
true_spam_in_spam = 0
for i in range(total):
    if f1000_pre[i] == 1:
        total_pre_spam += 1
        if test_label[i] == 1:
            true_spam_in_spam += 1
spam_precision_mbf_1000 = true_spam_in_spam / total_pre_spam   
print("spam precision: ",spam_precision_mbf_1000)

# spam recall is defined as fraction of true spam e-mails predicted as spam.
pre_spam_in_true = 0
total_true_spam = 0
for i in range(total):
    if test_label[i] == 1:
        total_true_spam += 1
        if f1000_pre[i] == 1:
            pre_spam_in_true += 1
        
spam_recall_mbf_1000 = pre_spam_in_true / total_true_spam  
print("spam_recall: ",spam_recall_mbf_1000)

"""***Adding feature seems helps multinomial NB with binary feature positively***

# Multinomial NB with term frequency (TF) features

*Load test data*
"""

test_label = []  
f10_test_mtf = []  # test emails' feature matrix for top 10
f100_test_mtf = []  # test emails' feature matrix for top 100
f1000_test_mtf = []  # test emails' feature matrix for top 1000
emails = [os.path.join(test_path, file_name) for file_name in os.listdir(test_path)]  
for email in emails:
    # spam: 1, legit: 0
    if email.find('spmsg')!=-1:
        test_label.append(1)
    else:
        test_label.append(0)
    
    with open(email) as m:
        temp_f10_tf = []
        temp_f100_tf = []
        temp_f1000_tf = []

        for i, content in enumerate(m):  
            if i==2:  # content at 3rd line
                words_in_email = set(content.split())
                for w in f10:
                    if w in words_in_email:
                        temp_f10_tf.append(1)
                    else:
                        temp_f10_tf.append(0)
                for w in f100:
                    if w in words_in_email:
                        temp_f100_tf.append(1)
                    else:
                        temp_f100_tf.append(0)
                for w in f1000:
                    if w in words_in_email:
                        temp_f1000_tf.append(1)
                    else:
                        temp_f1000_tf.append(0)

        f10_test_mtf.append(temp_f10_tf)
        f100_test_mtf.append(temp_f100_tf)
        f1000_test_mtf.append(temp_f1000_tf)

"""*For Term Frequency we need new dictionary for words*"""

# occurence
word_occ_spam = {}
word_occ_legit = {}
for word in dist_words:
    word_occ_spam[word] = 0
    word_occ_legit[word] = 0

for path in paths:
    if(path[-2:] == '10'):  
        continue
    files = [os.path.join(path, f) for f in os.listdir(path)]  
    for email in files:
        if(email.find('spmsg') != -1):  
            is_spam = True
        else:
            is_spam = False
        # dist_set = set() # there could be repeated word, we only need  occurance, use set
        with open(email) as m:
            for i, content in enumerate(m):  
                if(i == 2):  
                    words = content.split()
                    for word in words:
                        if(word.isalpha() == False): # skip symbols
                            continue
                        else:
                            if(is_spam == True):
                                word_occ_spam[word] += 1
                            else:
                                word_occ_legit[word] += 1

# total #occurence of word in spam
total_occ_spam = 0
total_occ_legit = 0
for i in word_occ_spam.values():
    total_occ_spam += i
for i in word_occ_legit.values():
    total_occ_legit += i
print(f'total words\' occurence in spam is {total_occ_spam}, total occurence in legit is {total_occ_legit}')

"""**1. Top 10 feature**"""

N = 10

"""1.1 compute probability

*In TF computation, it requires a lot in precision so we need more decimal: import decimal library*
"""

# In TF computation, it requires a lot in precision so we need more decimal
from decimal import *
getcontext().prec = 32
# M = #distinct_term
M = len(dist_words)
# compute P(x|spam)
p_x_spam_10 = []
for case in f10_test_mtf:
    p = 1
    for i in range(N):
        if case[i] == 1:
            p *= Decimal(word_occ_spam[f10[i]] + 1) / Decimal(total_occ_spam + M) # with laplacian smoothing
            # if p == 0 : 
            #     print(word_occ_spam[f10[i]] + 1, " ", total_occ_spam + M)
        else:
            p *= 1 # different from bernoulli: ignore negative evidence 
    p_x_spam_10.append(p)    

# compute P(x|legit)
p_x_legit_10 = []
for case in f10_test_mtf:
    p = 1
    for i in range(N):
        if case[i] == 1:
            p *= Decimal(word_occ_legit[f10[i]] + 1) / Decimal(total_occ_legit + M) # with laplacian smoothing
        else:
            p *= 1 # different from bernoulli: ignore negative evidence
    p_x_legit_10.append(p)    

# compute P(spam)
p_spam = Decimal(spam)/Decimal((spam+legit))

# compute P(legit)
p_legit = Decimal(legit)/Decimal((spam+legit)) 

# compute P(x)
p_x_10 = []
for i in range(len(f10_test_mtf)):
    p = p_spam * p_x_spam_10[i] + p_legit * p_x_legit_10[i]
    # if p == 0:
    #     print(p_spam, " ",p_x_spam_10[i]," ", p_legit," ", p_x_legit_10[i])
    p_x_10.append(p)
 
#print(len(p_x_spam_10))
#print(len(p_x_10))

"""1.2 Classify"""

f10_pre = []
p_spam_x_10 = []
p_legit_x_10 = []
for i in range(len(f10_test_mbf)):
    psx = p_x_spam_10[i] * p_spam / p_x_10[i]
    p_spam_x_10.append(psx)
    plx = p_x_legit_10[i] * p_legit / p_x_10[i]
    p_legit_x_10.append(plx)
    if psx >= plx :
        f10_pre.append(1)
    else:
        f10_pre.append(0)

print("samples of P(X|spam) with Decimal: ")
print(p_spam_x_10[:10])

"""1.3 Report result"""

correct = 0;
total = len(test_label)
for i in range(total):
    if test_label[i] == f10_pre[i]:
        correct+=1
correct_rate = correct / total
print("correct rate: ",correct_rate)

# spam precision is defined as the fraction of true spam e-mails among all e-mails predicted as spam
total_pre_spam = 0
true_spam_in_spam = 0
for i in range(total):
    if f10_pre[i] == 1:
        total_pre_spam += 1
        if test_label[i] == 1:
            true_spam_in_spam += 1
spam_precision_mtf_10 = true_spam_in_spam / total_pre_spam   
print("spam precision: ",spam_precision_mtf_10)

# spam recall is defined as fraction of true spam e-mails predicted as spam.
pre_spam_in_true = 0
total_true_spam = 0
for i in range(total):
    if test_label[i] == 1:
        total_true_spam += 1
        if f10_pre[i] == 1:
            pre_spam_in_true += 1
        
spam_recall_mtf_10 = pre_spam_in_true / total_true_spam  
print("spam_recall: ",spam_recall_mtf_10)

"""**2. Top 100 feature**"""

N = 100

"""2.1 Compute probability"""

# In TF computation, it requires a lot in precision so we need more decimal
from decimal import *
getcontext().prec = 32
# M = #distinct_term
M = len(dist_words)
# compute P(x|spam)
p_x_spam_100 = []
for case in f100_test_mtf:
    p = 1
    for i in range(N):
        if case[i] == 1:
            p *= Decimal(word_occ_spam[f100[i]] + 1) / Decimal(total_occ_spam + M) # with laplacian smoothing
            # if p == 0 : 
            #     print(word_occ_spam[f100[i]] + 1, " ", total_occ_spam + M)
        else:
            p *= 1 # different from bernoulli: ignore negative evidence 
    p_x_spam_100.append(p)    

# compute P(x|legit)
p_x_legit_100 = []
for case in f100_test_mtf:
    p = 1
    for i in range(N):
        if case[i] == 1:
            p *= Decimal(word_occ_legit[f100[i]] + 1) / Decimal(total_occ_legit + M) # with laplacian smoothing
        else:
            p *= 1 # different from bernoulli: ignore negative evidence
    p_x_legit_100.append(p)    

# compute P(spam)
p_spam = Decimal(spam)/Decimal((spam+legit))

# compute P(legit)
p_legit = Decimal(legit)/Decimal((spam+legit)) 

# compute P(x)
p_x_100 = []
for i in range(len(f100_test_mtf)):
    p = p_spam * p_x_spam_100[i] + p_legit * p_x_legit_100[i]
    # if p == 0:
    #     print(p_spam, " ",p_x_spam_100[i]," ", p_legit," ", p_x_legit_100[i])
    p_x_100.append(p)
 
#print(len(p_x_spam_100))
#print(len(p_x_100))

"""2.2 Classify"""

f100_pre = []
p_spam_x_100 = []
p_legit_x_100 = []
for i in range(len(f100_test_mbf)):
    psx = p_x_spam_100[i] * p_spam / p_x_100[i]
    p_spam_x_100.append(psx)
    plx = p_x_legit_100[i] * p_legit / p_x_100[i]
    p_legit_x_100.append(plx)
    if psx >= plx :
        f100_pre.append(1)
    else:
        f100_pre.append(0)

print("samples of P(X|spam) with Decimal: ")
print(p_spam_x_100[:10])

"""2.3 Report result"""

correct = 0;
total = len(test_label)
for i in range(total):
    if test_label[i] == f100_pre[i]:
        correct+=1
correct_rate = correct / total
print("correct rate: ",correct_rate)

# spam precision is defined as the fraction of true spam e-mails among all e-mails predicted as spam
total_pre_spam = 0
true_spam_in_spam = 0
for i in range(total):
    if f100_pre[i] == 1:
        total_pre_spam += 1
        if test_label[i] == 1:
            true_spam_in_spam += 1
spam_precision_mtf_100 = true_spam_in_spam / total_pre_spam   
print("spam precision: ",spam_precision_mtf_100)

# spam recall is defined as fraction of true spam e-mails predicted as spam.
pre_spam_in_true = 0
total_true_spam = 0
for i in range(total):
    if test_label[i] == 1:
        total_true_spam += 1
        if f100_pre[i] == 1:
            pre_spam_in_true += 1
        
spam_recall_mtf_100 = pre_spam_in_true / total_true_spam  
print("spam_recall: ",spam_recall_mtf_100)

"""**3. Top 1000 feature**"""

N = 1000

"""3.1 Compute probability"""

# In TF computation, it requires a lot in precision so we need more decimal
from decimal import *
getcontext().prec = 32
# M = #distinct_term
M = len(dist_words)
# compute P(x|spam)
p_x_spam_1000 = []
for case in f1000_test_mtf:
    p = 1
    for i in range(N):
        if case[i] == 1:
            p *= Decimal(word_occ_spam[f1000[i]] + 1) / Decimal(total_occ_spam + M) # with laplacian smoothing
            # if p == 0 : 
            #     print(word_occ_spam[f1000[i]] + 1, " ", total_occ_spam + M)
        else:
            p *= 1 # different from bernoulli: ignore negative evidence 
    p_x_spam_1000.append(p)    

# compute P(x|legit)
p_x_legit_1000 = []
for case in f1000_test_mtf:
    p = 1
    for i in range(N):
        if case[i] == 1:
            p *= Decimal(word_occ_legit[f1000[i]] + 1) / Decimal(total_occ_legit + M) # with laplacian smoothing
        else:
            p *= 1 # different from bernoulli: ignore negative evidence
    p_x_legit_1000.append(p)    

# compute P(spam)
p_spam = Decimal(spam)/Decimal((spam+legit))

# compute P(legit)
p_legit = Decimal(legit)/Decimal((spam+legit)) 

# compute P(x)
p_x_1000 = []
for i in range(len(f1000_test_mtf)):
    p = p_spam * p_x_spam_1000[i] + p_legit * p_x_legit_1000[i]
    # if p == 0:
    #     print(p_spam, " ",p_x_spam_1000[i]," ", p_legit," ", p_x_legit_1000[i])
    p_x_1000.append(p)
 
#print(len(p_x_spam_1000))
#print(len(p_x_1000))

"""3.2 Classify"""

f1000_pre = []
p_spam_x_1000 = []
p_legit_x_1000 = []
for i in range(len(f1000_test_mbf)):
    psx = p_x_spam_1000[i] * p_spam / p_x_1000[i]
    p_spam_x_1000.append(psx)
    plx = p_x_legit_1000[i] * p_legit / p_x_1000[i]
    p_legit_x_1000.append(plx)
    if psx >= plx :
        f1000_pre.append(1)
    else:
        f1000_pre.append(0)

print("samples of P(X|spam) with Decimal: ")
print(p_spam_x_1000[:10])

"""3.3 Report"""

correct = 0;
total = len(test_label)
for i in range(total):
    if test_label[i] == f1000_pre[i]:
        correct+=1
correct_rate = correct / total
print("correct rate: ",correct_rate)

# spam precision is defined as the fraction of true spam e-mails among all e-mails predicted as spam
total_pre_spam = 0
true_spam_in_spam = 0
for i in range(total):
    if f1000_pre[i] == 1:
        total_pre_spam += 1
        if test_label[i] == 1:
            true_spam_in_spam += 1
spam_precision_mtf_1000 = true_spam_in_spam / total_pre_spam   
print("spam precision: ",spam_precision_mtf_1000)

# spam recall is defined as fraction of true spam e-mails predicted as spam.
pre_spam_in_true = 0
total_true_spam = 0
for i in range(total):
    if test_label[i] == 1:
        total_true_spam += 1
        if f1000_pre[i] == 1:
            pre_spam_in_true += 1
        
spam_recall_mtf_1000 = pre_spam_in_true / total_true_spam  
print("spam_recall: ",spam_recall_mtf_1000)

"""# Part2 report: spam precision and spam recall of combinations of NB filters and top-N feature"""

print(f"Bernoulli NB with binary feature and top-10: precesion = {spam_precision_bbf_10}, recall = {spam_recall_bbf_10}")
print(f"Bernoulli NB with binary feature and top-100: precesion = {spam_precision_bbf_100}, recall = {spam_recall_bbf_100}")
print(f"Bernoulli NB with binary feature and top-1000: precesion = {spam_precision_bbf_1000}, recall = {spam_recall_bbf_1000}")
print(f"Multinomial NB with binary feature and top-10: precesion = {spam_precision_mbf_10}, recall = {spam_recall_mbf_10}")
print(f"Multinomial NB with binary feature and top-100: precesion = {spam_precision_mbf_100}, recall = {spam_recall_mbf_100}")
print(f"Multinomial NB with binary feature and top-1000: precesion = {spam_precision_mbf_1000}, recall = {spam_recall_mbf_1000}")
print(f"Multinomial NB with term frequency and top-10: precesion = {spam_precision_mtf_10}, recall = {spam_recall_mtf_10}")
print(f"Multinomial NB with term frequency and top-100: precesion = {spam_precision_mtf_100}, recall = {spam_recall_mtf_100}")
print(f"Multinomial NB with term frequency and top-1000: precesion = {spam_precision_mtf_1000}, recall = {spam_recall_mtf_1000}")

"""# SVM based filter

load data, generate top-N feature matrix with IG
"""

train_path = './lingspam_public/lemm_stop'  # parent directory of dataset we use
paths = [os.path.join(train_path, folders) for folders in os.listdir(train_path)]  
all_words = [] 

train_data = [] # all feature
train_data_10 = []
train_data_100 = []
train_data_1000 = []
train_label = []

test_data = [] # all feature
test_data_10 = []
test_data_100 = []
test_data_1000 = []
test_label = []



for path in paths:
    files = [os.path.join(path, f) for f in os.listdir(path)]  # get file names of this part and generate path to files
    for email in files:
        if email.find('spmsg') != -1 : #check filenames to check if it is spam 
            if path[-2:] == '10' : # if it is test data
                test_label.append(1)
            else:
                train_label.append(1)
        else:
            if path[-2:] == '10' :
                test_label.append(0)
            else:
                train_label.append(0)
        
        with open(email) as m:
            for i, content in enumerate(m):  # i: line number , content: content of each line
                if(i == 2):  # open txt and find content is in the 3rd line
                    words_in_email = set(content.split())
                    temp_train_10 = []
                    temp_train_100 = []
                    temp_train_1000 = []
                    temp_test_10 = []
                    temp_test_100 = []
                    temp_test_1000 = []

                    temp_train = []
                    temp_test = []
                    for w in dist_words:
                        if w in words_in_email:
                            if path[-2:] == '10' :
                                temp_test.append(1)
                            else:
                                temp_train.append(1)
                        else:
                            if path[-2:] == '10' :
                                temp_test.append(0)
                            else:
                                temp_train.append(0)
                    for w in f10:
                        if w in words_in_email:
                            if path[-2:] == '10' :
                                temp_test_10.append(1)
                            else:
                                temp_train_10.append(1)
                        else:
                            if path[-2:] == '10' :
                                temp_test_10.append(0)
                            else:
                                temp_train_10.append(0)
                    for w in f100:
                        if w in words_in_email:
                            if path[-2:] == '10' :
                                temp_test_100.append(1)
                            else:
                                temp_train_100.append(1)
                        else:
                            if path[-2:] == '10' :
                                temp_test_100.append(0)
                            else:
                                temp_train_100.append(0)
                    for w in f1000:
                        if w in words_in_email:
                            if path[-2:] == '10' :
                                temp_test_1000.append(1)
                            else:
                                temp_train_1000.append(1)
                        else:
                            if path[-2:] == '10' :
                                temp_test_1000.append(0)
                            else:
                                temp_train_1000.append(0)

        if path[-2:] == '10':
            test_data.append(np.array(temp_test))
            test_data_10.append(np.array(temp_test_10))
            test_data_100.append(np.array(temp_test_100))
            test_data_1000.append(np.array(temp_test_1000))
        else :
            train_data.append(np.array(temp_train))
            train_data_10.append(np.array(temp_train_10))
            train_data_100.append(np.array(temp_train_100))
            train_data_1000.append(np.array(temp_train_1000))

"""*Memo for an exception : when I try to fit for SVM, it says "setting an array element with a sequence" which indicates my matrix is not a generalized "rectangular", then I found I mistakenly wrote temp_train_10.append() in a top-1000 situation(part10).*

**ALL feature**

*Memo: Be carefull with strength of regularization,  make it soft because what I have is 0-1 feature!*
"""

# Commented out IPython magic to ensure Python compatibility.
from sklearn import svm
svm_ = svm.SVC(kernel = 'rbf',C=1000,probability=False) # because it is 0-1 feature, no need for strength on regularization
# %time svm_.fit(train_data,train_label)

pre_svm = svm_.predict(test_data)
print(pre_svm)

total = len(test_label)
# correct rate
score = svm_.score(test_data, test_label)
print("correct rate: ",score)

# spam precision is defined as the fraction of true spam e-mails among all e-mails predicted as spam
total_pre_spam = 0
true_spam_in_spam = 0
for i in range(total):
    if pre_svm[i] == 1:
        total_pre_spam += 1
        if test_label[i] == 1:
            true_spam_in_spam += 1
spam_precision_svm_af = true_spam_in_spam / (total_pre_spam) 
print("spam precision: ",spam_precision_svm_af)

# spam recall is defined as fraction of true spam e-mails predicted as spam.
pre_spam_in_true = 0
total_true_spam = 0
for i in range(total):
    if test_label[i] == 1:
        total_true_spam += 1
        if pre_svm[i] == 1:
            pre_spam_in_true += 1
        
spam_recall_svm_af = pre_spam_in_true / total_true_spam  
print("spam_recall: ",spam_recall_svm_af)

"""**1. Top 10 feature**"""

# Commented out IPython magic to ensure Python compatibility.
from sklearn import svm
#svm_10 = svm.SVC(kernel = 'linear',C=1000,probability=False)
svm_10 = svm.SVC(kernel = 'rbf',C=1000,probability=False)
# %time svm_10.fit(train_data_10,train_label)

pre_svm_10 = svm_10.predict(test_data_10)
print(pre_svm_10)

total = len(test_label)
# correct rate
score_10 = svm_10.score(test_data_10, test_label)
print("correct rate: ",score_10)

# spam precision is defined as the fraction of true spam e-mails among all e-mails predicted as spam
total_pre_spam = 0
true_spam_in_spam = 0
for i in range(total):
    if pre_svm_10[i] == 1:
        total_pre_spam += 1
        if test_label[i] == 1:
            true_spam_in_spam += 1
spam_precision_svm_10 = true_spam_in_spam / (total_pre_spam) 
print("spam precision: ",spam_precision_svm_10)

# spam recall is defined as fraction of true spam e-mails predicted as spam.
pre_spam_in_true = 0
total_true_spam = 0
for i in range(total):
    if test_label[i] == 1:
        total_true_spam += 1
        if pre_svm_10[i] == 1:
            pre_spam_in_true += 1
        
spam_recall_svm_10 = pre_spam_in_true / total_true_spam  
print("spam_recall: ",spam_recall_svm_10)

"""**2. Top 100 feature**"""

# Commented out IPython magic to ensure Python compatibility.
from sklearn import svm
#svm_100 = svm.SVC(kernel = 'linear',C=1000,probability=False)
svm_100 = svm.SVC(kernel = 'rbf',C=1000,probability=False)
# %time svm_100.fit(train_data_100,train_label)

pre_svm_100 = svm_100.predict(test_data_100)
print(pre_svm_100)

total = len(test_label)
# correct rate
score_100 = svm_100.score(test_data_100, test_label)
print("correct rate: ",score_100)

# spam precision is defined as the fraction of true spam e-mails among all e-mails predicted as spam
total_pre_spam = 0
true_spam_in_spam = 0
for i in range(total):
    if pre_svm_100[i] == 1:
        total_pre_spam += 1
        if test_label[i] == 1:
            true_spam_in_spam += 1
spam_precision_svm_100 = true_spam_in_spam / (total_pre_spam) 
print("spam precision: ",spam_precision_svm_100)

# spam recall is defined as fraction of true spam e-mails predicted as spam.
pre_spam_in_true = 0
total_true_spam = 0
for i in range(total):
    if test_label[i] == 1:
        total_true_spam += 1
        if pre_svm_100[i] == 1:
            pre_spam_in_true += 1
        
spam_recall_svm_100 = pre_spam_in_true / total_true_spam  
print("spam_recall: ",spam_recall_svm_100)

"""**3. Top 1000 feature**"""

# Commented out IPython magic to ensure Python compatibility.
from sklearn import svm
#svm_1000 = svm.SVC(kernel = 'linear',C=1000,probability=False)
svm_1000 = svm.SVC(kernel = 'rbf',C=1000,probability=False)
# %time svm_1000.fit(train_data_1000,train_label)

pre_svm_1000 = svm_1000.predict(test_data_1000)
print(pre_svm_1000)

total = len(test_label)
# correct rate
score_1000 = svm_1000.score(test_data_1000, test_label)
print("correct rate: ",score_1000)

# spam precision is defined as the fraction of true spam e-mails among all e-mails predicted as spam
total_pre_spam = 0
true_spam_in_spam = 0
for i in range(total):
    if pre_svm_1000[i] == 1:
        total_pre_spam += 1
        if test_label[i] == 1:
            true_spam_in_spam += 1
spam_precision_svm_1000 = true_spam_in_spam / (total_pre_spam) 
print("spam precision: ",spam_precision_svm_1000)

# spam recall is defined as fraction of true spam e-mails predicted as spam.
pre_spam_in_true = 0
total_true_spam = 0
for i in range(total):
    if test_label[i] == 1:
        total_true_spam += 1
        if pre_svm_1000[i] == 1:
            pre_spam_in_true += 1
        
spam_recall_svm_1000 = pre_spam_in_true / total_true_spam  
print("spam_recall: ",spam_recall_svm_1000)

"""# Part4 Report: SVM spam precision and spam recall

I used SVM with RBF kernel, it perform obviously better on recall than linear kernel. And I set regularization very soft because I am using binary feature, it only has 0 or 1. 

For features, I used binary word feature, and select them by IG used before, I tried all feature and top-10, top-100, top-1000 from IG.

**It results into top-1000 with RBF kernel performs best in general, because its precision and recall rate are both high.**
"""

print(f"RBF kernal with all feature: precision = {spam_precision_svm_af}, recall = {spam_recall_svm_af}")
print(f"RBF kernal with top-10 feature: precision = {spam_precision_svm_10}, recall = {spam_recall_svm_10}")
print(f"RBF kernal with top-100 feature: precision = {spam_precision_svm_100}, recall = {spam_recall_svm_100}")
print(f"RBF kernal with top-1000 feature: precision = {spam_precision_svm_1000}, recall = {spam_recall_svm_1000}")

"""Some extra statistics when using linear kernel:

linear & top-10: precision = 0.88372, recall = 0.775510

linear & top-100: precision = 0.9743, recall = 0.775510

linear & top-1000: precision = 1.0, recall = 0.85714

# Adversarial Classification

baseline multinomial NB with binary feature and top-10 feature

**1. Before attacker's modification**
"""

N = 10
test_label = []  
f10_test_mbf = []  # test emails' feature matrix for top 10

emails = [os.path.join(test_path, file_name) for file_name in os.listdir(test_path)]  
for email in emails:
    # spam: 1, legit: 0
    if email.find('spmsg')!=-1:
        test_label.append(1)
    else:
        test_label.append(0)
    
    with open(email) as m:
        temp_f10_bf = []
        for i, content in enumerate(m):  
            if i==2:  # content at 3rd line
                words_in_email = set(content.split())
                for w in f10:
                    if w in words_in_email:
                        temp_f10_bf.append(1)
                    else:
                        temp_f10_bf.append(0)

        f10_test_mbf.append(temp_f10_bf)
        
# compute P(x|spam)
p_x_spam_10 = []
for case in f10_test_mbf:
    p = 1
    for i in range(N):
        if case[i] == 1:
            p *= (word_spam[f10[i]] + 1) / (spam + 2) 
        else:
            p *= 1 
    p_x_spam_10.append(p)    

# compute P(x|legit)
p_x_legit_10 = []
for case in f10_test_mbf:
    p = 1
    for i in range(N):
        if case[i] == 1:
            p *= (word_legit[f10[i]] + 1) / (legit + 2) 
        else:
            p *= 1 
    p_x_legit_10.append(p)    

# compute P(spam)
p_spam = spam/(spam+legit)

# compute P(legit)
p_legit = legit/(spam+legit) 

# compute P(x)
p_x_10 = []
for i in range(len(f10_test_mbf)):
    p = p_spam * p_x_spam_10[i] + p_legit * p_x_legit_10[i]
    p_x_10.append(p)
 
#print(len(p_x_spam_10))
#print(len(p_x_10))

f10_pre = []
p_spam_x_10 = []
p_legit_x_10 = []
for i in range(len(f10_test_mbf)):
    psx = p_x_spam_10[i] * p_spam / p_x_10[i]
    p_spam_x_10.append(psx)
    plx = p_x_legit_10[i] * p_legit / p_x_10[i]
    p_legit_x_10.append(plx)
    if psx >= plx :
        f10_pre.append(1)
    else:
        f10_pre.append(0)

# false negative
fn_before = 0
true_positive = 0
for i in range(total):
    if test_label[i] == 1 and f10_pre[i] == 1 :
        true_positive +=1
    if test_label[i] == 1 and f10_pre[i] == 0:
        fn_before+= 1
        
fnr_before = n_fn_before / (true_positive + n_fn_before)  
print("Flase negative rate before attacker's modification: ",fnr_before)

"""**2. After attacker's modification**

Compute P(Xi=xi|C=c)
"""

# compute P(x|spam)

p_1_spam_10 = []
p_0_spam_10 = []

for i in range(N):
    # 1_spam
    p = (word_spam[f10[i]] + 1) / (spam + 2) # with laplacian smoothing
    p_1_spam_10.append(p)
    # 0_spam
    p = 1 - (word_spam[f10[i]] + 1) / (spam + 2)   
    p_0_spam_10.append(p) 

# compute P(x|legit)
p_1_legit_10 = []
p_0_legit_10 = []
p = 1
for i in range(N):
    # 1_spam
    p = (word_legit[f10[i]] + 1) / (legit + 2) # with laplacian smoothing
    p_1_legit_10.append(p)
    # 0_spam
    p = 1 - (word_legit[f10[i]] + 1) / (legit + 2)
    p_0_legit_10.append(p)

"""Compute desired reduction log( P(spam | x) / P(legit | x) ) =  log( P(x | spam) / P(x | legit) ) + log(P(spam)/P(legit))"""

desired_delta = []
for case in f10_test_mbf:
    p = 1
    for i in range(N):
        if case[i] == 1:
            p *= p_1_spam_10[i] / p_1_legit_10[i]
        else:
            p *= p_0_spam_10[i] / p_0_legit_10[i]
    d = np.log2(p) + np.log2(spam/legit)
    desired_delta.append(d)

print("desired delta:",desired_delta[5:15])
print("corresponding label:",test_label[5:15])

"""compute  delta_LO_ix' = LOc(xi) - LOc(xi') for top-10 features

LO(xi) = log( P(xi | spam) / P(xi | legit) )

As assignment states, the adversary only use **ADD_WORDS** strategy, so I set reduction for term_i to 0 when each email has this term

Only consider adding word for reduction of each feature
"""

delta_add_word = []
for i in range(10):
    d = np.log2(p_0_spam_10[i] / p_0_legit_10[i]) - np.log2(p_1_spam_10[i] / p_1_legit_10[i]) # delta_LO_ix' = LOc(xi) - LOc(xi') old - new
    d = max(0,d)
    delta_add_word.append(d)

"""determine reduction acquired and **sort them** by value, now we can see top-10words in this sorted order"""

sorted_delta = []
for case in f10_test_mbf:
    delta_i = {}
    for i in range(10):
        if case[i] == 1:
            delta_i[f10[i]] = 0
        else:
            delta_i[f10[i]] = delta_add_word[i]
    sorted_delta_i = OrderedDict(sorted(delta_i.items(), key=lambda x : x[1], reverse=True))
    sorted_delta.append(sorted_delta_i)

#print("check f_index helper: index of remove",f_index["remove"])
print("orgin f10 sequence: ", f10)
#print("case 0:", f10_test_mbf[0])
print("desired delta of case 0: ",desired_delta[0])
for (w,d) in sorted_delta[0].items():
    print(w,":", d)

"""Because adding a term incur an unit cost, we can pick term with larger reduction first (*being greedy*) for minimum cost tring to meet desired reduction!

**Start modify**
"""

# feature index helper
f_index = {}
for i in range(10):
    f_index[f10[i]]=i

avg_cost = 0
test_spam = 0
f10_test_mbf_mod = []
for i in f10_test_mbf:
    f10_test_mbf_mod.append(i.copy())

for i in range(len(sorted_delta)):
    if test_label[i] == 0: # do not modify legit email
        continue
    else:
        test_spam+=1
    d = sorted_delta[i]
    sum = 0
    for (k,v) in d.items():
        if v <= 0: #if value reach 0 in sorted reduction values, can't reach desired reduction, break
            break
        else: # reduction > 0
            f10_test_mbf_mod[i][f_index[k]] = 1 # add word
            avg_cost+=1
            sum += v
            if desired_delta[i] <= sum: # meet desired reduction 
                break; 
avg_cost /= test_spam
print("average cost of attacker: ", avg_cost)

print(f10)
print("origin:",f10_test_mbf[0])
print("modify:",f10_test_mbf_mod[0])

"""perform classification after attacker's modification on test data"""

# compute P(x|spam)
p_x_spam_10 = []
for case in f10_test_mbf_mod:
    p = 1
    for i in range(N):
        if case[i] == 1:
            p *= (word_spam[f10[i]] + 1) / (spam + 2) # with laplacian smoothing
        else:
            p *= 1 
    p_x_spam_10.append(p)    

# compute P(x|legit)
p_x_legit_10 = []
for case in f10_test_mbf_mod:
    p = 1
    for i in range(N):
        if case[i] == 1:
            p *= (word_legit[f10[i]] + 1) / (legit + 2) # with laplacian smoothing
        else:
            p *= 1 
    p_x_legit_10.append(p)    

# compute P(x)
p_x_10 = []
for i in range(len(f10_test_mbf_mod)):
    p = p_spam * p_x_spam_10[i] + p_legit * p_x_legit_10[i]
    p_x_10.append(p)

f10_mod_pre = []
p_spam_x_10 = []
p_legit_x_10 = []
for i in range(len(f10_test_mbf_mod)):
    psx = p_x_spam_10[i] * p_spam / p_x_10[i]
    p_spam_x_10.append(psx)
    plx = p_x_legit_10[i] * p_legit / p_x_10[i]
    p_legit_x_10.append(plx)
    if psx >= plx :
        f10_mod_pre.append(1)
    else:
        f10_mod_pre.append(0)

# false negative
fn_after = 0
true_positive = 0
for i in range(total):
    if test_label[i] == 1 and f10_mod_pre[i] == 1 :
        true_positive +=1
    if test_label[i] == 1 and f10_mod_pre[i] == 0:
        fn_after+= 1
        
fnr_after = fn_after / (true_positive + fn_after)  
print("Flase negative rate after attacker's modification: ",fnr_after)

"""**3. After Update calssifier**

PA{spam|x'}/PA{legit|x'} = PA{x'|spam}*PA{SPAM} / (PA{x'|legit} * PA{legit})
"""

# # compute PA(x'|spam), x' can be generated by any possible x (ADD WORDS)so it is sum of P(x|spam)
# p_x_spam_10 = []
# for CASE in f10_test_mbf_mod: # facing attack
#     sum = 0
#     for case in f10_test_mbf_mod:
#         # can contain it self I think, or there might be zero availble x
#         if case == CASE:
#             continue
#         candidate = True
#         count_add = 0
#         for i in range(len(case)):
#             # because we assume classifier knows adversary will add word in a optimal way, so according to average cost of attack, I set limit to at most 1 word to add
#             if count_add > 1:  
#                 candidate = False
#             if CASE[i] == 0 and case[i] == 1:
#                 candidate = False
#             else: # can ADD WORDS to case to get CASE
#                 if CASE[i] == 1 and case[i] == 0:
#                     count_add+=1
#         p = 0
#         if candidate == True:
#             p = 1
#             for i in range(N):
#                 if case[i] == 1:
#                     p *= (word_spam[f10[i]] + 1) / (spam + 2) 
#                 else:
#                     p *= 1   
#         sum += p
#     p_x_spam_10.append(sum)    

# # compute PA(x'|legit)
# p_x_legit_10 = []
# for case in f10_test_mbf_mod:
#     p = 1
#     for i in range(N):
#         if case[i] == 1:
#             p *= (word_legit[f10[i]] + 1) / (legit + 2) 
#         else:
#             p *= 1
#     p_x_legit_10.append(p)    



# f10_update_pre = []

# for i in range(len(f10_test_mbf_mod)):
#     psx = p_x_spam_10[i] * p_spam 
#     p_spam_x_10.append(psx)
#     plx = p_x_legit_10[i] * p_legit 
#     p_legit_x_10.append(plx)
#     if psx / plx >=1 :
#         f10_update_pre.append(1)
#     else:
#         f10_update_pre.append(0)

# compute PA(x'|legit)
p_x_legit_10 = []
for case in f10_test_mbf_mod:
    p = 1
    for i in range(N):
        if case[i] == 1:
            p *= (word_legit[f10[i]] + 1) / (legit + 2) 
        else:
            p *= 1
    p_x_legit_10.append(p) 

# assume classifier know attack strategy: always add words with minimum cost
# compute PA(x'|spam), x' can be generated by any possible x (ADD WORDS), so it is sum of P(x|spam)
p_x_spam_10 = []
for i in range(len(f10_test_mbf_mod)):
    case = f10_test_mbf_mod[i] 
    sum = 0
    for w in sorted_delta[i].keys(): # according to sorted delta(reduction), inversely find optimal original x from x' by remove word with larger delta (greedy)
        if case[f_index[w]] == 1: # can remove word
            case[f_index[w]] = 0  # remove
            psx = 1
            plx = 1
            for i in range(N):
                if case[i] == 1:
                    psx *= (word_spam[f10[i]] + 1) / (spam + 2) 
                else:
                    psx *= 1
                if case[i] == 1:
                    plx *= (word_legit[f10[i]] + 1) / (legit + 2) 
                else:
                    plx *= 1
            if psx*p_spam / (plx*p_legit) >= 1: # the original one inversly recover from modified case is spam! we find attacker's original case to modify
                sum+=psx
                break
            else:
                 continue
    p_x_spam_10.append(sum)

f10_update_pre = []

for i in range(len(f10_test_mbf_mod)):
    psx = p_x_spam_10[i] * p_spam 
    p_spam_x_10.append(psx)
    plx = p_x_legit_10[i] * p_legit 
    p_legit_x_10.append(plx)
    if psx / plx >=1 :
        f10_update_pre.append(1)
    else:
        f10_update_pre.append(0)

# false negative
fn_update = 0
true_positive = 0
for i in range(total):
    if test_label[i] == 1 and f10_update_pre[i] == 1 :
        true_positive +=1
    if test_label[i] == 1 and f10_update_pre[i] == 0:
        fn_update+= 1
        
fnr_update = fn_update / (true_positive + fn_update)  
print("Flase negative rate after update classifier: ",fnr_update) 

# false positive
fp_update = 0
true_negative = 0
for i in range(total):
    if test_label[i] == 0 and f10_update_pre[i] == 0 :
        true_negative +=1
    if test_label[i] == 0 and f10_update_pre[i] == 1:
        fp_update+= 1
        
fpr_update = fp_update / (true_negative + fp_update)  
print("Flase positive rate after update classifier: ",fpr_update)

"""# Part4 report: adversarial classification"""

print("Flase negative rate before attacker's modification: ",fnr_before) 
print("Flase negative rate after attacker's modification: ",fnr_after)
print("average cost of attacker: ", avg_cost)  
print("Flase negative rate after update classifier: ",fnr_update)   
print("Flase positive rate after update classifier: ",fpr_update)
# -*- coding: utf-8 -*- MLCS_LAB1.ipynb  Automatically generated by Colaboratory.  Original file is located at     https://colab.research.google.com/drive/1KnN9oHgEbNbMZg8DooJNOqvXTWeBTj92  Upload dataset and unzip the file Import library Load words in emails into a word list. # parent directory of dataset we use # 1 for spam, 0 for legit # path[-2:] check if this is part10 which is used for testing, skip # get file names of this part and generate path to files #check filenames to check if it is spam # i: line number , content: content of each line # open txt and find content is in the 3rd line # remove symbols which have no big relationship with spam or legit # distinct words # Compute Information Gain(IG)  1. compute H(C), C is RV that determins if a document is spam or legit 2. Compute H(C|Xi), need to create dictionary for word:#spam and word:#legit to help computing H(C|Xi) # there could be repeated word, we only need  occurance, use set # skip symbols Compute H(C|Xi) = "combination" of P(X=x,C=c)log(P(C=c|X=x)) # c = legit, x = 0 #sum only when non-zero probability # c = legit, x = 1 # c = spam, x = 0 # c = spam, x = 1 3. Compute IG # Get top N features # sort by value # sorted can be applied to iterable containers # sorted return a new result rather than sort in place # sorted return a list object not a sorted dictionary!!!!!!! # sorted_IG = sorted(IG.items(), key=lambda x : x[1], reverse=True) # top 10 # top 100 # top 1000 # Part1 report: top-10 words: # Bernoulli NB classifier with binary feature  *Load test data* # test emails' feature matrix for top 10 # test emails' feature matrix for top 100 # test emails' feature matrix for top 1000 # spam: 1, legit: 0 # content at 3rd line **1. top 10 feature** 1.1 Compute probability # x is conbination of words # compute P(x|spam) # with laplacian smoothing # with laplacian smoothing # compute P(x|legit) # with laplacian smoothing # with laplacian smoothing # compute P(spam) # compute P(legit) # compute P(x) 1.2 Classify by computing P(spam|X) and P(legit|X) and compare 1.3 Report result #p_spam_x_10[:10] #p_legit_x_10[:10] # spam precision is defined as the fraction of true spam e-mails among all e-mails predicted as spam # spam recall is defined as fraction of true spam e-mails predicted as spam. **2. top 100 feature** 2.1 Compute probability # x is conbination of words # compute P(x|spam) # with laplacian smoothing # with laplacian smoothing # compute P(x|legit) # with laplacian smoothing # with laplacian smoothing # compute P(spam) # compute P(legit) # compute P(x) #print(len(p_x_spam_100)) #print(len(p_x_100)) 2.2 Classify by computing P(spam|X) and P(legit|X) and compare 2.3 Report result # spam precision is defined as the fraction of true spam e-mails among all e-mails predicted as spam # spam recall is defined as fraction of true spam e-mails predicted as spam. 3. top 100 feature **3.1 Compute probability** # x is conbination of words # compute P(x|spam) # with laplacian smoothing # with laplacian smoothing # compute P(x|legit) # with laplacian smoothing # with laplacian smoothing # compute P(spam) # compute P(legit) # compute P(x) #print(len(p_x_spam_1000)) #print(len(p_x_1000)) 3.2 Classify by computing P(spam|X) and P(legit|X) and compare 3.3 Report result # spam precision is defined as the fraction of true spam e-mails among all e-mails predicted as spam # spam recall is defined as fraction of true spam e-mails predicted as spam. ***It seems that spam precision goes up while spam recall goes down with more feature selected***  # Multinomial NB with binary features  *Load test data* # test emails' feature matrix for top 10 # test emails' feature matrix for top 100 # test emails' feature matrix for top 1000 # spam: 1, legit: 0 # content at 3rd line **1. top 10 feature** 1.1 Compute probability # compute P(x|spam) # with laplacian smoothing # different from bernoulli: ignore negative evidence # compute P(x|legit) # with laplacian smoothing # different from bernoulli: ignore negative evidence # compute P(spam) # compute P(legit) # compute P(x) #print(len(p_x_spam_10)) #print(len(p_x_10)) 1.2 Classify by computing P(spam|X) and P(legit|X) and compare 1.3 report result # spam precision is defined as the fraction of true spam e-mails among all e-mails predicted as spam # spam recall is defined as fraction of true spam e-mails predicted as spam. **2. top 100 feature** 2.1 Compute probability # compute P(x|spam) # with laplacian smoothing # different from bernoulli: ignore negative evidence # compute P(x|legit) # with laplacian smoothing # different from bernoulli: ignore negative evidence # compute P(spam) # compute P(legit) # compute P(x) #print(len(p_x_spam_100)) #print(len(p_x_100)) 2.2 Classify by computing P(spam|X) and P(legit|X) and compare 2.3 Report result # spam precision is defined as the fraction of true spam e-mails among all e-mails predicted as spam # spam recall is defined as fraction of true spam e-mails predicted as spam. **3. top 1000 feature** 3.1 Compute probability # compute P(x|spam) # with laplacian smoothing # different from bernoulli: ignore negative evidence # compute P(x|legit) # with laplacian smoothing # different from bernoulli: ignore negative evidence # compute P(spam) # compute P(legit) # compute P(x) #print(len(p_x_spam_1000)) #print(len(p_x_1000)) 3.2 Classify by computing P(spam|X) and P(legit|X) and compare 3.3 Report result # spam precision is defined as the fraction of true spam e-mails among all e-mails predicted as spam # spam recall is defined as fraction of true spam e-mails predicted as spam. ***Adding feature seems helps multinomial NB with binary feature positively***  # Multinomial NB with term frequency (TF) features  *Load test data* # test emails' feature matrix for top 10 # test emails' feature matrix for top 100 # test emails' feature matrix for top 1000 # spam: 1, legit: 0 # content at 3rd line *For Term Frequency we need new dictionary for words* # occurence # dist_set = set() # there could be repeated word, we only need  occurance, use set # skip symbols # total #occurence of word in spam **1. Top 10 feature** 1.1 compute probability  *In TF computation, it requires a lot in precision so we need more decimal: import decimal library* # In TF computation, it requires a lot in precision so we need more decimal # M = #distinct_term # compute P(x|spam) # with laplacian smoothing # if p == 0 : #     print(word_occ_spam[f10[i]] + 1, " ", total_occ_spam + M) # different from bernoulli: ignore negative evidence # compute P(x|legit) # with laplacian smoothing # different from bernoulli: ignore negative evidence # compute P(spam) # compute P(legit) # compute P(x) # if p == 0: #     print(p_spam, " ",p_x_spam_10[i]," ", p_legit," ", p_x_legit_10[i]) #print(len(p_x_spam_10)) #print(len(p_x_10)) 1.2 Classify 1.3 Report result # spam precision is defined as the fraction of true spam e-mails among all e-mails predicted as spam # spam recall is defined as fraction of true spam e-mails predicted as spam. **2. Top 100 feature** 2.1 Compute probability # In TF computation, it requires a lot in precision so we need more decimal # M = #distinct_term # compute P(x|spam) # with laplacian smoothing # if p == 0 : #     print(word_occ_spam[f100[i]] + 1, " ", total_occ_spam + M) # different from bernoulli: ignore negative evidence # compute P(x|legit) # with laplacian smoothing # different from bernoulli: ignore negative evidence # compute P(spam) # compute P(legit) # compute P(x) # if p == 0: #     print(p_spam, " ",p_x_spam_100[i]," ", p_legit," ", p_x_legit_100[i]) #print(len(p_x_spam_100)) #print(len(p_x_100)) 2.2 Classify 2.3 Report result # spam precision is defined as the fraction of true spam e-mails among all e-mails predicted as spam # spam recall is defined as fraction of true spam e-mails predicted as spam. **3. Top 1000 feature** 3.1 Compute probability # In TF computation, it requires a lot in precision so we need more decimal # M = #distinct_term # compute P(x|spam) # with laplacian smoothing # if p == 0 : #     print(word_occ_spam[f1000[i]] + 1, " ", total_occ_spam + M) # different from bernoulli: ignore negative evidence # compute P(x|legit) # with laplacian smoothing # different from bernoulli: ignore negative evidence # compute P(spam) # compute P(legit) # compute P(x) # if p == 0: #     print(p_spam, " ",p_x_spam_1000[i]," ", p_legit," ", p_x_legit_1000[i]) #print(len(p_x_spam_1000)) #print(len(p_x_1000)) 3.2 Classify 3.3 Report # spam precision is defined as the fraction of true spam e-mails among all e-mails predicted as spam # spam recall is defined as fraction of true spam e-mails predicted as spam. # Part2 report: spam precision and spam recall of combinations of NB filters and top-N feature # SVM based filter  load data, generate top-N feature matrix with IG # parent directory of dataset we use # all feature # all feature # get file names of this part and generate path to files #check filenames to check if it is spam # if it is test data # i: line number , content: content of each line # open txt and find content is in the 3rd line *Memo for an exception : when I try to fit for SVM, it says "setting an array element with a sequence" which indicates my matrix is not a generalized "rectangular", then I found I mistakenly wrote temp_train_10.append() in a top-1000 situation(part10).*  **ALL feature**  *Memo: Be carefull with strength of regularization,  make it soft because what I have is 0-1 feature!* # Commented out IPython magic to ensure Python compatibility. # because it is 0-1 feature, no need for strength on regularization # %time svm_.fit(train_data,train_label) # correct rate # spam precision is defined as the fraction of true spam e-mails among all e-mails predicted as spam # spam recall is defined as fraction of true spam e-mails predicted as spam. **1. Top 10 feature** # Commented out IPython magic to ensure Python compatibility. #svm_10 = svm.SVC(kernel = 'linear',C=1000,probability=False) # %time svm_10.fit(train_data_10,train_label) # correct rate # spam precision is defined as the fraction of true spam e-mails among all e-mails predicted as spam # spam recall is defined as fraction of true spam e-mails predicted as spam. **2. Top 100 feature** # Commented out IPython magic to ensure Python compatibility. #svm_100 = svm.SVC(kernel = 'linear',C=1000,probability=False) # %time svm_100.fit(train_data_100,train_label) # correct rate # spam precision is defined as the fraction of true spam e-mails among all e-mails predicted as spam # spam recall is defined as fraction of true spam e-mails predicted as spam. **3. Top 1000 feature** # Commented out IPython magic to ensure Python compatibility. #svm_1000 = svm.SVC(kernel = 'linear',C=1000,probability=False) # %time svm_1000.fit(train_data_1000,train_label) # correct rate # spam precision is defined as the fraction of true spam e-mails among all e-mails predicted as spam # spam recall is defined as fraction of true spam e-mails predicted as spam. # Part4 Report: SVM spam precision and spam recall  I used SVM with RBF kernel, it perform obviously better on recall than linear kernel. And I set regularization very soft because I am using binary feature, it only has 0 or 1.   For features, I used binary word feature, and select them by IG used before, I tried all feature and top-10, top-100, top-1000 from IG.  **It results into top-1000 with RBF kernel performs best in general, because its precision and recall rate are both high.** Some extra statistics when using linear kernel:  linear & top-10: precision = 0.88372, recall = 0.775510  linear & top-100: precision = 0.9743, recall = 0.775510  linear & top-1000: precision = 1.0, recall = 0.85714  # Adversarial Classification  baseline multinomial NB with binary feature and top-10 feature  **1. Before attacker's modification** # test emails' feature matrix for top 10 # spam: 1, legit: 0 # content at 3rd line # compute P(x|spam) # compute P(x|legit) # compute P(spam) # compute P(legit) # compute P(x) #print(len(p_x_spam_10)) #print(len(p_x_10)) # false negative **2. After attacker's modification**  Compute P(Xi=xi|C=c) # compute P(x|spam) # 1_spam # with laplacian smoothing # 0_spam # compute P(x|legit) # 1_spam # with laplacian smoothing # 0_spam Compute desired reduction log( P(spam | x) / P(legit | x) ) =  log( P(x | spam) / P(x | legit) ) + log(P(spam)/P(legit)) compute  delta_LO_ix' = LOc(xi) - LOc(xi') for top-10 features  LO(xi) = log( P(xi | spam) / P(xi | legit) )  As assignment states, the adversary only use **ADD_WORDS** strategy, so I set reduction for term_i to 0 when each email has this term  Only consider adding word for reduction of each feature # delta_LO_ix' = LOc(xi) - LOc(xi') old - new determine reduction acquired and **sort them** by value, now we can see top-10words in this sorted order #print("check f_index helper: index of remove",f_index["remove"]) #print("case 0:", f10_test_mbf[0]) Because adding a term incur an unit cost, we can pick term with larger reduction first (*being greedy*) for minimum cost tring to meet desired reduction!  **Start modify** # feature index helper # do not modify legit email #if value reach 0 in sorted reduction values, can't reach desired reduction, break # reduction > 0 # add word # meet desired reduction perform classification after attacker's modification on test data # compute P(x|spam) # with laplacian smoothing # compute P(x|legit) # with laplacian smoothing # compute P(x) # false negative **3. After Update calssifier**  PA{spam|x'}/PA{legit|x'} = PA{x'|spam}*PA{SPAM} / (PA{x'|legit} * PA{legit}) # # compute PA(x'|spam), x' can be generated by any possible x (ADD WORDS)so it is sum of P(x|spam) # p_x_spam_10 = [] # for CASE in f10_test_mbf_mod: # facing attack #     sum = 0 #     for case in f10_test_mbf_mod: #         # can contain it self I think, or there might be zero availble x #         if case == CASE: #             continue #         candidate = True #         count_add = 0 #         for i in range(len(case)): #             # because we assume classifier knows adversary will add word in a optimal way, so according to average cost of attack, I set limit to at most 1 word to add #             if count_add > 1: #                 candidate = False #             if CASE[i] == 0 and case[i] == 1: #                 candidate = False #             else: # can ADD WORDS to case to get CASE #                 if CASE[i] == 1 and case[i] == 0: #                     count_add+=1 #         p = 0 #         if candidate == True: #             p = 1 #             for i in range(N): #                 if case[i] == 1: #                     p *= (word_spam[f10[i]] + 1) / (spam + 2) #                 else: #                     p *= 1 #         sum += p #     p_x_spam_10.append(sum) # # compute PA(x'|legit) # p_x_legit_10 = [] # for case in f10_test_mbf_mod: #     p = 1 #     for i in range(N): #         if case[i] == 1: #             p *= (word_legit[f10[i]] + 1) / (legit + 2) #         else: #             p *= 1 #     p_x_legit_10.append(p) # f10_update_pre = [] # for i in range(len(f10_test_mbf_mod)): #     psx = p_x_spam_10[i] * p_spam #     p_spam_x_10.append(psx) #     plx = p_x_legit_10[i] * p_legit #     p_legit_x_10.append(plx) #     if psx / plx >=1 : #         f10_update_pre.append(1) #     else: #         f10_update_pre.append(0) # compute PA(x'|legit) # assume classifier know attack strategy: always add words with minimum cost # compute PA(x'|spam), x' can be generated by any possible x (ADD WORDS), so it is sum of P(x|spam) # according to sorted delta(reduction), inversely find optimal original x from x' by remove word with larger delta (greedy) # can remove word # remove # the original one inversly recover from modified case is spam! we find attacker's original case to modify # false negative # false positive # Part4 report: adversarial classification