# -*- coding: utf-8 -*-
"""HW1 - COE426.ipynb

Automatically generated by Colaboratory.

Original file is located at
    

#<NAME>
#201535050
"""

#import all needed libraries
import pandas as pd
import numpy as np
import sys, resource
import re
import plotly.express as px
from scipy.stats import entropy
import random

#unlock colab resources for infinite memory and recursion :)
resource.setrlimit(resource.RLIMIT_STACK, (2**29,-1))
sys.setrecursionlimit(10**6)

#Set Columns of data set
Columns = ["Age", "Gender", "Marital", "Race_Status", "Birth_Place", "Language", "Occupation", "Income_(K)"]

#Load raw data set and append columns to it
Original_DF = pd.read_csv("https://raw.githubusercontent.com/Alnasser0/COE426-HWs/master/HW1/ipums-assign1.csv", names=Columns)

#View data set
Original_DF

#Export data set (Can be viewed/download from left panel)
Original_DF.to_csv('tablewithcolumns.csv', index=False)

"""#Task1: Linkage attack (20 pts)
Download the file named ”ipums.txt” from blackboard and unzip
it. Using Table 2 as external background information, perform
a data linkage attack to **find the annual salary of each person in
the table**. You are free to use any tool/programming language to
complete this task, e.g., Excel, Python, Java, etc.
"""

#Load columns of table 2
Columns2 = ["Name", "Age", "Birth_Place"]

#Create a data frame and fill data of table 2
Task1_DF = pd.DataFrame(columns=Columns2)
Task1_DF=Task1_DF.append({'Name': "Ahmed", 'Age': 28, 'Birth_Place': 110}, ignore_index=True)
Task1_DF=Task1_DF.append({'Name': "Fatma", 'Age': 44, 'Birth_Place': 4}, ignore_index=True)
Task1_DF=Task1_DF.append({'Name': "Ali", 'Age': 17, 'Birth_Place': 199}, ignore_index=True)
Task1_DF=Task1_DF.append({'Name': "Abeer", 'Age': 34, 'Birth_Place': 260}, ignore_index=True)
Task1_DF=Task1_DF.append({'Name': "Muhamad", 'Age': 40, 'Birth_Place': 15}, ignore_index=True)

#view table 2
Task1_DF

#Find the intersection of Original Data and Table 2
merged_inner = pd.merge(left=Task1_DF, right=Original_DF, on=['Age', 'Birth_Place'])
merged_inner

"""#Sol of Task 1
###We can see above that Ahmed income is 983K, Ali is 562K, Abeer is 536K, while Muhamad and Fatma cannot be determined due to different solutions, but we have narrowed their possibilities. If we had more attribute we could have got their salaries.

-------------------------------------------------------------------

#Task 2 - K-anonymization Implementation
Implement the greedy partitioning algorithm that was discussed
in the class using your preferred programming language. The sensitive attribute is Income. The remaining attributes are QuasiIdentifiers. The steps of the algorithm is shown in Figure 1.
Please read below for instructions on how to find and select the
mean value.
"""

#Anonymize(partition) - done
#if (no allowable multidimensional cut for partition) - done
  #return Ø : partition → summary - done
#else{ - done
  #  dim ← choose_dimension() - done
  #  fs ← frequency_set(partition, dim) - done
  #  splitVal ← find_median(fs) - done
  #  lhs ← {t Є partition : t:dim ≤ split} - done
  #  rhs ← {t Є partition : t:dim > split} - done
  #  return Anonymize(rhs) ∪ Anonymize(lhs) - done
#} - done

"""(a) Number of records (n) is odd: the median is the value at
the position n+1/2 of the sorted list of values.
(b) Number of records (n) is even:
i. Find the value at position n/2
ii. Find the value at position n/2 + 1
iii. The median is either of them.
*the median is the value at the position n/2 of the sorted list

###Note 
if you would like to try the functions and see their output, you can put print statement inside them and call them. You need to run them before calling them. I believe you should only use Numeric Quasi values for the HW code to work. Always assume the data you use in algorithm is similar to HW. For example, Columns of Quasi comes first and at the end the column of Sensetive values. Quasi values should be always numeric, but Sensetive I think can be any. Furthermore, it may not work if the data has other values than Sensetive and Quasi (Like unsensitive or Categorical). All these are assumptions, and may be wrong or true.
"""

#Initialize parameters of algorithm. YOU HAVE TO SET THEM.
K = 9 #K Value, Can be changed later no worries
cycles = 0 #Should always be 0 when first calling algorithm. Used to check if cant cut partition.
Sensitive = ['Income_(K)']
Quasi = ["Age", "Gender", "Marital", "Race_Status", "Birth_Place", "Language", "Occupation"]
Columns = ["Age", "Gender", "Marital", "Race_Status", "Birth_Place", "Language", "Occupation", "Income_(K)"]

#A function of Anon Algorithm. 
#it returns text -Col name-, where dim_count between length of Quasi indexes
#if the program is running ok, cycles<1, return dimension with highest Entropy (Shannon Entropy). 
#Why did I choose this method? To have the best cuts. Shanon Entropy tell us which data column has the highest evenly distrbuted data.
#I thought first of using the Standard Div, but it is not good if the data is not distrbuted, as it depends heavely on mean, which can be
#affected by outliers and make bad cuts in terms of privacy/utility.
#If program is not running ok (Old_Part == Part or Cant Cut Part), 
#return Random Quasi column
def choose_dimension(table, cycles):
  Entropies = []
  for i in range(len(Quasi)):
    series = table[Quasi[i]]
    series = series.value_counts()
    ColEntropy = entropy(series)
    Entropies.append(ColEntropy)
  if (cycles == 0):
   return Quasi[Entropies.index(max(Entropies))]
  else:
   return Quasi[random.randint(0, len(Quasi)-1)]

#A function of Anon Algorithm. 
#it returns frequency table of a dim.
#IT IS NO LONGER USED BECAUSE IT MAKES Anonymize() INEFFICIENT.
def frequency_set(partition, dim):
  fs = partition[dim].value_counts().to_frame().reset_index().rename(columns = {dim:'Count','index':dim}).sort_values(by=[dim])
  return fs

#A function of Anon Algorithm.
#it returns median value of a column/pandas series based on HW criteria
def find_median(partition, dim):
  if (len(partition[dim])%2 != 0): #if records odd
    return np.median(partition[dim])
  else:
    return partition[dim].sort_values().iloc[int(len(partition[dim])/2-1)] #if even

#A function of Anon Algorithm.
#it returns True if the algorithm cant cut partition. 
#It is impossible to cut partition if the records are less than or equal K
def Cant_Cut(K, partition):
  if(len(partition.index) <= K):
    return True
  else:
    return False

#A function of Anon Algorithm.
#it returns a summarized partition. The function takes a partition, 
#extracts the Numerical values of Quasi columns, and it returns 
#a categorical table with upper and lower bound values of that table for
#all Quasi columns.
def Summarize(partition):
  if partition.empty:
    return partition
  else:
    for i in range(len(partition.columns)-1):
      partition[Quasi[i]] = pd.cut(partition[Quasi[i]], bins=[partition[Quasi[i]].min(), partition[Quasi[i]].max()+1], right=False)
    return partition

#Define calls and Anonymize Algorithm
def Anonymize(partition, K):
  global Empty_Summary # Reference empty pandas dataframe to use for summary.
  global cycles #A variable to use as a counter if partition or algorithm bugged.
  if (Cant_Cut(K, partition) or cycles>=len(Quasi)): #if partition is minimum (cant cut more) or algorithm bugged
    cycles = 0
    Empty_Summary=Empty_Summary.append(Summarize(partition), ignore_index=True) #Append to output
    #print(Empty_Summary) #Remove this comment if you want to see the process of Anonymize.
    return
  else: #function calls and logic
    dim = choose_dimension(partition, cycles) #get dim
    #fs = frequency_set(partition, dim) # NOT NEEDED, INEFFICIENT.
    splitVal = find_median(partition, dim) #get median
    lhs = partition[partition[dim] <= splitVal]
    rhs = partition[partition[dim] > splitVal]
    if (Cant_Cut(K, lhs) or Cant_Cut(K, rhs)): #dont allow cuts not allowed (partition less then K)
      cycles = cycles+1 #increment bugged factor
      Anonymize(partition, K) #find other random cut if possible
      return
    else:
      cycles = 0
    Anonymize(lhs, K) #keep partioning
    Anonymize(rhs, K) #keep partioning
    return

#Initialize empty dataframe, so you can append the result of summarized partition
#Get Copy of HW data set
#apply alogrithm and save into Colab (Access from left panel).
#Repeat
#Uncomment the code if you want to try something.
'''
K=3
Empty_Summary=pd.DataFrame(columns=Columns)
Partition = Original_DF.copy()
Anonymize(Partition,K)
Empty_Summary.to_csv('Anon-K{}.csv'.format(K), index=False)
K=5
Empty_Summary=pd.DataFrame(columns=Columns)
Partition = Original_DF.copy()
Anonymize(Partition,K)
Empty_Summary.to_csv('Anon-K{}.csv'.format(K), index=False)
K=7
Empty_Summary=pd.DataFrame(columns=Columns)
Partition = Original_DF.copy()
Anonymize(Partition,K)
Empty_Summary.to_csv('Anon-K{}.csv'.format(K), index=False)
K=9
Empty_Summary=pd.DataFrame(columns=Columns)
Partition = Original_DF.copy()
Anonymize(Partition,K)
Empty_Summary.to_csv('Anon-K{}.csv'.format(K), index=False)
'''

#Check empty_set from after applying the function.
#Empty_Summary

#Load output of Greedy Algorithm for K3.
K3_Greedy = pd.read_csv("https://github.com/Alnasser0/COE426-HWs/raw/master/HW1/Anon-K3.csv")

#Validate that Empty_Set (Output) has classes >=K
def Validate(Table, K): #This algorithm checks every class if they violate k rule.
  i=0 #index of row
  while (i < len(Table.index)): #Stop when you reach record 20k, since last index 19999
    selected_row=Table.iloc[i,:] #get a row.
    same_class = Table[ #Get All Rows that have same Quasi Values (Same Class)
            (Table.Age==selected_row.Age)
          & (Table.Gender==selected_row.Gender) 
          & (Table.Marital==selected_row.Marital)
          & (Table.Race_Status==selected_row.Race_Status)
          & (Table.Birth_Place==selected_row.Birth_Place)
          & (Table.Language==selected_row.Language)
          & (Table.Occupation==selected_row.Occupation)
    ]
    number = len(same_class) #Get How many records in a class
    if (number < K): #Return Violation and Class if values less than K
      return same_class, True
    i=i+len(same_class.index) #else continue iteration through the all classes
  return _, False #if no violation is found, return to user.

#test above algorithm
K=3
Table, Output_Boolean = Validate(K3_Greedy, K) #call method with call.
if (Output_Boolean):
  print("Greedy Algorithm does violate the rule of K={}, and the class is: \n".format(K))
  print(Table)
else:
  print("The Algorithm works ok and your table is safe! :)")

"""#Task 3 - Utility Privacy Trade-off
###Using your implementation of the anonymization algorithm in
###Task2, find the anonymized table with k=3,5,7, and 9.
###For each anonymized table compute the Discernibility metric
###CDM and generalized information loss ILOSS given by the following equations.
###Then, draw a figure for each metric against the value of k to
###depict the privacy trade off. The x-axis should be the value of K, Y=metric
"""

#These tables I made using ARX
K3_ARX = pd.read_csv("https://raw.githubusercontent.com/Alnasser0/COE426-HWs/master/HW1/ARX-K3.csv")
K5_ARX = pd.read_csv("https://raw.githubusercontent.com/Alnasser0/COE426-HWs/master/HW1/ARX-K5.csv")
K7_ARX = pd.read_csv("https://raw.githubusercontent.com/Alnasser0/COE426-HWs/master/HW1/ARX-K7.csv")
K9_ARX = pd.read_csv("https://raw.githubusercontent.com/Alnasser0/COE426-HWs/master/HW1/ARX-K9.csv")

#These tables I made using above Greedy Algorithm
K3_Greedy = pd.read_csv("https://github.com/Alnasser0/COE426-HWs/raw/master/HW1/Anon-K3.csv")
K5_Greedy = pd.read_csv("https://github.com/Alnasser0/COE426-HWs/raw/master/HW1/Anon-K5.csv")
K7_Greedy = pd.read_csv("https://github.com/Alnasser0/COE426-HWs/raw/master/HW1/Anon-K7.csv")
K9_Greedy = pd.read_csv("https://github.com/Alnasser0/COE426-HWs/raw/master/HW1/Anon-K9.csv")

"""You can use EITHER tables, my algorithm works on both :)
- Make sure you set the correct parameters from Task 2.
"""

def CDM(Table):
  CDM = 0
  i=0
  while (i < len(Table.index)): #Stop when you reach record 20k, since last index 19999
    selected_row=Table.iloc[i,:] #Select One Row
    same_class = Table[ #Get All Rows that have same Quasi Values (Same Class)
           (Table.Age==selected_row.Age)
         & (Table.Gender==selected_row.Gender) 
         & (Table.Marital==selected_row.Marital)
         & (Table.Race_Status==selected_row.Race_Status)
         & (Table.Birth_Place==selected_row.Birth_Place)
         & (Table.Language==selected_row.Language)
         & (Table.Occupation==selected_row.Occupation)
    ]
    CDM = CDM + np.power(len(same_class.index), 2) #Apply Equation to a Class
    i=i+len(same_class.index) #Increment Selected Row.
  return CDM #Return CDM Value

#This function calculates Denominator of I_LOSS
def Get_Denominator(table, attribute): #it takes Quasi + Table
  Attribute_Column = table[Quasi[attribute]] #it selects Attribute
  Attribute_Column = Attribute_Column.sort_values() #it sorts values
  Attribute_Column = Attribute_Column[Attribute_Column != "*"] #Ignore * values for ARX
  Attribute_Column = Attribute_Column[~Attribute_Column.str.contains(">=")] #Ignore >= values for ARX
  Ui = Attribute_Column.iloc[len(Attribute_Column)-1] #Get Biggest Value in that Column.
  Ui = [int(s) for s in re.findall('\d+', Ui)] #Convert Text Value to Numeric List
  Ui = Ui[1]-1 #Get Upper Bound Value
  Li = Attribute_Column.iloc[0] #Get Smallest Value in that Column
  Li = [int(s) for s in re.findall('\d+', Li)] #Convert Text Value to Numeric List
  Li = Li[0] #Get Lower Bound Value
  Denominator = Ui-Li #Find Denominator
  return Denominator #Return Denominator of that Column

def Get_Numerator(i, table, attribute_selection): #ith row, a table, attribute
  selected_row=table.iloc[i,:] #Select a row
  same_class = table[ #Get All Rows that have same Quasi Values (Same Class)
           (table.Age==selected_row.Age)
         & (table.Gender==selected_row.Gender) 
         & (table.Marital==selected_row.Marital)
         & (table.Race_Status==selected_row.Race_Status)
         & (table.Birth_Place==selected_row.Birth_Place)
         & (table.Language==selected_row.Language)
         & (table.Occupation==selected_row.Occupation)
    ]
  E = len(same_class.index) #Find Length of E Class (# of Records)
  i=i+len(same_class.index) #Shift/Increment Selected Row.
  cell = same_class.iloc[0,attribute_selection] #Get cell values in a Class
  if (("*" in cell) or (">=" in cell)): #ignore * and >= (for ARX)
    return 0, 0, i #Return No Values, just i shift.
  Cell_Values = [int(s) for s in re.findall('\d+', cell)] #Get a cell values (We Assume all cells we have in Quasi columns contain 2 values (upper and lower) or have (*/>=) values!)
  UpperCell = Cell_Values[1]-1 #Get Upper Bound Value
  LowerCell = Cell_Values[0] #Get Lower Bound Value
  Numerator = UpperCell-LowerCell #Find Numerator
  return Numerator, E, i #Return Numerator, Class Size, and shifted i.

def GenCalc(table): #Loss function, only takes a table
  GenLoss = 0 #intilize Gen. Loss.
  AttributeLength = len(Quasi) #Get Columns length
  attribute_selection = 0 #Initialize iterate value to go over columns
  Cardinality_Table = len(table.index) #Get T Value
  Coefficient = 1/(AttributeLength*Cardinality_Table) #Calculate Coefficient
  i=0 #Start from First Row.
  while (attribute_selection < AttributeLength): #Iterate over all columns, and for each do.
    Den = Get_Denominator(table, attribute_selection) #Get Get_Denominator for that column.
    while (i < len(table.index)): #Find Loss of each class for the selected column. #Stop when you reach record 20k, since last index 19999
      Nom, E, i = Get_Numerator(i, table, attribute_selection) #Get Numerator of a Class with Class Size and Shifted Position.
      GenLoss = GenLoss + (E*Nom)/Den #Calculate Loss of a complete class in a column. Then, continue doing it for all classes.
    attribute_selection = attribute_selection+1 #do the same thing for next column
    i = 0 #Reset Row Selection
  return Coefficient*GenLoss #Return Value of Loss.

#Call CDM function on a set of tables and store their result in an array.
#These calculations will take time, uncomment and run on your responsibility

#CDM_Answer_Greedy = [CDM(K3_Greedy),CDM(K5_Greedy),CDM(K7_Greedy),CDM(K9_Greedy)]
#CDM_Answer_ARX = [CDM(K3_ARX),CDM(K5_ARX),CDM(K7_ARX),CDM(K9_ARX)]

#Call I_Loss function on a set of tables and store their result in an array.
#These calculations will take time, uncomment and run on your responsibility

#I_Loss_Greedy = [GenCalc(K3_Greedy),GenCalc(K5_Greedy),GenCalc(K7_Greedy),GenCalc(K9_Greedy)]
#I_Loss_ARX = [GenCalc(K3_ARX),GenCalc(K5_ARX),GenCalc(K7_ARX),GenCalc(K9_ARX)]

#Define K Values in array.
K = [3, 5, 7, 9]

#Store values in a Frame to Plot them.
CDM_Frame_Greedy = pd.DataFrame(data={'K':K,'CDM':CDM_Answer_Greedy})
CDM_Frame_Greedy

#Store values in a Frame to Plot them.
CDM_Frame_ARX = pd.DataFrame(data={'K':K,'CDM':CDM_Answer_ARX})
CDM_Frame_ARX

#Store values in a Frame to Plot them.
I_Loss_Frame_Greedy = pd.DataFrame(data={'K':K,'I_Loss':I_Loss_Greedy})
I_Loss_Frame_Greedy

#Store values in a Frame to Plot them.
I_Loss_Frame_ARX = pd.DataFrame(data={'K':K,'I_Loss':I_Loss_ARX})
I_Loss_Frame_ARX

"""Most plots show that Loss increases as K increases, which reduces utility and increases privacy. Arx with l=2 has better I_Loss and Greedy algorithm has better CDM.

### Greedy Algorithm Plots
"""

fig = px.line(CDM_Frame_Greedy, x="K", y="CDM")
fig.show()

fig = px.line(I_Loss_Frame_Greedy, x="K", y="I_Loss")
fig.show()

"""### ARX Plots"""

fig = px.line(CDM_Frame_ARX, x="K", y="CDM")
fig.show()

fig = px.line(I_Loss_Frame_ARX, x="K", y="I_Loss")
fig.show()

"""#Task4: l-diversity
###Using the anonymized table with k = 9 from Task 3, check if
###the 9-anonymized table is distinct l-diverse for each l = 2 and 5.
###In the case when the 9-anonymized table violates the l-diversity
###requirements, print at least one equivalence class that violates
###the diversity requirement.
"""

#Each should have class at least l' different values of Income(K) (>= l)

#Load Task 3 Tables
Arx_Table = pd.read_csv("https://github.com/Alnasser0/COE426-HWs/raw/master/HW1/ARX-K9.csv")
Greedy_Table = pd.read_csv("https://github.com/Alnasser0/COE426-HWs/raw/master/HW1/Anon-K9.csv")

def DoesViolateLdistinct(Table, l): #This algorithm checks every class if they violate l rule.
  i=0 #index of row
  while (i < len(Table.index)): #Stop when you reach record 20k, since last index 19999
    selected_row=Table.iloc[i,:] #get a row.
    same_class = Table[ #Get All Rows that have same Quasi Values (Same Class)
            (Table.Age==selected_row.Age)
          & (Table.Gender==selected_row.Gender) 
          & (Table.Marital==selected_row.Marital)
          & (Table.Race_Status==selected_row.Race_Status)
          & (Table.Birth_Place==selected_row.Birth_Place)
          & (Table.Language==selected_row.Language)
          & (Table.Occupation==selected_row.Occupation)
      ]
    if (all((~same_class.Age.str.contains("\*"))) and all((~same_class.Age.str.contains(">=")))): #Dont Check classes with * or >= values (For ARX)
      number = len(same_class["Income_(K)"].unique()) #Get How many unique sensetive values we have in a class
      if (number < l): #Return Violation and Class if unique values is less than l
       return same_class, True
    i=i+len(same_class.index) #else continue iteration through the all classes
  return _, False #if no violation is found, return to user.

Greedy_Class_2, Greedy_Boolean_2 = DoesViolateLdistinct(Greedy_Table, 2)
Greedy_Class_5, Greedy_Boolean_5 = DoesViolateLdistinct(Greedy_Table, 5)
ARX_Class_2, ARX_Boolean_2 = DoesViolateLdistinct(Arx_Table, 2)
ARX_Class_5, ARX_Boolean_5 = DoesViolateLdistinct(Arx_Table, 5)

if (Greedy_Boolean_2):
  print("Greedy Algorithm does violate the rule of l=2, and the class is: \n")
  print(Greedy_Class_2)
if (Greedy_Boolean_5):
  print("\nGreedy Algorithm does violate the rule of l=5, and the class is: \n")
  print(Greedy_Class_5)
if (ARX_Boolean_2):
  print("\nARX output does violate the rule of l=2, and the class is: \n")
  print(ARX_Class_2)
if (ARX_Boolean_5):
  print("\nARX output does violate the rule of l=5, and the class is: \n")
  print(ARX_Class_5)
# -*- coding: utf-8 -*-
"""HW1 - COE426.ipynb

Automatically generated by Colaboratory.

Original file is located at
    

#<NAME>
#201535050
"""

#import all needed libraries
import pandas as pd
import numpy as np
import sys, resource
import re
import plotly.express as px
from scipy.stats import entropy
import random

#unlock colab resources for infinite memory and recursion :)
resource.setrlimit(resource.RLIMIT_STACK, (2**29,-1))
sys.setrecursionlimit(10**6)

#Set Columns of data set
Columns = ["Age", "Gender", "Marital", "Race_Status", "Birth_Place", "Language", "Occupation", "Income_(K)"]

#Load raw data set and append columns to it
Original_DF = pd.read_csv("https://raw.githubusercontent.com/Alnasser0/COE426-HWs/master/HW1/ipums-assign1.csv", names=Columns)

#View data set
Original_DF

#Export data set (Can be viewed/download from left panel)
Original_DF.to_csv('tablewithcolumns.csv', index=False)

"""#Task1: Linkage attack (20 pts)
Download the file named ”ipums.txt” from blackboard and unzip
it. Using Table 2 as external background information, perform
a data linkage attack to **find the annual salary of each person in
the table**. You are free to use any tool/programming language to
complete this task, e.g., Excel, Python, Java, etc.
"""

#Load columns of table 2
Columns2 = ["Name", "Age", "Birth_Place"]

#Create a data frame and fill data of table 2
Task1_DF = pd.DataFrame(columns=Columns2)
Task1_DF=Task1_DF.append({'Name': "Ahmed", 'Age': 28, 'Birth_Place': 110}, ignore_index=True)
Task1_DF=Task1_DF.append({'Name': "Fatma", 'Age': 44, 'Birth_Place': 4}, ignore_index=True)
Task1_DF=Task1_DF.append({'Name': "Ali", 'Age': 17, 'Birth_Place': 199}, ignore_index=True)
Task1_DF=Task1_DF.append({'Name': "Abeer", 'Age': 34, 'Birth_Place': 260}, ignore_index=True)
Task1_DF=Task1_DF.append({'Name': "Muhamad", 'Age': 40, 'Birth_Place': 15}, ignore_index=True)

#view table 2
Task1_DF

#Find the intersection of Original Data and Table 2
merged_inner = pd.merge(left=Task1_DF, right=Original_DF, on=['Age', 'Birth_Place'])
merged_inner

"""#Sol of Task 1
###We can see above that Ahmed income is 983K, Ali is 562K, Abeer is 536K, while Muhamad and Fatma cannot be determined due to different solutions, but we have narrowed their possibilities. If we had more attribute we could have got their salaries.

-------------------------------------------------------------------

#Task 2 - K-anonymization Implementation
Implement the greedy partitioning algorithm that was discussed
in the class using your preferred programming language. The sensitive attribute is Income. The remaining attributes are QuasiIdentifiers. The steps of the algorithm is shown in Figure 1.
Please read below for instructions on how to find and select the
mean value.
"""

#Anonymize(partition) - done
#if (no allowable multidimensional cut for partition) - done
  #return Ø : partition → summary - done
#else{ - done
  #  dim ← choose_dimension() - done
  #  fs ← frequency_set(partition, dim) - done
  #  splitVal ← find_median(fs) - done
  #  lhs ← {t Є partition : t:dim ≤ split} - done
  #  rhs ← {t Є partition : t:dim > split} - done
  #  return Anonymize(rhs) ∪ Anonymize(lhs) - done
#} - done

"""(a) Number of records (n) is odd: the median is the value at
the position n+1/2 of the sorted list of values.
(b) Number of records (n) is even:
i. Find the value at position n/2
ii. Find the value at position n/2 + 1
iii. The median is either of them.
*the median is the value at the position n/2 of the sorted list

###Note 
if you would like to try the functions and see their output, you can put print statement inside them and call them. You need to run them before calling them. I believe you should only use Numeric Quasi values for the HW code to work. Always assume the data you use in algorithm is similar to HW. For example, Columns of Quasi comes first and at the end the column of Sensetive values. Quasi values should be always numeric, but Sensetive I think can be any. Furthermore, it may not work if the data has other values than Sensetive and Quasi (Like unsensitive or Categorical). All these are assumptions, and may be wrong or true.
"""

#Initialize parameters of algorithm. YOU HAVE TO SET THEM.
K = 9 #K Value, Can be changed later no worries
cycles = 0 #Should always be 0 when first calling algorithm. Used to check if cant cut partition.
Sensitive = ['Income_(K)']
Quasi = ["Age", "Gender", "Marital", "Race_Status", "Birth_Place", "Language", "Occupation"]
Columns = ["Age", "Gender", "Marital", "Race_Status", "Birth_Place", "Language", "Occupation", "Income_(K)"]

#A function of Anon Algorithm. 
#it returns text -Col name-, where dim_count between length of Quasi indexes
#if the program is running ok, cycles<1, return dimension with highest Entropy (Shannon Entropy). 
#Why did I choose this method? To have the best cuts. Shanon Entropy tell us which data column has the highest evenly distrbuted data.
#I thought first of using the Standard Div, but it is not good if the data is not distrbuted, as it depends heavely on mean, which can be
#affected by outliers and make bad cuts in terms of privacy/utility.
#If program is not running ok (Old_Part == Part or Cant Cut Part), 
#return Random Quasi column
def choose_dimension(table, cycles):
  Entropies = []
  for i in range(len(Quasi)):
    series = table[Quasi[i]]
    series = series.value_counts()
    ColEntropy = entropy(series)
    Entropies.append(ColEntropy)
  if (cycles == 0):
   return Quasi[Entropies.index(max(Entropies))]
  else:
   return Quasi[random.randint(0, len(Quasi)-1)]

#A function of Anon Algorithm. 
#it returns frequency table of a dim.
#IT IS NO LONGER USED BECAUSE IT MAKES Anonymize() INEFFICIENT.
def frequency_set(partition, dim):
  fs = partition[dim].value_counts().to_frame().reset_index().rename(columns = {dim:'Count','index':dim}).sort_values(by=[dim])
  return fs

#A function of Anon Algorithm.
#it returns median value of a column/pandas series based on HW criteria
def find_median(partition, dim):
  if (len(partition[dim])%2 != 0): #if records odd
    return np.median(partition[dim])
  else:
    return partition[dim].sort_values().iloc[int(len(partition[dim])/2-1)] #if even

#A function of Anon Algorithm.
#it returns True if the algorithm cant cut partition. 
#It is impossible to cut partition if the records are less than or equal K
def Cant_Cut(K, partition):
  if(len(partition.index) <= K):
    return True
  else:
    return False

#A function of Anon Algorithm.
#it returns a summarized partition. The function takes a partition, 
#extracts the Numerical values of Quasi columns, and it returns 
#a categorical table with upper and lower bound values of that table for
#all Quasi columns.
def Summarize(partition):
  if partition.empty:
    return partition
  else:
    for i in range(len(partition.columns)-1):
      partition[Quasi[i]] = pd.cut(partition[Quasi[i]], bins=[partition[Quasi[i]].min(), partition[Quasi[i]].max()+1], right=False)
    return partition

#Define calls and Anonymize Algorithm
def Anonymize(partition, K):
  global Empty_Summary # Reference empty pandas dataframe to use for summary.
  global cycles #A variable to use as a counter if partition or algorithm bugged.
  if (Cant_Cut(K, partition) or cycles>=len(Quasi)): #if partition is minimum (cant cut more) or algorithm bugged
    cycles = 0
    Empty_Summary=Empty_Summary.append(Summarize(partition), ignore_index=True) #Append to output
    #print(Empty_Summary) #Remove this comment if you want to see the process of Anonymize.
    return
  else: #function calls and logic
    dim = choose_dimension(partition, cycles) #get dim
    #fs = frequency_set(partition, dim) # NOT NEEDED, INEFFICIENT.
    splitVal = find_median(partition, dim) #get median
    lhs = partition[partition[dim] <= splitVal]
    rhs = partition[partition[dim] > splitVal]
    if (Cant_Cut(K, lhs) or Cant_Cut(K, rhs)): #dont allow cuts not allowed (partition less then K)
      cycles = cycles+1 #increment bugged factor
      Anonymize(partition, K) #find other random cut if possible
      return
    else:
      cycles = 0
    Anonymize(lhs, K) #keep partioning
    Anonymize(rhs, K) #keep partioning
    return

#Initialize empty dataframe, so you can append the result of summarized partition
#Get Copy of HW data set
#apply alogrithm and save into Colab (Access from left panel).
#Repeat
#Uncomment the code if you want to try something.
'''
K=3
Empty_Summary=pd.DataFrame(columns=Columns)
Partition = Original_DF.copy()
Anonymize(Partition,K)
Empty_Summary.to_csv('Anon-K{}.csv'.format(K), index=False)
K=5
Empty_Summary=pd.DataFrame(columns=Columns)
Partition = Original_DF.copy()
Anonymize(Partition,K)
Empty_Summary.to_csv('Anon-K{}.csv'.format(K), index=False)
K=7
Empty_Summary=pd.DataFrame(columns=Columns)
Partition = Original_DF.copy()
Anonymize(Partition,K)
Empty_Summary.to_csv('Anon-K{}.csv'.format(K), index=False)
K=9
Empty_Summary=pd.DataFrame(columns=Columns)
Partition = Original_DF.copy()
Anonymize(Partition,K)
Empty_Summary.to_csv('Anon-K{}.csv'.format(K), index=False)
'''

#Check empty_set from after applying the function.
#Empty_Summary

#Load output of Greedy Algorithm for K3.
K3_Greedy = pd.read_csv("https://github.com/Alnasser0/COE426-HWs/raw/master/HW1/Anon-K3.csv")

#Validate that Empty_Set (Output) has classes >=K
def Validate(Table, K): #This algorithm checks every class if they violate k rule.
  i=0 #index of row
  while (i < len(Table.index)): #Stop when you reach record 20k, since last index 19999
    selected_row=Table.iloc[i,:] #get a row.
    same_class = Table[ #Get All Rows that have same Quasi Values (Same Class)
            (Table.Age==selected_row.Age)
          & (Table.Gender==selected_row.Gender) 
          & (Table.Marital==selected_row.Marital)
          & (Table.Race_Status==selected_row.Race_Status)
          & (Table.Birth_Place==selected_row.Birth_Place)
          & (Table.Language==selected_row.Language)
          & (Table.Occupation==selected_row.Occupation)
    ]
    number = len(same_class) #Get How many records in a class
    if (number < K): #Return Violation and Class if values less than K
      return same_class, True
    i=i+len(same_class.index) #else continue iteration through the all classes
  return _, False #if no violation is found, return to user.

#test above algorithm
K=3
Table, Output_Boolean = Validate(K3_Greedy, K) #call method with call.
if (Output_Boolean):
  print("Greedy Algorithm does violate the rule of K={}, and the class is: \n".format(K))
  print(Table)
else:
  print("The Algorithm works ok and your table is safe! :)")

"""#Task 3 - Utility Privacy Trade-off
###Using your implementation of the anonymization algorithm in
###Task2, find the anonymized table with k=3,5,7, and 9.
###For each anonymized table compute the Discernibility metric
###CDM and generalized information loss ILOSS given by the following equations.
###Then, draw a figure for each metric against the value of k to
###depict the privacy trade off. The x-axis should be the value of K, Y=metric
"""

#These tables I made using ARX
K3_ARX = pd.read_csv("https://raw.githubusercontent.com/Alnasser0/COE426-HWs/master/HW1/ARX-K3.csv")
K5_ARX = pd.read_csv("https://raw.githubusercontent.com/Alnasser0/COE426-HWs/master/HW1/ARX-K5.csv")
K7_ARX = pd.read_csv("https://raw.githubusercontent.com/Alnasser0/COE426-HWs/master/HW1/ARX-K7.csv")
K9_ARX = pd.read_csv("https://raw.githubusercontent.com/Alnasser0/COE426-HWs/master/HW1/ARX-K9.csv")

#These tables I made using above Greedy Algorithm
K3_Greedy = pd.read_csv("https://github.com/Alnasser0/COE426-HWs/raw/master/HW1/Anon-K3.csv")
K5_Greedy = pd.read_csv("https://github.com/Alnasser0/COE426-HWs/raw/master/HW1/Anon-K5.csv")
K7_Greedy = pd.read_csv("https://github.com/Alnasser0/COE426-HWs/raw/master/HW1/Anon-K7.csv")
K9_Greedy = pd.read_csv("https://github.com/Alnasser0/COE426-HWs/raw/master/HW1/Anon-K9.csv")

"""You can use EITHER tables, my algorithm works on both :)
- Make sure you set the correct parameters from Task 2.
"""

def CDM(Table):
  CDM = 0
  i=0
  while (i < len(Table.index)): #Stop when you reach record 20k, since last index 19999
    selected_row=Table.iloc[i,:] #Select One Row
    same_class = Table[ #Get All Rows that have same Quasi Values (Same Class)
           (Table.Age==selected_row.Age)
         & (Table.Gender==selected_row.Gender) 
         & (Table.Marital==selected_row.Marital)
         & (Table.Race_Status==selected_row.Race_Status)
         & (Table.Birth_Place==selected_row.Birth_Place)
         & (Table.Language==selected_row.Language)
         & (Table.Occupation==selected_row.Occupation)
    ]
    CDM = CDM + np.power(len(same_class.index), 2) #Apply Equation to a Class
    i=i+len(same_class.index) #Increment Selected Row.
  return CDM #Return CDM Value

#This function calculates Denominator of I_LOSS
def Get_Denominator(table, attribute): #it takes Quasi + Table
  Attribute_Column = table[Quasi[attribute]] #it selects Attribute
  Attribute_Column = Attribute_Column.sort_values() #it sorts values
  Attribute_Column = Attribute_Column[Attribute_Column != "*"] #Ignore * values for ARX
  Attribute_Column = Attribute_Column[~Attribute_Column.str.contains(">=")] #Ignore >= values for ARX
  Ui = Attribute_Column.iloc[len(Attribute_Column)-1] #Get Biggest Value in that Column.
  Ui = [int(s) for s in re.findall('\d+', Ui)] #Convert Text Value to Numeric List
  Ui = Ui[1]-1 #Get Upper Bound Value
  Li = Attribute_Column.iloc[0] #Get Smallest Value in that Column
  Li = [int(s) for s in re.findall('\d+', Li)] #Convert Text Value to Numeric List
  Li = Li[0] #Get Lower Bound Value
  Denominator = Ui-Li #Find Denominator
  return Denominator #Return Denominator of that Column

def Get_Numerator(i, table, attribute_selection): #ith row, a table, attribute
  selected_row=table.iloc[i,:] #Select a row
  same_class = table[ #Get All Rows that have same Quasi Values (Same Class)
           (table.Age==selected_row.Age)
         & (table.Gender==selected_row.Gender) 
         & (table.Marital==selected_row.Marital)
         & (table.Race_Status==selected_row.Race_Status)
         & (table.Birth_Place==selected_row.Birth_Place)
         & (table.Language==selected_row.Language)
         & (table.Occupation==selected_row.Occupation)
    ]
  E = len(same_class.index) #Find Length of E Class (# of Records)
  i=i+len(same_class.index) #Shift/Increment Selected Row.
  cell = same_class.iloc[0,attribute_selection] #Get cell values in a Class
  if (("*" in cell) or (">=" in cell)): #ignore * and >= (for ARX)
    return 0, 0, i #Return No Values, just i shift.
  Cell_Values = [int(s) for s in re.findall('\d+', cell)] #Get a cell values (We Assume all cells we have in Quasi columns contain 2 values (upper and lower) or have (*/>=) values!)
  UpperCell = Cell_Values[1]-1 #Get Upper Bound Value
  LowerCell = Cell_Values[0] #Get Lower Bound Value
  Numerator = UpperCell-LowerCell #Find Numerator
  return Numerator, E, i #Return Numerator, Class Size, and shifted i.

def GenCalc(table): #Loss function, only takes a table
  GenLoss = 0 #intilize Gen. Loss.
  AttributeLength = len(Quasi) #Get Columns length
  attribute_selection = 0 #Initialize iterate value to go over columns
  Cardinality_Table = len(table.index) #Get T Value
  Coefficient = 1/(AttributeLength*Cardinality_Table) #Calculate Coefficient
  i=0 #Start from First Row.
  while (attribute_selection < AttributeLength): #Iterate over all columns, and for each do.
    Den = Get_Denominator(table, attribute_selection) #Get Get_Denominator for that column.
    while (i < len(table.index)): #Find Loss of each class for the selected column. #Stop when you reach record 20k, since last index 19999
      Nom, E, i = Get_Numerator(i, table, attribute_selection) #Get Numerator of a Class with Class Size and Shifted Position.
      GenLoss = GenLoss + (E*Nom)/Den #Calculate Loss of a complete class in a column. Then, continue doing it for all classes.
    attribute_selection = attribute_selection+1 #do the same thing for next column
    i = 0 #Reset Row Selection
  return Coefficient*GenLoss #Return Value of Loss.

#Call CDM function on a set of tables and store their result in an array.
#These calculations will take time, uncomment and run on your responsibility

#CDM_Answer_Greedy = [CDM(K3_Greedy),CDM(K5_Greedy),CDM(K7_Greedy),CDM(K9_Greedy)]
#CDM_Answer_ARX = [CDM(K3_ARX),CDM(K5_ARX),CDM(K7_ARX),CDM(K9_ARX)]

#Call I_Loss function on a set of tables and store their result in an array.
#These calculations will take time, uncomment and run on your responsibility

#I_Loss_Greedy = [GenCalc(K3_Greedy),GenCalc(K5_Greedy),GenCalc(K7_Greedy),GenCalc(K9_Greedy)]
#I_Loss_ARX = [GenCalc(K3_ARX),GenCalc(K5_ARX),GenCalc(K7_ARX),GenCalc(K9_ARX)]

#Define K Values in array.
K = [3, 5, 7, 9]

#Store values in a Frame to Plot them.
CDM_Frame_Greedy = pd.DataFrame(data={'K':K,'CDM':CDM_Answer_Greedy})
CDM_Frame_Greedy

#Store values in a Frame to Plot them.
CDM_Frame_ARX = pd.DataFrame(data={'K':K,'CDM':CDM_Answer_ARX})
CDM_Frame_ARX

#Store values in a Frame to Plot them.
I_Loss_Frame_Greedy = pd.DataFrame(data={'K':K,'I_Loss':I_Loss_Greedy})
I_Loss_Frame_Greedy

#Store values in a Frame to Plot them.
I_Loss_Frame_ARX = pd.DataFrame(data={'K':K,'I_Loss':I_Loss_ARX})
I_Loss_Frame_ARX

"""Most plots show that Loss increases as K increases, which reduces utility and increases privacy. Arx with l=2 has better I_Loss and Greedy algorithm has better CDM.

### Greedy Algorithm Plots
"""

fig = px.line(CDM_Frame_Greedy, x="K", y="CDM")
fig.show()

fig = px.line(I_Loss_Frame_Greedy, x="K", y="I_Loss")
fig.show()

"""### ARX Plots"""

fig = px.line(CDM_Frame_ARX, x="K", y="CDM")
fig.show()

fig = px.line(I_Loss_Frame_ARX, x="K", y="I_Loss")
fig.show()

"""#Task4: l-diversity
###Using the anonymized table with k = 9 from Task 3, check if
###the 9-anonymized table is distinct l-diverse for each l = 2 and 5.
###In the case when the 9-anonymized table violates the l-diversity
###requirements, print at least one equivalence class that violates
###the diversity requirement.
"""

#Each should have class at least l' different values of Income(K) (>= l)

#Load Task 3 Tables
Arx_Table = pd.read_csv("https://github.com/Alnasser0/COE426-HWs/raw/master/HW1/ARX-K9.csv")
Greedy_Table = pd.read_csv("https://github.com/Alnasser0/COE426-HWs/raw/master/HW1/Anon-K9.csv")

def DoesViolateLdistinct(Table, l): #This algorithm checks every class if they violate l rule.
  i=0 #index of row
  while (i < len(Table.index)): #Stop when you reach record 20k, since last index 19999
    selected_row=Table.iloc[i,:] #get a row.
    same_class = Table[ #Get All Rows that have same Quasi Values (Same Class)
            (Table.Age==selected_row.Age)
          & (Table.Gender==selected_row.Gender) 
          & (Table.Marital==selected_row.Marital)
          & (Table.Race_Status==selected_row.Race_Status)
          & (Table.Birth_Place==selected_row.Birth_Place)
          & (Table.Language==selected_row.Language)
          & (Table.Occupation==selected_row.Occupation)
      ]
    if (all((~same_class.Age.str.contains("\*"))) and all((~same_class.Age.str.contains(">=")))): #Dont Check classes with * or >= values (For ARX)
      number = len(same_class["Income_(K)"].unique()) #Get How many unique sensetive values we have in a class
      if (number < l): #Return Violation and Class if unique values is less than l
       return same_class, True
    i=i+len(same_class.index) #else continue iteration through the all classes
  return _, False #if no violation is found, return to user.

Greedy_Class_2, Greedy_Boolean_2 = DoesViolateLdistinct(Greedy_Table, 2)
Greedy_Class_5, Greedy_Boolean_5 = DoesViolateLdistinct(Greedy_Table, 5)
ARX_Class_2, ARX_Boolean_2 = DoesViolateLdistinct(Arx_Table, 2)
ARX_Class_5, ARX_Boolean_5 = DoesViolateLdistinct(Arx_Table, 5)

if (Greedy_Boolean_2):
  print("Greedy Algorithm does violate the rule of l=2, and the class is: \n")
  print(Greedy_Class_2)
if (Greedy_Boolean_5):
  print("\nGreedy Algorithm does violate the rule of l=5, and the class is: \n")
  print(Greedy_Class_5)
if (ARX_Boolean_2):
  print("\nARX output does violate the rule of l=2, and the class is: \n")
  print(ARX_Class_2)
if (ARX_Boolean_5):
  print("\nARX output does violate the rule of l=5, and the class is: \n")
  print(ARX_Class_5)
# -*- coding: utf-8 -*- HW1 - COE426.ipynb  Automatically generated by Colaboratory.  Original file is located at       #<NAME> #201535050 #import all needed libraries #unlock colab resources for infinite memory and recursion :) #Set Columns of data set #Load raw data set and append columns to it #View data set #Export data set (Can be viewed/download from left panel) #Task1: Linkage attack (20 pts) Download the file named ”ipums.txt” from blackboard and unzip it. Using Table 2 as external background information, perform a data linkage attack to **find the annual salary of each person in the table**. You are free to use any tool/programming language to complete this task, e.g., Excel, Python, Java, etc. #Load columns of table 2 #Create a data frame and fill data of table 2 #view table 2 #Find the intersection of Original Data and Table 2 #Sol of Task 1 ###We can see above that Ahmed income is 983K, Ali is 562K, Abeer is 536K, while Muhamad and Fatma cannot be determined due to different solutions, but we have narrowed their possibilities. If we had more attribute we could have got their salaries.  -------------------------------------------------------------------  #Task 2 - K-anonymization Implementation Implement the greedy partitioning algorithm that was discussed in the class using your preferred programming language. The sensitive attribute is Income. The remaining attributes are QuasiIdentifiers. The steps of the algorithm is shown in Figure 1. Please read below for instructions on how to find and select the mean value. #Anonymize(partition) - done #if (no allowable multidimensional cut for partition) - done #return Ø : partition → summary - done #else{ - done #  dim ← choose_dimension() - done #  fs ← frequency_set(partition, dim) - done #  splitVal ← find_median(fs) - done #  lhs ← {t Є partition : t:dim ≤ split} - done #  rhs ← {t Є partition : t:dim > split} - done #  return Anonymize(rhs) ∪ Anonymize(lhs) - done #} - done (a) Number of records (n) is odd: the median is the value at the position n+1/2 of the sorted list of values. (b) Number of records (n) is even: i. Find the value at position n/2 ii. Find the value at position n/2 + 1 iii. The median is either of them. *the median is the value at the position n/2 of the sorted list  ###Note  if you would like to try the functions and see their output, you can put print statement inside them and call them. You need to run them before calling them. I believe you should only use Numeric Quasi values for the HW code to work. Always assume the data you use in algorithm is similar to HW. For example, Columns of Quasi comes first and at the end the column of Sensetive values. Quasi values should be always numeric, but Sensetive I think can be any. Furthermore, it may not work if the data has other values than Sensetive and Quasi (Like unsensitive or Categorical). All these are assumptions, and may be wrong or true. #Initialize parameters of algorithm. YOU HAVE TO SET THEM. #K Value, Can be changed later no worries #Should always be 0 when first calling algorithm. Used to check if cant cut partition. #A function of Anon Algorithm. #it returns text -Col name-, where dim_count between length of Quasi indexes #if the program is running ok, cycles<1, return dimension with highest Entropy (Shannon Entropy). #Why did I choose this method? To have the best cuts. Shanon Entropy tell us which data column has the highest evenly distrbuted data. #I thought first of using the Standard Div, but it is not good if the data is not distrbuted, as it depends heavely on mean, which can be #affected by outliers and make bad cuts in terms of privacy/utility. #If program is not running ok (Old_Part == Part or Cant Cut Part), #return Random Quasi column #A function of Anon Algorithm. #it returns frequency table of a dim. #IT IS NO LONGER USED BECAUSE IT MAKES Anonymize() INEFFICIENT. #A function of Anon Algorithm. #it returns median value of a column/pandas series based on HW criteria #if records odd #if even #A function of Anon Algorithm. #it returns True if the algorithm cant cut partition. #It is impossible to cut partition if the records are less than or equal K #A function of Anon Algorithm. #it returns a summarized partition. The function takes a partition, #extracts the Numerical values of Quasi columns, and it returns #a categorical table with upper and lower bound values of that table for #all Quasi columns. #Define calls and Anonymize Algorithm # Reference empty pandas dataframe to use for summary. #A variable to use as a counter if partition or algorithm bugged. #if partition is minimum (cant cut more) or algorithm bugged #Append to output #print(Empty_Summary) #Remove this comment if you want to see the process of Anonymize. #function calls and logic #get dim #fs = frequency_set(partition, dim) # NOT NEEDED, INEFFICIENT. #get median #dont allow cuts not allowed (partition less then K) #increment bugged factor #find other random cut if possible #keep partioning #keep partioning #Initialize empty dataframe, so you can append the result of summarized partition #Get Copy of HW data set #apply alogrithm and save into Colab (Access from left panel). #Repeat #Uncomment the code if you want to try something. K=3 Empty_Summary=pd.DataFrame(columns=Columns) Partition = Original_DF.copy() Anonymize(Partition,K) Empty_Summary.to_csv('Anon-K{}.csv'.format(K), index=False) K=5 Empty_Summary=pd.DataFrame(columns=Columns) Partition = Original_DF.copy() Anonymize(Partition,K) Empty_Summary.to_csv('Anon-K{}.csv'.format(K), index=False) K=7 Empty_Summary=pd.DataFrame(columns=Columns) Partition = Original_DF.copy() Anonymize(Partition,K) Empty_Summary.to_csv('Anon-K{}.csv'.format(K), index=False) K=9 Empty_Summary=pd.DataFrame(columns=Columns) Partition = Original_DF.copy() Anonymize(Partition,K) Empty_Summary.to_csv('Anon-K{}.csv'.format(K), index=False) #Check empty_set from after applying the function. #Empty_Summary #Load output of Greedy Algorithm for K3. #Validate that Empty_Set (Output) has classes >=K #This algorithm checks every class if they violate k rule. #index of row #Stop when you reach record 20k, since last index 19999 #get a row. #Get All Rows that have same Quasi Values (Same Class) #Get How many records in a class #Return Violation and Class if values less than K #else continue iteration through the all classes #if no violation is found, return to user. #test above algorithm #call method with call. #Task 3 - Utility Privacy Trade-off ###Using your implementation of the anonymization algorithm in ###Task2, find the anonymized table with k=3,5,7, and 9. ###For each anonymized table compute the Discernibility metric ###CDM and generalized information loss ILOSS given by the following equations. ###Then, draw a figure for each metric against the value of k to ###depict the privacy trade off. The x-axis should be the value of K, Y=metric #These tables I made using ARX #These tables I made using above Greedy Algorithm You can use EITHER tables, my algorithm works on both :) - Make sure you set the correct parameters from Task 2. #Stop when you reach record 20k, since last index 19999 #Select One Row #Get All Rows that have same Quasi Values (Same Class) #Apply Equation to a Class #Increment Selected Row. #Return CDM Value #This function calculates Denominator of I_LOSS #it takes Quasi + Table #it selects Attribute #it sorts values #Ignore * values for ARX #Ignore >= values for ARX #Get Biggest Value in that Column. #Convert Text Value to Numeric List #Get Upper Bound Value #Get Smallest Value in that Column #Convert Text Value to Numeric List #Get Lower Bound Value #Find Denominator #Return Denominator of that Column #ith row, a table, attribute #Select a row #Get All Rows that have same Quasi Values (Same Class) #Find Length of E Class (# of Records) #Shift/Increment Selected Row. #Get cell values in a Class #ignore * and >= (for ARX) #Return No Values, just i shift. #Get a cell values (We Assume all cells we have in Quasi columns contain 2 values (upper and lower) or have (*/>=) values!) #Get Upper Bound Value #Get Lower Bound Value #Find Numerator #Return Numerator, Class Size, and shifted i. #Loss function, only takes a table #intilize Gen. Loss. #Get Columns length #Initialize iterate value to go over columns #Get T Value #Calculate Coefficient #Start from First Row. #Iterate over all columns, and for each do. #Get Get_Denominator for that column. #Find Loss of each class for the selected column. #Stop when you reach record 20k, since last index 19999 #Get Numerator of a Class with Class Size and Shifted Position. #Calculate Loss of a complete class in a column. Then, continue doing it for all classes. #do the same thing for next column #Reset Row Selection #Return Value of Loss. #Call CDM function on a set of tables and store their result in an array. #These calculations will take time, uncomment and run on your responsibility #CDM_Answer_Greedy = [CDM(K3_Greedy),CDM(K5_Greedy),CDM(K7_Greedy),CDM(K9_Greedy)] #CDM_Answer_ARX = [CDM(K3_ARX),CDM(K5_ARX),CDM(K7_ARX),CDM(K9_ARX)] #Call I_Loss function on a set of tables and store their result in an array. #These calculations will take time, uncomment and run on your responsibility #I_Loss_Greedy = [GenCalc(K3_Greedy),GenCalc(K5_Greedy),GenCalc(K7_Greedy),GenCalc(K9_Greedy)] #I_Loss_ARX = [GenCalc(K3_ARX),GenCalc(K5_ARX),GenCalc(K7_ARX),GenCalc(K9_ARX)] #Define K Values in array. #Store values in a Frame to Plot them. #Store values in a Frame to Plot them. #Store values in a Frame to Plot them. #Store values in a Frame to Plot them. Most plots show that Loss increases as K increases, which reduces utility and increases privacy. Arx with l=2 has better I_Loss and Greedy algorithm has better CDM.  ### Greedy Algorithm Plots ### ARX Plots #Task4: l-diversity ###Using the anonymized table with k = 9 from Task 3, check if ###the 9-anonymized table is distinct l-diverse for each l = 2 and 5. ###In the case when the 9-anonymized table violates the l-diversity ###requirements, print at least one equivalence class that violates ###the diversity requirement. #Each should have class at least l' different values of Income(K) (>= l) #Load Task 3 Tables #This algorithm checks every class if they violate l rule. #index of row #Stop when you reach record 20k, since last index 19999 #get a row. #Get All Rows that have same Quasi Values (Same Class) #Dont Check classes with * or >= values (For ARX) #Get How many unique sensetive values we have in a class #Return Violation and Class if unique values is less than l #else continue iteration through the all classes #if no violation is found, return to user.