<gh_stars>1-10
# -*- coding: utf-8 -*-
"""Ising Model Bot Detector.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1Ou1TXypk5YA-DxSFRi55HsNwiELue7Cl

# Ising Model Bot Detection

This notebook lets you detect bots in a retweet network using the Ising model algorithm from the paper "Detecting Bots and Assessing Their Impact in Social Networks" https://arxiv.org/abs/1810.12398.  

You will need a retweet graph saved as a networkx object and the helper file networkClassifierHELPER.py and the file ioHELPER.py
"""

import os
import sys
import math
import datetime, time
import random
import numpy as np
import networkx as nx
import sqlite3,sys,os,string
import pandas as pd
import matplotlib.pyplot as plt

from sklearn import metrics
from scipy.sparse import csc_matrix

from networkClassifierHELPER import *

"""## Load Retweet  graph

path_data = folder where the retweet graph file is

Gretweet = retweet graph networkx DiGraph object.  

The format of the edges is (u,v,$weight$).  
This means u retweeted v $weight$ times.

Another way to view this is a node's out-degree is the number of nodes it retweeted, a node's in-degree is the number of people who retweet it.
We expect dout to be large for bots, din to be large for popular humans.
"""

path_data = "C://Users//Zlisto//Dropbox (Personal)//MIDAC//BotDetection//Pizzagate//"
   
fname_Gretweet = path_data + "Gretweet_pizzagate.gpickle"
Gretweet = nx.read_gpickle(fname_Gretweet)

n = Gretweet.number_of_nodes()
ne = Gretweet.number_of_edges()
Din = Gretweet.in_degree(weight='weight')
Dout = Gretweet.out_degree(weight='weight')
Dout_list = [x[1] for x in Dout]
Din_list = [x[1] for x in Din]
print("Gretweet has %s node, %s edges"%(n,ne))
print("Nodes max retweet = %.2f"%max(Dout_list))
print("Nodes max  retweet count %.2f"%max(Din_list))

"""## Define Ising model algorithm parameters

Use the default values provided here.  

lambdaij = edge energy scale factor for node type $i$ retweeting node type $j$.  The types are human = 0, bot = 1.

epsilon = edge energy parameter

alpha_out = upper quantiles of out-degree of graph (how much a node retweets)

alpha_in = upper quantiles of in-degree of graph (how much a node is retweeted)

mu = scale of edge energies w.r.t. node energies.  Set this to 1.
"""

print("Define Ising model parameters")
mu = 1

epsilon = 10**(-3) ##named delta in paper, should be close to 0 (eg. 0.001) in order for lambda10 to be slightly > to lambda00+lambda11-1.
lambda01 = 1
lambda00 = 0.61
lambda11 = 0.83
lambda10 = lambda00+ lambda11 - lambda01 + epsilon

q=0.999
alpha_in = np.quantile(Din_list,q)
alpha_out = np.quantile(Dout_list,q)

alpha=[mu,alpha_out,alpha_in]

print("alpha_out = %.2f"%alpha_out)
print("alpha_in = %.2f"%alpha_in)

"""## Create energy graph

PiBot = prior probability of being a bot for each node

link_data = list of edges, bidirectional indicator, and weight

edgelist_data = list of edges, edge energies
"""

PiBot = {}
for v in Gretweet.nodes(): 
    PiBot[v]=0.5
#link_data[i] = [u,v,is (u,v) in E, is (v,u) in E, number times u rewteets v]
link_data = getLinkDataRestrained(Gretweet)

start_time = time.time()
print("Make edgelist_data")
#edgelist_data[i] = [u,v,(Psi00,Psi01,Psi10,Psi11)], these are the edge energies
#on edge (i,j) for the graph cut
edgelist_data =[(i[0], i[1], psi(i[0],i[1],i[4], Din, Dout,alpha,lambda00,lambda11,epsilon)) for i in link_data]
print("\tEdgelist has %s edges"%len(edgelist_data))
print("--- %s seconds ---" % (time.time() - start_time))

"""## Find Min-Cut of energy graph

H = energy graph

BotsIsing = list of nodes who are bots in min-cut

HumansIsing = list of nodes who are humans in min-cut
"""

start_time = time.time()
print("Cut graph")
H, BotsIsing, user_data = computeH(Gretweet, PiBot, edgelist_data, Dout, Din)
Nodes = []
for v in Gretweet.nodes(): Nodes.append(v)
HumansIsing = list(set(Nodes) - set(BotsIsing))
print('\tCompleted graph cut')
print("%s bots in %s nodes"%(len(BotsIsing),Gretweet.number_of_nodes()))
print("--- %s seconds ---" % (time.time() - start_time))

"""## Calculate Bot Probability 
Find the probability each node is a bot using classification found from min-cut of energy graph.

THIS TAKES A LONG TIME

PiBotFinal = dictionary of bot probabilities.
"""

start_time = time.time()   
print("Calculate bot probability for each labeled node in retweet graph")
PiBotFinal = {}

for counter,node in enumerate(Gretweet.nodes()):
    if counter%1000==0:print("Node %s"%counter)
    if node in Gretweet.nodes():
        neighbors=list(np.unique([i for i in nx.all_neighbors(H,node) if i not in [0,1]])) 
        ebots=list(np.unique(np.intersect1d(neighbors,BotsIsing))) 
        ehumans=list(set(neighbors)-set(ebots)) 
        psi_l= sum([H[node][j]['capacity'] for j in ehumans])- sum([H[node][i]['capacity'] for i in ebots]) 

        psi_l_bis= psi_l + H[node][0]['capacity'] - H[1][node]['capacity'] ##probability to be in 1 = notPL

        if (psi_l_bis)>12:
            PiBotFinal[node] = 0
        else:
            PiBotFinal[node] = 1./(1+np.exp(psi_l_bis)) #Probability in the target (0) class


print("--- %s seconds ---" % (time.time() - start_time))

"""## Save probabilities to file
Convert dictionary of bot probabilities to a dataframe and write to a csv file.
"""

dfPiBot = pd.DataFrame(list(PiBotFinal.items()),columns = ['screen_name','bot_probability'])
FilenamePiBot = path_data+ "Pibot_pizzagate.csv"
dfPiBot.to_csv(FilenamePiBot)
print("Wrote bot probabilities to %s"%FilenamePiBot)

"""## Histogram of Bot Probabilities
Plot a histogram of the bot probabilities so you can see what a good threshold is
"""

data = dfPiBot.bot_probability
num_bins = round(len(data)/10)
counts, bin_edges = np.histogram (data, bins=num_bins, normed=True)
cdf = np.cumsum (counts)
plt.plot (bin_edges[1:], cdf/cdf[-1])
plt.grid()
plt.xlabel("Bot probability")
plt.ylabel("CDF")

nlow = len(dfPiBot[dfPiBot.bot_probability<0.5])
nhigh = len(dfPiBot[dfPiBot.bot_probability>0.5])
nmid = len(dfPiBot[dfPiBot.bot_probability==0.5])
print("%s users bot prob<0.5\n%s users bot prob>0.5\n%s users bot prob=0.5\n"%(nlow,nmid,nhigh))

plt.hist(dfPiBot.bot_probability[dfPiBot.bot_probability<0.5]);
plt.hist(dfPiBot.bot_probability[dfPiBot.bot_probability>0.5]);
plt.grid()
plt.xlabel("Bot probability")
plt.ylabel("Frequency")
plt.title("No 0.5 probability users")


# -*- coding: utf-8 -*-
"""Ising Model Bot Detector.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1Ou1TXypk5YA-DxSFRi55HsNwiELue7Cl

# Ising Model Bot Detection

This notebook lets you detect bots in a retweet network using the Ising model algorithm from the paper "Detecting Bots and Assessing Their Impact in Social Networks" https://arxiv.org/abs/1810.12398.  

You will need a retweet graph saved as a networkx object and the helper file networkClassifierHELPER.py and the file ioHELPER.py
"""

import os
import sys
import math
import datetime, time
import random
import numpy as np
import networkx as nx
import sqlite3,sys,os,string
import pandas as pd
import matplotlib.pyplot as plt

from sklearn import metrics
from scipy.sparse import csc_matrix

from networkClassifierHELPER import *

"""## Load Retweet  graph

path_data = folder where the retweet graph file is

Gretweet = retweet graph networkx DiGraph object.  

The format of the edges is (u,v,$weight$).  
This means u retweeted v $weight$ times.

Another way to view this is a node's out-degree is the number of nodes it retweeted, a node's in-degree is the number of people who retweet it.
We expect dout to be large for bots, din to be large for popular humans.
"""

path_data = "C://Users//Zlisto//Dropbox (Personal)//MIDAC//BotDetection//Pizzagate//"
   
fname_Gretweet = path_data + "Gretweet_pizzagate.gpickle"
Gretweet = nx.read_gpickle(fname_Gretweet)

n = Gretweet.number_of_nodes()
ne = Gretweet.number_of_edges()
Din = Gretweet.in_degree(weight='weight')
Dout = Gretweet.out_degree(weight='weight')
Dout_list = [x[1] for x in Dout]
Din_list = [x[1] for x in Din]
print("Gretweet has %s node, %s edges"%(n,ne))
print("Nodes max retweet = %.2f"%max(Dout_list))
print("Nodes max  retweet count %.2f"%max(Din_list))

"""## Define Ising model algorithm parameters

Use the default values provided here.  

lambdaij = edge energy scale factor for node type $i$ retweeting node type $j$.  The types are human = 0, bot = 1.

epsilon = edge energy parameter

alpha_out = upper quantiles of out-degree of graph (how much a node retweets)

alpha_in = upper quantiles of in-degree of graph (how much a node is retweeted)

mu = scale of edge energies w.r.t. node energies.  Set this to 1.
"""

print("Define Ising model parameters")
mu = 1

epsilon = 10**(-3) ##named delta in paper, should be close to 0 (eg. 0.001) in order for lambda10 to be slightly > to lambda00+lambda11-1.
lambda01 = 1
lambda00 = 0.61
lambda11 = 0.83
lambda10 = lambda00+ lambda11 - lambda01 + epsilon

q=0.999
alpha_in = np.quantile(Din_list,q)
alpha_out = np.quantile(Dout_list,q)

alpha=[mu,alpha_out,alpha_in]

print("alpha_out = %.2f"%alpha_out)
print("alpha_in = %.2f"%alpha_in)

"""## Create energy graph

PiBot = prior probability of being a bot for each node

link_data = list of edges, bidirectional indicator, and weight

edgelist_data = list of edges, edge energies
"""

PiBot = {}
for v in Gretweet.nodes(): 
    PiBot[v]=0.5
#link_data[i] = [u,v,is (u,v) in E, is (v,u) in E, number times u rewteets v]
link_data = getLinkDataRestrained(Gretweet)

start_time = time.time()
print("Make edgelist_data")
#edgelist_data[i] = [u,v,(Psi00,Psi01,Psi10,Psi11)], these are the edge energies
#on edge (i,j) for the graph cut
edgelist_data =[(i[0], i[1], psi(i[0],i[1],i[4], Din, Dout,alpha,lambda00,lambda11,epsilon)) for i in link_data]
print("\tEdgelist has %s edges"%len(edgelist_data))
print("--- %s seconds ---" % (time.time() - start_time))

"""## Find Min-Cut of energy graph

H = energy graph

BotsIsing = list of nodes who are bots in min-cut

HumansIsing = list of nodes who are humans in min-cut
"""

start_time = time.time()
print("Cut graph")
H, BotsIsing, user_data = computeH(Gretweet, PiBot, edgelist_data, Dout, Din)
Nodes = []
for v in Gretweet.nodes(): Nodes.append(v)
HumansIsing = list(set(Nodes) - set(BotsIsing))
print('\tCompleted graph cut')
print("%s bots in %s nodes"%(len(BotsIsing),Gretweet.number_of_nodes()))
print("--- %s seconds ---" % (time.time() - start_time))

"""## Calculate Bot Probability 
Find the probability each node is a bot using classification found from min-cut of energy graph.

THIS TAKES A LONG TIME

PiBotFinal = dictionary of bot probabilities.
"""

start_time = time.time()   
print("Calculate bot probability for each labeled node in retweet graph")
PiBotFinal = {}

for counter,node in enumerate(Gretweet.nodes()):
    if counter%1000==0:print("Node %s"%counter)
    if node in Gretweet.nodes():
        neighbors=list(np.unique([i for i in nx.all_neighbors(H,node) if i not in [0,1]])) 
        ebots=list(np.unique(np.intersect1d(neighbors,BotsIsing))) 
        ehumans=list(set(neighbors)-set(ebots)) 
        psi_l= sum([H[node][j]['capacity'] for j in ehumans])- sum([H[node][i]['capacity'] for i in ebots]) 

        psi_l_bis= psi_l + H[node][0]['capacity'] - H[1][node]['capacity'] ##probability to be in 1 = notPL

        if (psi_l_bis)>12:
            PiBotFinal[node] = 0
        else:
            PiBotFinal[node] = 1./(1+np.exp(psi_l_bis)) #Probability in the target (0) class


print("--- %s seconds ---" % (time.time() - start_time))

"""## Save probabilities to file
Convert dictionary of bot probabilities to a dataframe and write to a csv file.
"""

dfPiBot = pd.DataFrame(list(PiBotFinal.items()),columns = ['screen_name','bot_probability'])
FilenamePiBot = path_data+ "Pibot_pizzagate.csv"
dfPiBot.to_csv(FilenamePiBot)
print("Wrote bot probabilities to %s"%FilenamePiBot)

"""## Histogram of Bot Probabilities
Plot a histogram of the bot probabilities so you can see what a good threshold is
"""

data = dfPiBot.bot_probability
num_bins = round(len(data)/10)
counts, bin_edges = np.histogram (data, bins=num_bins, normed=True)
cdf = np.cumsum (counts)
plt.plot (bin_edges[1:], cdf/cdf[-1])
plt.grid()
plt.xlabel("Bot probability")
plt.ylabel("CDF")

nlow = len(dfPiBot[dfPiBot.bot_probability<0.5])
nhigh = len(dfPiBot[dfPiBot.bot_probability>0.5])
nmid = len(dfPiBot[dfPiBot.bot_probability==0.5])
print("%s users bot prob<0.5\n%s users bot prob>0.5\n%s users bot prob=0.5\n"%(nlow,nmid,nhigh))

plt.hist(dfPiBot.bot_probability[dfPiBot.bot_probability<0.5]);
plt.hist(dfPiBot.bot_probability[dfPiBot.bot_probability>0.5]);
plt.grid()
plt.xlabel("Bot probability")
plt.ylabel("Frequency")
plt.title("No 0.5 probability users")
# -*- coding: utf-8 -*- Ising Model Bot Detector.ipynb  Automatically generated by Colaboratory.  Original file is located at     https://colab.research.google.com/drive/1Ou1TXypk5YA-DxSFRi55HsNwiELue7Cl  # Ising Model Bot Detection  This notebook lets you detect bots in a retweet network using the Ising model algorithm from the paper "Detecting Bots and Assessing Their Impact in Social Networks" https://arxiv.org/abs/1810.12398.    You will need a retweet graph saved as a networkx object and the helper file networkClassifierHELPER.py and the file ioHELPER.py ## Load Retweet  graph  path_data = folder where the retweet graph file is  Gretweet = retweet graph networkx DiGraph object.    The format of the edges is (u,v,$weight$).   This means u retweeted v $weight$ times.  Another way to view this is a node's out-degree is the number of nodes it retweeted, a node's in-degree is the number of people who retweet it. We expect dout to be large for bots, din to be large for popular humans. ## Define Ising model algorithm parameters  Use the default values provided here.    lambdaij = edge energy scale factor for node type $i$ retweeting node type $j$.  The types are human = 0, bot = 1.  epsilon = edge energy parameter  alpha_out = upper quantiles of out-degree of graph (how much a node retweets)  alpha_in = upper quantiles of in-degree of graph (how much a node is retweeted)  mu = scale of edge energies w.r.t. node energies.  Set this to 1. ##named delta in paper, should be close to 0 (eg. 0.001) in order for lambda10 to be slightly > to lambda00+lambda11-1. ## Create energy graph  PiBot = prior probability of being a bot for each node  link_data = list of edges, bidirectional indicator, and weight  edgelist_data = list of edges, edge energies #link_data[i] = [u,v,is (u,v) in E, is (v,u) in E, number times u rewteets v] #edgelist_data[i] = [u,v,(Psi00,Psi01,Psi10,Psi11)], these are the edge energies #on edge (i,j) for the graph cut ## Find Min-Cut of energy graph  H = energy graph  BotsIsing = list of nodes who are bots in min-cut  HumansIsing = list of nodes who are humans in min-cut ## Calculate Bot Probability  Find the probability each node is a bot using classification found from min-cut of energy graph.  THIS TAKES A LONG TIME  PiBotFinal = dictionary of bot probabilities. ##probability to be in 1 = notPL #Probability in the target (0) class ## Save probabilities to file Convert dictionary of bot probabilities to a dataframe and write to a csv file. ## Histogram of Bot Probabilities Plot a histogram of the bot probabilities so you can see what a good threshold is