<filename>visualizing_node_embeddings.py
# -*- coding: utf-8 -*-
"""224W_FinalProj_VisualizingNodeEmbeddings.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1hRD5ubiZTEtLjLrpyCNq0TFpEUFxC1KU

# **CS224W - Final Project - Visualizing Node Embeddings**
> By <NAME> and <NAME>

For our final project, we are implementing a fully connected graph neural network to learn node embeddings for a link prediction task. We were inspired by the MGCN paper to learn embeddings informed not only from local neighborhood structure, but from larger graph substructures. A 2-layer GNN can only capture information about the local 2-hop neighborhood of a node, but we can induce the network to capture extra information critical to the link prediction task by training layers on **hypergraphs** desgined to capture additional graph structual information.

In this colab, we constructed a model that learns using only a single nn.Embedding layer (essentially, a single Linear layer) to establish a baseline understanding for the complexity of link prediction on the OGB-DDI dataset. We learned that a single layer is sorely inadequate for this task, which demands more complicated GNN architecture.
"""

# Install packages
!pip install -q torch-scatter -f https://pytorch-geometric.com/whl/torch-1.8.0+cu101.html
!pip install -q torch-sparse -f https://pytorch-geometric.com/whl/torch-1.8.0+cu101.html
!pip install -q torch-geometric
!pip install ogb

# Import packages
import networkx as nx, pandas as pd, numpy as np
import math, torch
from torch.nn.parameter import Parameter
from torch.nn.modules.module import Module
import torch.nn.functional as F
import torch.nn as nn
import sys

# Get Google SDK credentials
from pydrive.auth import GoogleAuth
from pydrive.drive import GoogleDrive
from google.colab import auth
from oauth2client.client import GoogleCredentials

# Authenticate and create the PyDrive client
auth.authenticate_user()
gauth = GoogleAuth()
gauth.credentials = GoogleCredentials.get_application_default()
drive = GoogleDrive(gauth)

# Import OGB tools
from ogb.linkproppred import PygLinkPropPredDataset, Evaluator
from torch_geometric.utils import to_networkx, to_undirected
from torch_geometric.data import Data, Dataset, InMemoryDataset, DataLoader

# Load the OGB-ddi dataset
dataset = PygLinkPropPredDataset(name = "ogbl-ddi", root = 'dataset/')

# Get data (in the form of an edge index)
data = dataset[0] 
#adj_t = data.adj_t.to(device)
edge_index = data['edge_index']   # Size = 2, 2135822

# Split edges into train, test, and validation sets
split_edge = dataset.get_edge_split() 
train_edge = split_edge['train']['edge'].t()  # Size = 2, 1067911

valid_edge = split_edge['valid']['edge'].t()          # Size = 2, 133489
valid_neg_edge = split_edge['valid']['edge_neg'].t()  # Size = 2, 101882
test_edge = split_edge['test']['edge'].t()          # Size = 2, 133489
test_neg_edge = split_edge['test']['edge_neg'].t()  # Size = 2, 95599

print(data.num_nodes)
print(train_edge.size(1))

# print(test_neg_edge_index.size())
# train_loader = DataLoader(dataset[split_edge["train"]], batch_size=32, shuffle=True)
# valid_loader = DataLoader(dataset[split_edge["valid"]], batch_size=32, shuffle=False)
# test_loader = DataLoader(dataset[split_edge["test"]], batch_size=32, shuffle=False)

# Create negative training edges
from torch_geometric.utils import negative_sampling

train_neg_edge = negative_sampling(edge_index, num_nodes=data.num_nodes,
                                 num_neg_samples=train_edge.size(1), method='dense', force_undirected=True)

print(train_edge)
print(train_neg_edge)

"""# Learning Node Embeddings 
Using a single nn.Embedding layer.
"""

import torch
import torch.nn as nn
import matplotlib.pyplot as plt
from sklearn.decomposition import PCA

print(torch.__version__)

# Convert this to a NetworkX graph
G = nx.Graph()
G.add_nodes_from(range(data.num_nodes))
for i, (u, v) in enumerate(data.edge_index.t().tolist()):  
    G.add_edge(u, v)

# Do not change / reset the random seed
torch.manual_seed(1)

embedding_dim = 256
emb = torch.nn.Embedding(data.num_nodes, embedding_dim)
emb.weight.data = torch.rand(data.num_nodes, embedding_dim)

def visualize_emb(emb):
  X = emb.weight.data.numpy()
  pca = PCA(n_components=2)
  components = pca.fit_transform(X)
  plt.figure(figsize=(15, 15))
  plt.scatter(components[:,0], components[:,1])
  plt.show()

# Visualize the initial random embeddding
visualize_emb(emb)

from torch.optim import SGD

def accuracy(pred, label):
  # TODO: Implement the accuracy function. This function takes the 
  # pred tensor (the resulting tensor after sigmoid) and the label 
  # tensor (torch.LongTensor). Predicted value greater than 0.5 will 
  # be classified as label 1. Else it will be classified as label 0.
  # The returned accuracy should be rounded to 4 decimal places. 
  # For example, accuracy 0.82956 will be rounded to 0.8296.

  accu = 0.0

  ############# Your code here ############
  pred_mod = torch.flatten(pred)
  pred_mod = (pred_mod>0.5).float()
  correct = (pred_mod == label).float().sum()

  accu = round(correct.item() / pred_mod.size()[0], 4)

  #########################################

  return accu

def train(emb, loss_fn, sigmoid, train_label, train_edge):
  
  epochs = 100 # previously 500
  learning_rate = 0.5 # previously 0.1

  optimizer = SGD(emb.parameters(), lr=learning_rate, momentum=0.9)

  for i in range(epochs):

    ############# Your code here ############
    #optimizer.zero_grad()  # Clear gradients.

    # (1) Get the embeddings of the nodes in train_edge

    source_idx, target_idx = torch.LongTensor(train_edge[0][:].long()), torch.LongTensor(train_edge[1][:].long())
    source_embed = emb(source_idx)
    target_embed = emb(target_idx)

    # (2) Dot product the embeddings between each node pair
    prod = torch.sum(source_embed*target_embed, dim=(1))

    # (3) Feed the dot product result into sigmoid
    sigmoid_output = sigmoid(prod)
    train_label_reshape = torch.transpose(torch.unsqueeze(train_label, 0), 0, 1)
    train_label_reshape = torch.unsqueeze(train_label_reshape, 1)

    # (4) Feed the sigmoid output into the loss_fn
    loss = loss_fn(sigmoid_output, train_label)

    print(sigmoid_output)

    # (5) Print both loss and accuracy of each epoch 
    print("Epoch {}-- loss: {}, accuracy: {}".format(i, loss, accuracy(sigmoid_output, train_label)))
    
    loss.backward()  # Derive gradients.
    optimizer.step()  # Update parameters based on gradients.
    print(emb.weight.grad)

    #########################################

loss_fn = nn.BCELoss()
sigmoid = nn.Sigmoid()

# Generate the positive and negative labels
pos_label = torch.ones(train_edge.size(1), )
neg_label = torch.zeros(train_neg_edge.size(1), )

# Concat positive and negative labels into one tensor
train_label = torch.cat([pos_label, neg_label], dim=0)

# Concat positive and negative edges into one tensor
train_edge_total = torch.cat([train_edge, train_neg_edge], dim=1)

train(emb, loss_fn, sigmoid, train_label, train_edge_total)

# Visualize the final learned embedding
visualize_emb(emb)


<filename>visualizing_node_embeddings.py
# -*- coding: utf-8 -*-
"""224W_FinalProj_VisualizingNodeEmbeddings.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1hRD5ubiZTEtLjLrpyCNq0TFpEUFxC1KU

# **CS224W - Final Project - Visualizing Node Embeddings**
> By <NAME> and <NAME>

For our final project, we are implementing a fully connected graph neural network to learn node embeddings for a link prediction task. We were inspired by the MGCN paper to learn embeddings informed not only from local neighborhood structure, but from larger graph substructures. A 2-layer GNN can only capture information about the local 2-hop neighborhood of a node, but we can induce the network to capture extra information critical to the link prediction task by training layers on **hypergraphs** desgined to capture additional graph structual information.

In this colab, we constructed a model that learns using only a single nn.Embedding layer (essentially, a single Linear layer) to establish a baseline understanding for the complexity of link prediction on the OGB-DDI dataset. We learned that a single layer is sorely inadequate for this task, which demands more complicated GNN architecture.
"""

# Install packages
!pip install -q torch-scatter -f https://pytorch-geometric.com/whl/torch-1.8.0+cu101.html
!pip install -q torch-sparse -f https://pytorch-geometric.com/whl/torch-1.8.0+cu101.html
!pip install -q torch-geometric
!pip install ogb

# Import packages
import networkx as nx, pandas as pd, numpy as np
import math, torch
from torch.nn.parameter import Parameter
from torch.nn.modules.module import Module
import torch.nn.functional as F
import torch.nn as nn
import sys

# Get Google SDK credentials
from pydrive.auth import GoogleAuth
from pydrive.drive import GoogleDrive
from google.colab import auth
from oauth2client.client import GoogleCredentials

# Authenticate and create the PyDrive client
auth.authenticate_user()
gauth = GoogleAuth()
gauth.credentials = GoogleCredentials.get_application_default()
drive = GoogleDrive(gauth)

# Import OGB tools
from ogb.linkproppred import PygLinkPropPredDataset, Evaluator
from torch_geometric.utils import to_networkx, to_undirected
from torch_geometric.data import Data, Dataset, InMemoryDataset, DataLoader

# Load the OGB-ddi dataset
dataset = PygLinkPropPredDataset(name = "ogbl-ddi", root = 'dataset/')

# Get data (in the form of an edge index)
data = dataset[0] 
#adj_t = data.adj_t.to(device)
edge_index = data['edge_index']   # Size = 2, 2135822

# Split edges into train, test, and validation sets
split_edge = dataset.get_edge_split() 
train_edge = split_edge['train']['edge'].t()  # Size = 2, 1067911

valid_edge = split_edge['valid']['edge'].t()          # Size = 2, 133489
valid_neg_edge = split_edge['valid']['edge_neg'].t()  # Size = 2, 101882
test_edge = split_edge['test']['edge'].t()          # Size = 2, 133489
test_neg_edge = split_edge['test']['edge_neg'].t()  # Size = 2, 95599

print(data.num_nodes)
print(train_edge.size(1))

# print(test_neg_edge_index.size())
# train_loader = DataLoader(dataset[split_edge["train"]], batch_size=32, shuffle=True)
# valid_loader = DataLoader(dataset[split_edge["valid"]], batch_size=32, shuffle=False)
# test_loader = DataLoader(dataset[split_edge["test"]], batch_size=32, shuffle=False)

# Create negative training edges
from torch_geometric.utils import negative_sampling

train_neg_edge = negative_sampling(edge_index, num_nodes=data.num_nodes,
                                 num_neg_samples=train_edge.size(1), method='dense', force_undirected=True)

print(train_edge)
print(train_neg_edge)

"""# Learning Node Embeddings 
Using a single nn.Embedding layer.
"""

import torch
import torch.nn as nn
import matplotlib.pyplot as plt
from sklearn.decomposition import PCA

print(torch.__version__)

# Convert this to a NetworkX graph
G = nx.Graph()
G.add_nodes_from(range(data.num_nodes))
for i, (u, v) in enumerate(data.edge_index.t().tolist()):  
    G.add_edge(u, v)

# Do not change / reset the random seed
torch.manual_seed(1)

embedding_dim = 256
emb = torch.nn.Embedding(data.num_nodes, embedding_dim)
emb.weight.data = torch.rand(data.num_nodes, embedding_dim)

def visualize_emb(emb):
  X = emb.weight.data.numpy()
  pca = PCA(n_components=2)
  components = pca.fit_transform(X)
  plt.figure(figsize=(15, 15))
  plt.scatter(components[:,0], components[:,1])
  plt.show()

# Visualize the initial random embeddding
visualize_emb(emb)

from torch.optim import SGD

def accuracy(pred, label):
  # TODO: Implement the accuracy function. This function takes the 
  # pred tensor (the resulting tensor after sigmoid) and the label 
  # tensor (torch.LongTensor). Predicted value greater than 0.5 will 
  # be classified as label 1. Else it will be classified as label 0.
  # The returned accuracy should be rounded to 4 decimal places. 
  # For example, accuracy 0.82956 will be rounded to 0.8296.

  accu = 0.0

  ############# Your code here ############
  pred_mod = torch.flatten(pred)
  pred_mod = (pred_mod>0.5).float()
  correct = (pred_mod == label).float().sum()

  accu = round(correct.item() / pred_mod.size()[0], 4)

  #########################################

  return accu

def train(emb, loss_fn, sigmoid, train_label, train_edge):
  
  epochs = 100 # previously 500
  learning_rate = 0.5 # previously 0.1

  optimizer = SGD(emb.parameters(), lr=learning_rate, momentum=0.9)

  for i in range(epochs):

    ############# Your code here ############
    #optimizer.zero_grad()  # Clear gradients.

    # (1) Get the embeddings of the nodes in train_edge

    source_idx, target_idx = torch.LongTensor(train_edge[0][:].long()), torch.LongTensor(train_edge[1][:].long())
    source_embed = emb(source_idx)
    target_embed = emb(target_idx)

    # (2) Dot product the embeddings between each node pair
    prod = torch.sum(source_embed*target_embed, dim=(1))

    # (3) Feed the dot product result into sigmoid
    sigmoid_output = sigmoid(prod)
    train_label_reshape = torch.transpose(torch.unsqueeze(train_label, 0), 0, 1)
    train_label_reshape = torch.unsqueeze(train_label_reshape, 1)

    # (4) Feed the sigmoid output into the loss_fn
    loss = loss_fn(sigmoid_output, train_label)

    print(sigmoid_output)

    # (5) Print both loss and accuracy of each epoch 
    print("Epoch {}-- loss: {}, accuracy: {}".format(i, loss, accuracy(sigmoid_output, train_label)))
    
    loss.backward()  # Derive gradients.
    optimizer.step()  # Update parameters based on gradients.
    print(emb.weight.grad)

    #########################################

loss_fn = nn.BCELoss()
sigmoid = nn.Sigmoid()

# Generate the positive and negative labels
pos_label = torch.ones(train_edge.size(1), )
neg_label = torch.zeros(train_neg_edge.size(1), )

# Concat positive and negative labels into one tensor
train_label = torch.cat([pos_label, neg_label], dim=0)

# Concat positive and negative edges into one tensor
train_edge_total = torch.cat([train_edge, train_neg_edge], dim=1)

train(emb, loss_fn, sigmoid, train_label, train_edge_total)

# Visualize the final learned embedding
visualize_emb(emb)


# -*- coding: utf-8 -*- 224W_FinalProj_VisualizingNodeEmbeddings.ipynb  Automatically generated by Colaboratory.  Original file is located at     https://colab.research.google.com/drive/1hRD5ubiZTEtLjLrpyCNq0TFpEUFxC1KU  # **CS224W - Final Project - Visualizing Node Embeddings** > By <NAME> and <NAME>  For our final project, we are implementing a fully connected graph neural network to learn node embeddings for a link prediction task. We were inspired by the MGCN paper to learn embeddings informed not only from local neighborhood structure, but from larger graph substructures. A 2-layer GNN can only capture information about the local 2-hop neighborhood of a node, but we can induce the network to capture extra information critical to the link prediction task by training layers on **hypergraphs** desgined to capture additional graph structual information.  In this colab, we constructed a model that learns using only a single nn.Embedding layer (essentially, a single Linear layer) to establish a baseline understanding for the complexity of link prediction on the OGB-DDI dataset. We learned that a single layer is sorely inadequate for this task, which demands more complicated GNN architecture. # Install packages # Import packages # Get Google SDK credentials # Authenticate and create the PyDrive client # Import OGB tools # Load the OGB-ddi dataset # Get data (in the form of an edge index) #adj_t = data.adj_t.to(device) # Size = 2, 2135822 # Split edges into train, test, and validation sets # Size = 2, 1067911 # Size = 2, 133489 # Size = 2, 101882 # Size = 2, 133489 # Size = 2, 95599 # print(test_neg_edge_index.size()) # train_loader = DataLoader(dataset[split_edge["train"]], batch_size=32, shuffle=True) # valid_loader = DataLoader(dataset[split_edge["valid"]], batch_size=32, shuffle=False) # test_loader = DataLoader(dataset[split_edge["test"]], batch_size=32, shuffle=False) # Create negative training edges # Learning Node Embeddings  Using a single nn.Embedding layer. # Convert this to a NetworkX graph # Do not change / reset the random seed # Visualize the initial random embeddding # TODO: Implement the accuracy function. This function takes the # pred tensor (the resulting tensor after sigmoid) and the label # tensor (torch.LongTensor). Predicted value greater than 0.5 will # be classified as label 1. Else it will be classified as label 0. # The returned accuracy should be rounded to 4 decimal places. # For example, accuracy 0.82956 will be rounded to 0.8296. ############# Your code here ############ ######################################### # previously 500 # previously 0.1 ############# Your code here ############ #optimizer.zero_grad()  # Clear gradients. # (1) Get the embeddings of the nodes in train_edge # (2) Dot product the embeddings between each node pair # (3) Feed the dot product result into sigmoid # (4) Feed the sigmoid output into the loss_fn # (5) Print both loss and accuracy of each epoch # Derive gradients. # Update parameters based on gradients. ######################################### # Generate the positive and negative labels # Concat positive and negative labels into one tensor # Concat positive and negative edges into one tensor # Visualize the final learned embedding