{ "cells": [ { "cell_type": "code", "execution_count": null, "id": "c809ceed", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Using cache found in /home/pranjul/.cache/torch/hub/pytorch_vision_v0.6.0\n" ] } ], "source": [ "import matplotlib.pyplot as plt\n", "import numpy as np\n", "from scipy.misc import face\n", "from scipy.ndimage import zoom\n", "from scipy.special import logsumexp\n", "import torch\n", "\n", "import deepgaze_pytorch\n", "\n", "DEVICE = 'cuda'\n", "\n", "# you can use DeepGazeI or DeepGazeIIE\n", "model_dg1 = deepgaze_pytorch.DeepGazeI(pretrained=True).to(DEVICE)\n", "\n", "image = face()\n", "\n", "# load precomputed centerbias log density (from MIT1003) over a 1024x1024 image\n", "# you can download the centerbias from https://github.com/matthias-k/DeepGaze/releases/download/v1.0.0/centerbias_mit1003.npy\n", "# alternatively, you can use a uniform centerbias via `centerbias_template = np.zeros((1024, 1024))`.\n", "centerbias_template = np.load('centerbias_mit1003.npy')\n", "# rescale to match image size\n", "centerbias = zoom(centerbias_template, (image.shape[0]/centerbias_template.shape[0], image.shape[1]/centerbias_template.shape[1]), order=0, mode='nearest')\n", "# renormalize log density\n", "centerbias -= logsumexp(centerbias)\n", "\n", "image_tensor = torch.tensor([image.transpose(2, 0, 1)]).to(DEVICE)\n", "centerbias_tensor = torch.tensor([centerbias]).to(DEVICE)\n", "\n", "log_density_prediction = model_dg1(image_tensor, centerbias_tensor)\n", "\n", "f, axs = plt.subplots(nrows=1, ncols=2, figsize=(8, 3))\n", "axs[0].imshow(image)\n", "#axs[0].plot(fixation_history_x, fixation_history_y, 'o-', color='red')\n", "#axs[0].scatter(fixation_history_x[-1], fixation_history_y[-1], 100, color='yellow', zorder=100)\n", "axs[0].set_axis_off()\n", "axs[1].matshow(log_density_prediction.detach().cpu().numpy()[0]) # first image in batch, first (and only) channel\n", "#axs[1].plot(fixation_history_x, fixation_history_y, 'o-', color='red')\n", "#axs[1].scatter(fixation_history_x[-1], fixation_history_y[-1], 100, color='yellow', zorder=100)\n", "axs[1].set_axis_off()" ] }, { "cell_type": "code", "execution_count": null, "id": "c0a46c9d", "metadata": {}, "outputs": [], "source": [ "import torch\n", "import numpy as np\n", "import helper_consol as helper\n", "import pandas as pd\n", "import pingouin as pg\n", "import rsatoolbox\n", "import torchvision\n", "import rsatoolbox.data as rsd # abbreviation to deal with dataset\n", "import rsatoolbox.rdm as rsr\n", "from sklearn import preprocessing\n", "import matplotlib.pyplot as plt\n", "\n" ] }, { "cell_type": "code", "execution_count": null, "id": "11c42ad4", "metadata": {}, "outputs": [], "source": [ "# pareidolia hypothesis - faces and pareidolia are same\n", "# all distances are 0 between and within faces and pareidolia, but 1 between faces/pareidolia and objects\n", "a = np.zeros((32,32))\n", "b = np.zeros((32,32))\n", "c = np.ones((32,32))\n", "d = np.zeros((32,32))\n", "e = np.zeros((32,32))\n", "f = np.ones((32,32))\n", "g = np.ones((32,32))\n", "h = np.ones((32,32))\n", "i = np.zeros((32,32))\n", "hypo_1 = np.bmat([[a, b, c], [d, e, f], [g, h, i]])\n", "plt.imshow(hypo_1)" ] }, { "cell_type": "code", "execution_count": null, "id": "0858982f", "metadata": {}, "outputs": [], "source": [ "# face hypothesis - objects and pareidolia are same\n", "# all distances are 0 between and within objects and pareidolia, but 1 between objects/pareidolia and faces\n", "a = np.zeros((32,32))\n", "b = np.ones((32,32))\n", "c = np.ones((32,32))\n", "d = np.ones((32,32))\n", "e = np.zeros((32,32))\n", "f = np.zeros((32,32))\n", "g = np.ones((32,32))\n", "h = np.zeros((32,32))\n", "i = np.zeros((32,32))\n", "hypo_2 = np.bmat([[a, b, c], [d, e, f], [g, h, i]])\n", "plt.imshow(hypo_2)" ] }, { "cell_type": "code", "execution_count": null, "id": "614acbaa", "metadata": {}, "outputs": [], "source": [ "network_name = 'face_inanimate_400k_facesconsolidated'\n", "config = helper.Config(config_file='./configs/vgg/' + network_name + '.yaml')\n", "model, ckpt_data = config.get_model(pretrained=True, ngpus=1, dataParallel=False, epoch=110, consol=True)\n", "# model, ckpt_data = config.get_model(pretrained=True, ngpus=1, dataParallel=True, epoch=-1)\n", "model = model.cuda()" ] }, { "cell_type": "code", "execution_count": null, "id": "8173f9bc", "metadata": {}, "outputs": [], "source": [ "#network_name = 'face_inanimate_400k_facesconsolidated'\n", "#config = helper.Config(config_file='./configs/vgg/' + network_name + '.yaml')\n", "#model, ckpt_data = config.get_model(pretrained=True, ngpus=1, dataParallel=False, epoch=110, consol=True)\n", "# model, ckpt_data = config.get_model(pretrained=True, ngpus=1, dataParallel=True, epoch=-1)\n", "model_dg1 = model_dg1.cuda()\n", "\n", "'''\n", "# Define the layer(s) you want to extract activations from\n", "layers = [model.module.features[1], model.module.features[3], model.module.features[6],\n", " model.module.features[8], model.module.features[11], model.module.features[13],\n", " model.module.features[15], model.module.features[18], model.module.features[20],\n", " model.module.features[22], model.module.features[25], model.module.features[27],\n", " model.module.features[29], model.module.classifier[1], model.module.classifier[4]]\n", "\n", "\n", "layers = [model.features[1], model.features[3], model.features[6],\n", " model.features[8], model.features[11], model.features[13],\n", " model.features[15], model.features[18], model.features[20],\n", " model.features[22], model.features[25], model.features[27],\n", " model.features[29], model.classifier[1], model.classifier[4]]\n", "'''\n", "\n", "layers = [model_dg1.features.features[1].features[1], model_dg1.features.features[1].features[4],\n", " model_dg1.features.features[1].features[7], model_dg1.features.features[1].features[9],\n", " model_dg1.features.features[1].features[11], model_dg1.features.features[1].classifier[2],\n", " model_dg1.features.features[1].classifier[5], model_dg1.readout_network.conv0,\n", " model_dg1.finalizer.gauss]\n", "\n", "\n", "# Register forward hooks to the selected layers\n", "activation = {}\n", "\n", "def get_activation(name):\n", " def hook(model_dg1, input, output):\n", " activation[name] = output.detach()\n", " return hook\n", "\n", "for i, layer in enumerate(layers):\n", " layer.register_forward_hook(get_activation(f'layer_{i+1}'))" ] }, { "cell_type": "code", "execution_count": null, "id": "35566e1a", "metadata": { "scrolled": true }, "outputs": [], "source": [ "bts_no = 1000\n", "folder_dir = ['/home/pranjul/DeepGaze/all_images_net_input/']\n", "\n", "validator = helper.Validator(name='things_validator',\n", " model=model,\n", " batch_size=10,\n", " data_dir=folder_dir,\n", " ngpus=1,\n", " workers=4,\n", " task=None,\n", " max_samples={'all_images_net_input': 32},\n", " maxout=True,\n", " read_seed=0,\n", " shuffle=False,\n", " data_subdir='test',\n", " includePaths=True)\n", "\n", "num_imgs = len(validator.dataset)\n", "centerbias_template = np.load('centerbias_mit1003.npy')\n", "\n", "img_act = []\n", "spearmanr_values_hypo_1_bts, spearmanr_values_hypo_2_bts = [], []\n", "\n", "for i, image in enumerate(validator.dataset):\n", " \n", " filename = image[-1].split('/')[-1]\n", " image = image[0].unsqueeze(0).cuda()\n", " \n", " \n", " # rescale to match image size\n", " centerbias = zoom(centerbias_template, (image.shape[2]/centerbias_template.shape[0], image.shape[3]/centerbias_template.shape[1]), order=0, mode='nearest')\n", " # renormalize log density\n", " centerbias -= logsumexp(centerbias)\n", "\n", " centerbias_tensor = torch.tensor([centerbias]).to(DEVICE)\n", "\n", " # Forward pass the image through the model\n", " output = model_dg1(image, centerbias_tensor)\n", "\n", " # Retrieve the extracted activations\n", " activations = [activation[f'layer_{i+1}'] for i in range(len(layers))]\n", "\n", " # Print out the extracted activations\n", " for j, act in enumerate(activations):\n", " print(f\"Layer {j+1} activation shape for {filename}: {act.shape}\")\n", " act = act.cpu().detach().numpy().squeeze()\n", " act = np.reshape(act, (np.prod(act.shape[0:])))\n", " img_act.append(np.array(act))\n", "\n", "\n", "for boot_itr in range(bts_no):\n", " \n", " print(\"boot_itr: \", boot_itr + 1)\n", " \n", " np.random.seed(boot_itr)\n", "\n", " randomRows_0 = []\n", " randomRows_1 = []\n", " randomRows_2 = []\n", "\n", " randomRows_0 = np.random.choice(np.arange(0, 32), size=np.arange(32).shape, replace=True)\n", " randomRows_1 = np.random.choice(np.arange(32, 64), size=np.arange(32).shape, replace=True)\n", " randomRows_2 = np.random.choice(np.arange(64, 96), size=np.arange(32).shape, replace=True)\n", " #print(randomRows)\n", "\n", " bootstrap_idx = [*randomRows_0, *randomRows_1, *randomRows_2]\n", " # print(bootstrap_idx)\n", "\n", " \n", " img_act_layer = []\n", " dist_metric_layers_dual_task_fc = []\n", " \n", " for i in range(len(layers)):\n", " for j in bootstrap_idx:\n", " img_act_layer.append(img_act[i + (j*len(layers))])\n", "\n", " for i in range(len(layers)):\n", " dist_metric = 1 - np.corrcoef(np.array(img_act_layer[i*96:(i+1)*96]))\n", " dist_metric_layers_dual_task_fc.append(dist_metric[np.triu_indices(96, k = 1)])\n", "\n", "\n", " data = np.vstack([dist_metric_layers_dual_task_fc, \n", " np.array(hypo_1[np.triu_indices(96, k = 1)]),\n", " np.array(hypo_2[np.triu_indices(96, k = 1)])\n", " ])\n", "\n", " # print(np.shape(data))\n", " df = pd.DataFrame(data=data.T)\n", "\n", " col_names = []\n", "\n", " for i in range(len(layers)):\n", " col_names.append('layer_' + str(i + 1))\n", "\n", " col_names.extend(('hypo_1', 'hypo_2'))\n", " df.columns = col_names\n", "\n", " spearmanr_values_hypo_1, spearmanr_values_hypo_2 = [], []\n", " layer_no = np.arange(1, len(layers)+1)\n", "\n", " for i in range(len(layers)):\n", "\n", " zero_indices = df.loc[df['layer_'+ str(i + 1)] == 0].index \n", " df_0_removed = df.drop(zero_indices)\n", " \n", " spearmanr_values_hypo_1.append(pg.partial_corr(data=df_0_removed, x='layer_' + str(i + 1), y='hypo_1', covar=['hypo_2'], method='spearman').round(3)['r'][0])\n", " spearmanr_values_hypo_2.append(pg.partial_corr(data=df_0_removed, x='layer_' + str(i + 1), y='hypo_2', covar=['hypo_1'], method='spearman').round(3)['r'][0])\n", " \n", " spearmanr_values_hypo_1_bts.append(np.array(spearmanr_values_hypo_1))\n", " spearmanr_values_hypo_2_bts.append(np.array(spearmanr_values_hypo_2))\n", " \n", "#np.save('spearmanr_values_hypo_1_FC_bootstrap_new_2hypos.npy', np.array(spearmanr_values_hypo_1_bts))\n", "#np.save('spearmanr_values_hypo_2_FC_bootstrap_new_2hypos.npy', np.array(spearmanr_values_hypo_2_bts))\n", "\n", "'''\n", "# Plotting\n", "fig = plt.figure(figsize=(14, 8))\n", "ax = fig.add_subplot(111)\n", "plt.plot(layer_no, spearmanr_values_hypo_1, '-o', label='Faces~Pareidolia', markersize=10)\n", "plt.plot(layer_no, spearmanr_values_hypo_2, '-o', label='Pareidolia~Objects', markersize=10)\n", "plt.plot(layer_no, spearmanr_values_hypo_3, '-o', label='Faces~Objects', markersize=10)\n", "\n", "plt.xticks(layer_no, fontsize=18)\n", "plt.yticks(fontsize=18)\n", "ax.set_xticklabels(layer_no)\n", "plt.ylabel('Partial spearmanr (rho)', fontsize=18)\n", "plt.yticks(np.arange(-0.15, 0.65, 0.05))\n", "leg = plt.legend(loc = 2, prop={'size': 15})\n", "leg.get_frame().set_edgecolor('k')\n", "fig.set_size_inches(14.,8.)\n", "#plt.savefig('3_hypos_partial_spearman.png', dpi=600)\n", "\n", "plt.show()\n", "'''" ] }, { "cell_type": "code", "execution_count": null, "id": "f4ab47a3", "metadata": {}, "outputs": [], "source": [ "num_imgs = len(validator.dataset)\n", "\n", "img_act = []\n", "\n", "for i, image in enumerate(validator.dataset):\n", " \n", " filename = image[-1].split('/')[-1]\n", " image = image[0].unsqueeze(0).cuda()\n", "\n", " # rescale to match image size\n", " centerbias = zoom(centerbias_template, (image.shape[2]/centerbias_template.shape[0], image.shape[3]/centerbias_template.shape[1]), order=0, mode='nearest')\n", " # renormalize log density\n", " centerbias -= logsumexp(centerbias)\n", "\n", " centerbias_tensor = torch.tensor([centerbias]).to(DEVICE)\n", "\n", " # Forward pass the image through the model\n", " output = model_dg1(image, centerbias_tensor)\n", "\n", " # Retrieve the extracted activations\n", " activations = [activation[f'layer_{i+1}'] for i in range(len(layers))]\n", "\n", " # Print out the extracted activations\n", " for j, act in enumerate(activations):\n", " # print(f\"Layer {j+1} activation shape for {filename}: {act.shape}\")\n", " act = act.cpu().detach().numpy().squeeze()\n", " act = np.reshape(act, (np.prod(act.shape[0:])))\n", " img_act.append(np.array(act))\n", "\n", "img_act_layer = []\n", "dist_metric_layers_dual_task_fc = []\n", "\n", "\n", "for i in range(len(layers)):\n", " for j in range(96):\n", " img_act_layer.append(img_act[i + (j*len(layers))])\n", "\n", "for i in range(len(layers)):\n", " dist_metric = 1 - np.corrcoef(np.array(img_act_layer[i*96:(i+1)*96]))\n", " dist_metric_layers_dual_task_fc.append(dist_metric[np.triu_indices(96, k = 1)])\n", "\n", "#np.save('dist_metric_layers_dual_task_fc_2_hypos.npy', np.array(dist_metric_layers_dual_task_fc))\n", "\n", "data = np.vstack([dist_metric_layers_dual_task_fc, \n", " np.array(hypo_1[np.triu_indices(96, k = 1)]),\n", " np.array(hypo_2[np.triu_indices(96, k = 1)]),\n", " ])\n", "\n", "# print(np.shape(data))\n", "df = pd.DataFrame(data=data.T)\n", "\n", "col_names = []\n", "\n", "for i in range(len(layers)):\n", " col_names.append('layer_' + str(i + 1))\n", "\n", "col_names.extend(('hypo_1', 'hypo_2'))\n", "df.columns = col_names\n", "\n", "spearmanr_values_hypo_1, spearmanr_values_hypo_2 = [], []\n", "layer_no = np.arange(1, len(layers)+1)\n", "\n", "for i in range(len(layers)):\n", "\n", " spearmanr_values_hypo_1.append(pg.partial_corr(data=df, x='layer_' + str(i + 1), y='hypo_1', covar=['hypo_2'], method='spearman').round(3)['r'][0])\n", " spearmanr_values_hypo_2.append(pg.partial_corr(data=df, x='layer_' + str(i + 1), y='hypo_2', covar=['hypo_1'], method='spearman').round(3)['r'][0])\n", "\n", "#np.save('inanimate_vgg_large_spearmanr_values_hypo_1_2hypos.npy', np.array(spearmanr_values_hypo_1))\n", "#np.save('inanimate_vgg_large_spearmanr_values_hypo_2_2hypos.npy', np.array(spearmanr_values_hypo_2))\n", "\n", "#face_inanimate_400k_seed_spearmanr_values_hypo_1_bootstrap_new_2hypos\n", "\n", "# Plotting\n", "fig = plt.figure(figsize=(14, 8))\n", "ax = fig.add_subplot(111)\n", "plt.plot(layer_no, spearmanr_values_hypo_1, '-o', label='Faces~Pareidolia', markersize=10)\n", "plt.plot(layer_no, spearmanr_values_hypo_2, '-o', label='Pareidolia~Objects', markersize=10)\n", "\n", "plt.xticks(layer_no, fontsize=18)\n", "plt.yticks(fontsize=18)\n", "ax.set_xticklabels(layer_no)\n", "plt.ylabel('Partial spearmanr (rho)', fontsize=18)\n", "plt.yticks(np.arange(-0.15, 0.65, 0.05))\n", "leg = plt.legend(loc = 2, prop={'size': 15})\n", "leg.get_frame().set_edgecolor('k')\n", "fig.set_size_inches(14.,8.)\n", "#plt.savefig('3_hypos_partial_spearman.png', dpi=600)\n", "\n", "plt.show()\n" ] }, { "cell_type": "code", "execution_count": null, "id": "78693e0a", "metadata": {}, "outputs": [], "source": [ "face_vgg_large_spearmanr_values_hypo_1 = spearmanr_values_hypo_1_bts\n", "face_vgg_large_spearmanr_values_hypo_1 = np.reshape(face_vgg_large_spearmanr_values_hypo_1, (bts_no, len(layers)))\n", "face_vgg_large_spearmanr_values_hypo_2 = spearmanr_values_hypo_2_bts\n", "face_vgg_large_spearmanr_values_hypo_2 = np.reshape(face_vgg_large_spearmanr_values_hypo_2, (bts_no, len(layers)))" ] }, { "cell_type": "code", "execution_count": null, "id": "6aa9c33e", "metadata": {}, "outputs": [], "source": [ "mean_1 = spearmanr_values_hypo_1\n", "mean_2 = spearmanr_values_hypo_2\n", "\n", "std_1 = []\n", "std_2 = []\n", "\n", "\n", "for i in range(len(layers)):\n", " std_1.append(np.std(face_vgg_large_spearmanr_values_hypo_1, axis = 0)[i])\n", " std_2.append(np.std(face_vgg_large_spearmanr_values_hypo_2, axis = 0)[i])\n", "\n", "std_1 = np.array(std_1)\n", "std_2 = np.array(std_2)" ] }, { "cell_type": "code", "execution_count": null, "id": "1b813dc6", "metadata": {}, "outputs": [], "source": [ "# Plotting\n", "fig = plt.figure(figsize=(14, 8))\n", "ax = fig.add_subplot(111)\n", "#plt.axis('on')\n", "#ax = plt.gca()\n", "#plt.plot(layer_no, mean_1, '-o', label='Faces~Pareidolia', markersize=10)\n", "#plt.plot(layer_no, mean_2, '-o', label='Pareidolia~Objects', markersize=10)\n", "#plt.errorbar(layer_no, mean_1, yerr=[confidence_interval_1_0, confidence_interval_1_1], fmt='-o', color = 'tab:blue', capsize=8, label='Faces~Pareidolia', markersize=2)\n", "#plt.errorbar(layer_no, mean_2, yerr=[confidence_interval_2_0, confidence_interval_2_1], fmt='-o', color = 'tab:orange', capsize=8, label='Pareidolia~Objects', markersize=3)\n", "#plt.errorbar(layer_no, mean_3, yerr=[confidence_interval_3_0, confidence_interval_3_1], fmt='-o', color = 'tab:green', capsize=8, label='Objects~Faces', markersize=3)\n", "\n", "plt.plot(layer_no, mean_1, color = 'tab:blue', label='Faces~Pareidolia')\n", "plt.fill_between(layer_no, mean_1-std_1, mean_1+std_1, color = 'lightblue', alpha=.3)\n", "\n", "plt.plot(layer_no, mean_2, color = 'tab:orange', label='Pareidolia~Objects')\n", "plt.fill_between(layer_no, mean_2-std_2, mean_2+std_2, color = 'peachpuff', alpha=.3)\n", "\n", "# plt.plot(layer_no, mean_3, color = 'tab:green', label='Objects~Faces')\n", "# plt.fill_between(layer_no, mean_3-std_3, mean_3+std_3, color = 'lightgreen', alpha=.3)\n", "\n", "#plt.plot(layer_no, spearmanr_values_hypo_3, '-o', label='Faces~Objects', markersize=10)\n", "#plt.axhline(y=0.5, color='r', linestyle='--', label='Random chance')\n", "#plt.axvline(x=4, color='k', linestyle='--')\n", "#plt.axvline(x=13,color='k', linestyle='--')\n", "\n", "plt.xticks(layer_no, fontsize=30)\n", "plt.yticks(fontsize=30)\n", "#ax.tick_params(axis='both', which='major')\n", "ax.set_xticklabels(layer_no)\n", "plt.axis([0, len(layers)+1, -0.2, 0.3])\n", "plt.xlabel('CNN layer', fontsize=30)\n", "plt.ylabel(\"CNN idealized model correlation \\n [Partial Spearman's r]\", fontsize=30)\n", "plt.title('DG1_RSA', fontsize=30)\n", "#ax.grid(which='both')\n", "#ax.grid(which='minor', alpha=0.2)\n", "#ax.grid(which='major', alpha=0.5)\n", "#ax.grid(color='k', alpha=1, linestyle='--')\n", "#plt.legend(loc = 4, prop={'size': 20})\n", "leg = plt.legend(loc = 2, prop={'size': 20})\n", "leg.get_frame().set_edgecolor('k')\n", "#fig.set_size_inches(14.,8.)\n", "plt.tight_layout()\n", "plt.savefig('DG1_RSA.png', dpi=600)\n", "\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": null, "id": "a0eac54e", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "5f71f0c7", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "04f5607f", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "8da34189", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "82e25b3b", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "f7817cbe", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "d1b73b99", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.5" } }, "nbformat": 4, "nbformat_minor": 5 }