{ "cells": [ { "cell_type": "code", "execution_count": 25, "id": "ef9e1556-7840-4004-b181-a2c97ac2ab17", "metadata": {}, "outputs": [], "source": [ "import os\n", "import torch\n", "import torch.nn as nn\n", "import numpy as np\n", "import matplotlib.pyplot as plt" ] }, { "cell_type": "markdown", "id": "b6f12dd4-f3aa-4981-b604-b72e67229011", "metadata": {}, "source": [ "# DinoV2" ] }, { "cell_type": "code", "execution_count": 26, "id": "2a604617-b602-4503-b288-e9828684505e", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Using cache found in /fsx/proj-fmri/shared/cache/dinov2/hub/facebookresearch_dinov2_main\n" ] } ], "source": [ "# need to change TORCH_HOME env variable to specify pretrained model should go in shared folder, not home directory\n", "os.environ['TORCH_HOME'] = '/fsx/proj-fmri/shared/cache/dinov2'\n", "dinov2_model = torch.hub.load('facebookresearch/dinov2', 'dinov2_vitl14')\n", "# remove initial image patching\n", "dinov2_model.patch_embed = nn.Identity()\n", "dinov2_model.patch_embed = nn.Identity()" ] }, { "cell_type": "code", "execution_count": 27, "id": "32da913d-d931-4967-a5e8-bd40c21d1ad9", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "torch.Size([2, 33, 1024])\n" ] } ], "source": [ "dinov2_model.to(\"cuda\")\n", "input = torch.randn((2,33,1024)).to(\"cuda\")\n", "\n", "for block in dinov2_model.blocks: input = block(input)\n", "input = dinov2_model.norm(input)\n", "\n", "print(input.shape)" ] }, { "cell_type": "markdown", "id": "febe89c0-06d0-4309-b378-a8d58b99bf4c", "metadata": {}, "source": [ "# eva" ] }, { "cell_type": "code", "execution_count": 28, "id": "690204d0-13d7-452b-97af-14d144800e81", "metadata": {}, "outputs": [], "source": [ "from urllib.request import urlopen\n", "from PIL import Image\n", "import timm\n", "\n", "img = Image.open(urlopen(\n", " 'https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/beignets-task-guide.png'\n", "))\n", "\n", "model = timm.create_model(\n", " \"eva02_enormous_patch14_clip_224.laion2b\",\n", " pretrained=True,\n", " num_classes=0, # remove classifier nn.Linear\n", ")\n", "model = model.eval()" ] }, { "cell_type": "code", "execution_count": 39, "id": "035e3e9d-86c9-4ddf-b760-7b78dded7d2e", "metadata": {}, "outputs": [ { "ename": "ValueError", "evalue": "You have to specify pixel_values", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)", "Cell \u001b[0;32mIn[39], line 5\u001b[0m\n\u001b[1;32m 2\u001b[0m data_config \u001b[38;5;241m=\u001b[39m timm\u001b[38;5;241m.\u001b[39mdata\u001b[38;5;241m.\u001b[39mresolve_model_data_config(model)\n\u001b[1;32m 3\u001b[0m transforms \u001b[38;5;241m=\u001b[39m timm\u001b[38;5;241m.\u001b[39mdata\u001b[38;5;241m.\u001b[39mcreate_transform(\u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mdata_config, is_training\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m)\n\u001b[0;32m----> 5\u001b[0m output \u001b[38;5;241m=\u001b[39m \u001b[43mmodel\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtransforms\u001b[49m\u001b[43m(\u001b[49m\u001b[43mimg\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43munsqueeze\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;66;03m# output is (batch_size, num_features) shaped tensor\u001b[39;00m\n\u001b[1;32m 6\u001b[0m \u001b[38;5;28mprint\u001b[39m(output\u001b[38;5;241m.\u001b[39mshape)\n\u001b[1;32m 7\u001b[0m \u001b[38;5;66;03m# or equivalently (without needing to set num_classes=0)\u001b[39;00m\n", "File \u001b[0;32m~/miniconda3/envs/mindeye/lib/python3.10/site-packages/torch/nn/modules/module.py:1501\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1496\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m 1497\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m 1498\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m 1499\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m 1500\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1501\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1502\u001b[0m \u001b[38;5;66;03m# Do not call functions when jit is used\u001b[39;00m\n\u001b[1;32m 1503\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[38;5;241m=\u001b[39m [], []\n", "File \u001b[0;32m~/miniconda3/envs/mindeye/lib/python3.10/site-packages/transformers/models/clipseg/modeling_clipseg.py:1433\u001b[0m, in \u001b[0;36mCLIPSegForImageSegmentation.forward\u001b[0;34m(self, input_ids, pixel_values, conditional_pixel_values, conditional_embeddings, attention_mask, position_ids, labels, output_attentions, output_hidden_states, return_dict)\u001b[0m\n\u001b[1;32m 1431\u001b[0m \u001b[38;5;66;03m# step 1: forward the query images through the frozen CLIP vision encoder\u001b[39;00m\n\u001b[1;32m 1432\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m torch\u001b[38;5;241m.\u001b[39mno_grad():\n\u001b[0;32m-> 1433\u001b[0m vision_outputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mclip\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mvision_model\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 1434\u001b[0m \u001b[43m \u001b[49m\u001b[43mpixel_values\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpixel_values\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1435\u001b[0m \u001b[43m \u001b[49m\u001b[43moutput_attentions\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moutput_attentions\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1436\u001b[0m \u001b[43m \u001b[49m\u001b[43moutput_hidden_states\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;66;43;03m# we need the intermediate hidden states\u001b[39;49;00m\n\u001b[1;32m 1437\u001b[0m \u001b[43m \u001b[49m\u001b[43mreturn_dict\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mreturn_dict\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1438\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1439\u001b[0m pooled_output \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mclip\u001b[38;5;241m.\u001b[39mvisual_projection(vision_outputs[\u001b[38;5;241m1\u001b[39m])\n\u001b[1;32m 1441\u001b[0m hidden_states \u001b[38;5;241m=\u001b[39m vision_outputs\u001b[38;5;241m.\u001b[39mhidden_states \u001b[38;5;28;01mif\u001b[39;00m return_dict \u001b[38;5;28;01melse\u001b[39;00m vision_outputs[\u001b[38;5;241m2\u001b[39m]\n", "File \u001b[0;32m~/miniconda3/envs/mindeye/lib/python3.10/site-packages/torch/nn/modules/module.py:1501\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1496\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m 1497\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m 1498\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m 1499\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m 1500\u001b[0m \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1501\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1502\u001b[0m \u001b[38;5;66;03m# Do not call functions when jit is used\u001b[39;00m\n\u001b[1;32m 1503\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[38;5;241m=\u001b[39m [], []\n", "File \u001b[0;32m~/miniconda3/envs/mindeye/lib/python3.10/site-packages/transformers/models/clipseg/modeling_clipseg.py:872\u001b[0m, in \u001b[0;36mCLIPSegVisionTransformer.forward\u001b[0;34m(self, pixel_values, output_attentions, output_hidden_states, return_dict)\u001b[0m\n\u001b[1;32m 869\u001b[0m return_dict \u001b[38;5;241m=\u001b[39m return_dict \u001b[38;5;28;01mif\u001b[39;00m return_dict \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39muse_return_dict\n\u001b[1;32m 871\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m pixel_values \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 872\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mYou have to specify pixel_values\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 874\u001b[0m hidden_states \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39membeddings(pixel_values)\n\u001b[1;32m 875\u001b[0m hidden_states \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpre_layrnorm(hidden_states)\n", "\u001b[0;31mValueError\u001b[0m: You have to specify pixel_values" ] } ], "source": [ "# get model specific transforms (normalization, resize)\n", "data_config = timm.data.resolve_model_data_config(model)\n", "transforms = timm.data.create_transform(**data_config, is_training=False)\n", "\n", "output = model(transforms(img).unsqueeze(0)) # output is (batch_size, num_features) shaped tensor\n", "print(output.shape)\n", "# or equivalently (without needing to set num_classes=0)\n", "\n", "output = model.forward_features(transforms(img).unsqueeze(0))\n", "# output is unpooled, a (1, 257, 768) shaped tensor\n", "print(output.shape)\n", "\n", "output = model.forward_head(output, pre_logits=True)\n", "# output is a (1, num_features) shaped tensor\n", "print(output.shape)" ] }, { "cell_type": "code", "execution_count": null, "id": "54275c4c-e506-4959-92f1-29e584f5ce51", "metadata": {}, "outputs": [], "source": [ "model.forward_features(transforms(img).unsqueeze(0)).shape" ] }, { "cell_type": "markdown", "id": "6546c673-f3ab-4d43-a051-cab20e782bab", "metadata": {}, "source": [ "# Eva02-clip" ] }, { "cell_type": "code", "execution_count": 29, "id": "dfbc95de-9af9-4583-98fc-b8061114ef64", "metadata": {}, "outputs": [], "source": [ "import timm \n", "# couldnt figure out how to load pretrained model from shared folder rather than home directory using timm...\n", "eva02_model = timm.create_model(\"eva02_enormous_patch14_clip_224.laion2b\", pretrained=True)\n", "# eva02_model.head_drop = nn.Identity()\n", "# eva02_model.head = nn.Identity()" ] }, { "cell_type": "code", "execution_count": 30, "id": "97e3ea29-ae6b-4bd2-b3d7-17839098a6e4", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "torch.Size([2, 1024])" ] }, "execution_count": 30, "metadata": {}, "output_type": "execute_result" } ], "source": [ "eva02_model(torch.randn((2,3,224,224))).shape" ] }, { "cell_type": "code", "execution_count": 31, "id": "069b76f0-029f-42b1-85f5-a492ee1cc5d1", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "torch.Size([2, 256, 1024])\n" ] } ], "source": [ "image = torch.randn((2,3,224,224))\n", "\n", "input = eva02_model.patch_embed(image)\n", "input = eva02_model.pos_drop(input)\n", "for block in eva02_model.blocks: input = block(input)\n", "input = eva02_model.norm(input)\n", "input = eva02_model.fc_norm(input)\n", "input = eva02_model.head_drop(input)\n", "input = eva02_model.head(input)\n", "\n", "print(input.shape)" ] }, { "cell_type": "code", "execution_count": 32, "id": "90e4e8e7-3dd1-43b0-a305-066a6ec13c2e", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Help on Eva in module timm.models.eva object:\n", "\n", "class Eva(torch.nn.modules.module.Module)\n", " | Eva(img_size: Union[int, Tuple[int, int]] = 224, patch_size: Union[int, Tuple[int, int]] = 16, in_chans: int = 3, num_classes: int = 1000, global_pool: str = 'avg', embed_dim: int = 768, depth: int = 12, num_heads: int = 12, qkv_bias: bool = True, qkv_fused: bool = True, mlp_ratio: float = 4.0, swiglu_mlp: bool = False, scale_mlp: bool = False, scale_attn_inner: bool = False, drop_rate: float = 0.0, pos_drop_rate: float = 0.0, patch_drop_rate: float = 0.0, proj_drop_rate: float = 0.0, attn_drop_rate: float = 0.0, drop_path_rate: float = 0.0, norm_layer: Callable = , init_values: Optional[float] = None, class_token: bool = True, use_abs_pos_emb: bool = True, use_rot_pos_emb: bool = False, use_post_norm: bool = False, ref_feat_shape: Union[int, Tuple[int, int], NoneType] = None, head_init_scale: float = 0.001)\n", " | \n", " | Eva Vision Transformer w/ Abs & Rotary Pos Embed\n", " | \n", " | This class implements the EVA and EVA02 models that were based on the BEiT ViT variant\n", " | * EVA - abs pos embed, global avg pool\n", " | * EVA02 - abs + rope pos embed, global avg pool, SwiGLU, scale Norm in MLP (ala normformer)\n", " | \n", " | Method resolution order:\n", " | Eva\n", " | torch.nn.modules.module.Module\n", " | builtins.object\n", " | \n", " | Methods defined here:\n", " | \n", " | __init__(self, img_size: Union[int, Tuple[int, int]] = 224, patch_size: Union[int, Tuple[int, int]] = 16, in_chans: int = 3, num_classes: int = 1000, global_pool: str = 'avg', embed_dim: int = 768, depth: int = 12, num_heads: int = 12, qkv_bias: bool = True, qkv_fused: bool = True, mlp_ratio: float = 4.0, swiglu_mlp: bool = False, scale_mlp: bool = False, scale_attn_inner: bool = False, drop_rate: float = 0.0, pos_drop_rate: float = 0.0, patch_drop_rate: float = 0.0, proj_drop_rate: float = 0.0, attn_drop_rate: float = 0.0, drop_path_rate: float = 0.0, norm_layer: Callable = , init_values: Optional[float] = None, class_token: bool = True, use_abs_pos_emb: bool = True, use_rot_pos_emb: bool = False, use_post_norm: bool = False, ref_feat_shape: Union[int, Tuple[int, int], NoneType] = None, head_init_scale: float = 0.001)\n", " | Args:\n", " | img_size:\n", " | patch_size:\n", " | in_chans:\n", " | num_classes:\n", " | global_pool:\n", " | embed_dim:\n", " | depth:\n", " | num_heads:\n", " | qkv_bias:\n", " | qkv_fused:\n", " | mlp_ratio:\n", " | swiglu_mlp:\n", " | scale_mlp:\n", " | scale_attn_inner:\n", " | drop_rate:\n", " | pos_drop_rate:\n", " | proj_drop_rate:\n", " | attn_drop_rate:\n", " | drop_path_rate:\n", " | norm_layer:\n", " | init_values:\n", " | class_token:\n", " | use_abs_pos_emb:\n", " | use_rot_pos_emb:\n", " | use_post_norm:\n", " | ref_feat_shape:\n", " | head_init_scale:\n", " | \n", " | fix_init_weight(self)\n", " | \n", " | forward(self, x)\n", " | Defines the computation performed at every call.\n", " | \n", " | Should be overridden by all subclasses.\n", " | \n", " | .. note::\n", " | Although the recipe for forward pass needs to be defined within\n", " | this function, one should call the :class:`Module` instance afterwards\n", " | instead of this since the former takes care of running the\n", " | registered hooks while the latter silently ignores them.\n", " | \n", " | forward_features(self, x)\n", " | \n", " | forward_head(self, x, pre_logits: bool = False)\n", " | \n", " | get_classifier(self)\n", " | \n", " | group_matcher(self, coarse=False)\n", " | \n", " | no_weight_decay(self)\n", " | \n", " | reset_classifier(self, num_classes, global_pool=None)\n", " | \n", " | set_grad_checkpointing(self, enable=True)\n", " | \n", " | ----------------------------------------------------------------------\n", " | Data and other attributes defined here:\n", " | \n", " | __annotations__ = {}\n", " | \n", " | ----------------------------------------------------------------------\n", " | Methods inherited from torch.nn.modules.module.Module:\n", " | \n", " | __call__ = _call_impl(self, *args, **kwargs)\n", " | \n", " | __delattr__(self, name)\n", " | Implement delattr(self, name).\n", " | \n", " | __dir__(self)\n", " | Default dir() implementation.\n", " | \n", " | __getattr__(self, name: str) -> Union[torch.Tensor, ForwardRef('Module')]\n", " | \n", " | __repr__(self)\n", " | Return repr(self).\n", " | \n", " | __setattr__(self, name: str, value: Union[torch.Tensor, ForwardRef('Module')]) -> None\n", " | Implement setattr(self, name, value).\n", " | \n", " | __setstate__(self, state)\n", " | \n", " | add_module(self, name: str, module: Optional[ForwardRef('Module')]) -> None\n", " | Adds a child module to the current module.\n", " | \n", " | The module can be accessed as an attribute using the given name.\n", " | \n", " | Args:\n", " | name (str): name of the child module. The child module can be\n", " | accessed from this module using the given name\n", " | module (Module): child module to be added to the module.\n", " | \n", " | apply(self: ~T, fn: Callable[[ForwardRef('Module')], NoneType]) -> ~T\n", " | Applies ``fn`` recursively to every submodule (as returned by ``.children()``)\n", " | as well as self. Typical use includes initializing the parameters of a model\n", " | (see also :ref:`nn-init-doc`).\n", " | \n", " | Args:\n", " | fn (:class:`Module` -> None): function to be applied to each submodule\n", " | \n", " | Returns:\n", " | Module: self\n", " | \n", " | Example::\n", " | \n", " | >>> @torch.no_grad()\n", " | >>> def init_weights(m):\n", " | >>> print(m)\n", " | >>> if type(m) == nn.Linear:\n", " | >>> m.weight.fill_(1.0)\n", " | >>> print(m.weight)\n", " | >>> net = nn.Sequential(nn.Linear(2, 2), nn.Linear(2, 2))\n", " | >>> net.apply(init_weights)\n", " | Linear(in_features=2, out_features=2, bias=True)\n", " | Parameter containing:\n", " | tensor([[1., 1.],\n", " | [1., 1.]], requires_grad=True)\n", " | Linear(in_features=2, out_features=2, bias=True)\n", " | Parameter containing:\n", " | tensor([[1., 1.],\n", " | [1., 1.]], requires_grad=True)\n", " | Sequential(\n", " | (0): Linear(in_features=2, out_features=2, bias=True)\n", " | (1): Linear(in_features=2, out_features=2, bias=True)\n", " | )\n", " | \n", " | bfloat16(self: ~T) -> ~T\n", " | Casts all floating point parameters and buffers to ``bfloat16`` datatype.\n", " | \n", " | .. note::\n", " | This method modifies the module in-place.\n", " | \n", " | Returns:\n", " | Module: self\n", " | \n", " | buffers(self, recurse: bool = True) -> Iterator[torch.Tensor]\n", " | Returns an iterator over module buffers.\n", " | \n", " | Args:\n", " | recurse (bool): if True, then yields buffers of this module\n", " | and all submodules. Otherwise, yields only buffers that\n", " | are direct members of this module.\n", " | \n", " | Yields:\n", " | torch.Tensor: module buffer\n", " | \n", " | Example::\n", " | \n", " | >>> # xdoctest: +SKIP(\"undefined vars\")\n", " | >>> for buf in model.buffers():\n", " | >>> print(type(buf), buf.size())\n", " | (20L,)\n", " | (20L, 1L, 5L, 5L)\n", " | \n", " | children(self) -> Iterator[ForwardRef('Module')]\n", " | Returns an iterator over immediate children modules.\n", " | \n", " | Yields:\n", " | Module: a child module\n", " | \n", " | cpu(self: ~T) -> ~T\n", " | Moves all model parameters and buffers to the CPU.\n", " | \n", " | .. note::\n", " | This method modifies the module in-place.\n", " | \n", " | Returns:\n", " | Module: self\n", " | \n", " | cuda(self: ~T, device: Union[int, torch.device, NoneType] = None) -> ~T\n", " | Moves all model parameters and buffers to the GPU.\n", " | \n", " | This also makes associated parameters and buffers different objects. So\n", " | it should be called before constructing optimizer if the module will\n", " | live on GPU while being optimized.\n", " | \n", " | .. note::\n", " | This method modifies the module in-place.\n", " | \n", " | Args:\n", " | device (int, optional): if specified, all parameters will be\n", " | copied to that device\n", " | \n", " | Returns:\n", " | Module: self\n", " | \n", " | double(self: ~T) -> ~T\n", " | Casts all floating point parameters and buffers to ``double`` datatype.\n", " | \n", " | .. note::\n", " | This method modifies the module in-place.\n", " | \n", " | Returns:\n", " | Module: self\n", " | \n", " | eval(self: ~T) -> ~T\n", " | Sets the module in evaluation mode.\n", " | \n", " | This has any effect only on certain modules. See documentations of\n", " | particular modules for details of their behaviors in training/evaluation\n", " | mode, if they are affected, e.g. :class:`Dropout`, :class:`BatchNorm`,\n", " | etc.\n", " | \n", " | This is equivalent with :meth:`self.train(False) `.\n", " | \n", " | See :ref:`locally-disable-grad-doc` for a comparison between\n", " | `.eval()` and several similar mechanisms that may be confused with it.\n", " | \n", " | Returns:\n", " | Module: self\n", " | \n", " | extra_repr(self) -> str\n", " | Set the extra representation of the module\n", " | \n", " | To print customized extra information, you should re-implement\n", " | this method in your own modules. Both single-line and multi-line\n", " | strings are acceptable.\n", " | \n", " | float(self: ~T) -> ~T\n", " | Casts all floating point parameters and buffers to ``float`` datatype.\n", " | \n", " | .. note::\n", " | This method modifies the module in-place.\n", " | \n", " | Returns:\n", " | Module: self\n", " | \n", " | get_buffer(self, target: str) -> 'Tensor'\n", " | Returns the buffer given by ``target`` if it exists,\n", " | otherwise throws an error.\n", " | \n", " | See the docstring for ``get_submodule`` for a more detailed\n", " | explanation of this method's functionality as well as how to\n", " | correctly specify ``target``.\n", " | \n", " | Args:\n", " | target: The fully-qualified string name of the buffer\n", " | to look for. (See ``get_submodule`` for how to specify a\n", " | fully-qualified string.)\n", " | \n", " | Returns:\n", " | torch.Tensor: The buffer referenced by ``target``\n", " | \n", " | Raises:\n", " | AttributeError: If the target string references an invalid\n", " | path or resolves to something that is not a\n", " | buffer\n", " | \n", " | get_extra_state(self) -> Any\n", " | Returns any extra state to include in the module's state_dict.\n", " | Implement this and a corresponding :func:`set_extra_state` for your module\n", " | if you need to store extra state. This function is called when building the\n", " | module's `state_dict()`.\n", " | \n", " | Note that extra state should be picklable to ensure working serialization\n", " | of the state_dict. We only provide provide backwards compatibility guarantees\n", " | for serializing Tensors; other objects may break backwards compatibility if\n", " | their serialized pickled form changes.\n", " | \n", " | Returns:\n", " | object: Any extra state to store in the module's state_dict\n", " | \n", " | get_parameter(self, target: str) -> 'Parameter'\n", " | Returns the parameter given by ``target`` if it exists,\n", " | otherwise throws an error.\n", " | \n", " | See the docstring for ``get_submodule`` for a more detailed\n", " | explanation of this method's functionality as well as how to\n", " | correctly specify ``target``.\n", " | \n", " | Args:\n", " | target: The fully-qualified string name of the Parameter\n", " | to look for. (See ``get_submodule`` for how to specify a\n", " | fully-qualified string.)\n", " | \n", " | Returns:\n", " | torch.nn.Parameter: The Parameter referenced by ``target``\n", " | \n", " | Raises:\n", " | AttributeError: If the target string references an invalid\n", " | path or resolves to something that is not an\n", " | ``nn.Parameter``\n", " | \n", " | get_submodule(self, target: str) -> 'Module'\n", " | Returns the submodule given by ``target`` if it exists,\n", " | otherwise throws an error.\n", " | \n", " | For example, let's say you have an ``nn.Module`` ``A`` that\n", " | looks like this:\n", " | \n", " | .. code-block:: text\n", " | \n", " | A(\n", " | (net_b): Module(\n", " | (net_c): Module(\n", " | (conv): Conv2d(16, 33, kernel_size=(3, 3), stride=(2, 2))\n", " | )\n", " | (linear): Linear(in_features=100, out_features=200, bias=True)\n", " | )\n", " | )\n", " | \n", " | (The diagram shows an ``nn.Module`` ``A``. ``A`` has a nested\n", " | submodule ``net_b``, which itself has two submodules ``net_c``\n", " | and ``linear``. ``net_c`` then has a submodule ``conv``.)\n", " | \n", " | To check whether or not we have the ``linear`` submodule, we\n", " | would call ``get_submodule(\"net_b.linear\")``. To check whether\n", " | we have the ``conv`` submodule, we would call\n", " | ``get_submodule(\"net_b.net_c.conv\")``.\n", " | \n", " | The runtime of ``get_submodule`` is bounded by the degree\n", " | of module nesting in ``target``. A query against\n", " | ``named_modules`` achieves the same result, but it is O(N) in\n", " | the number of transitive modules. So, for a simple check to see\n", " | if some submodule exists, ``get_submodule`` should always be\n", " | used.\n", " | \n", " | Args:\n", " | target: The fully-qualified string name of the submodule\n", " | to look for. (See above example for how to specify a\n", " | fully-qualified string.)\n", " | \n", " | Returns:\n", " | torch.nn.Module: The submodule referenced by ``target``\n", " | \n", " | Raises:\n", " | AttributeError: If the target string references an invalid\n", " | path or resolves to something that is not an\n", " | ``nn.Module``\n", " | \n", " | half(self: ~T) -> ~T\n", " | Casts all floating point parameters and buffers to ``half`` datatype.\n", " | \n", " | .. note::\n", " | This method modifies the module in-place.\n", " | \n", " | Returns:\n", " | Module: self\n", " | \n", " | ipu(self: ~T, device: Union[int, torch.device, NoneType] = None) -> ~T\n", " | Moves all model parameters and buffers to the IPU.\n", " | \n", " | This also makes associated parameters and buffers different objects. So\n", " | it should be called before constructing optimizer if the module will\n", " | live on IPU while being optimized.\n", " | \n", " | .. note::\n", " | This method modifies the module in-place.\n", " | \n", " | Arguments:\n", " | device (int, optional): if specified, all parameters will be\n", " | copied to that device\n", " | \n", " | Returns:\n", " | Module: self\n", " | \n", " | load_state_dict(self, state_dict: Mapping[str, Any], strict: bool = True)\n", " | Copies parameters and buffers from :attr:`state_dict` into\n", " | this module and its descendants. If :attr:`strict` is ``True``, then\n", " | the keys of :attr:`state_dict` must exactly match the keys returned\n", " | by this module's :meth:`~torch.nn.Module.state_dict` function.\n", " | \n", " | Args:\n", " | state_dict (dict): a dict containing parameters and\n", " | persistent buffers.\n", " | strict (bool, optional): whether to strictly enforce that the keys\n", " | in :attr:`state_dict` match the keys returned by this module's\n", " | :meth:`~torch.nn.Module.state_dict` function. Default: ``True``\n", " | \n", " | Returns:\n", " | ``NamedTuple`` with ``missing_keys`` and ``unexpected_keys`` fields:\n", " | * **missing_keys** is a list of str containing the missing keys\n", " | * **unexpected_keys** is a list of str containing the unexpected keys\n", " | \n", " | Note:\n", " | If a parameter or buffer is registered as ``None`` and its corresponding key\n", " | exists in :attr:`state_dict`, :meth:`load_state_dict` will raise a\n", " | ``RuntimeError``.\n", " | \n", " | modules(self) -> Iterator[ForwardRef('Module')]\n", " | Returns an iterator over all modules in the network.\n", " | \n", " | Yields:\n", " | Module: a module in the network\n", " | \n", " | Note:\n", " | Duplicate modules are returned only once. In the following\n", " | example, ``l`` will be returned only once.\n", " | \n", " | Example::\n", " | \n", " | >>> l = nn.Linear(2, 2)\n", " | >>> net = nn.Sequential(l, l)\n", " | >>> for idx, m in enumerate(net.modules()):\n", " | ... print(idx, '->', m)\n", " | \n", " | 0 -> Sequential(\n", " | (0): Linear(in_features=2, out_features=2, bias=True)\n", " | (1): Linear(in_features=2, out_features=2, bias=True)\n", " | )\n", " | 1 -> Linear(in_features=2, out_features=2, bias=True)\n", " | \n", " | named_buffers(self, prefix: str = '', recurse: bool = True, remove_duplicate: bool = True) -> Iterator[Tuple[str, torch.Tensor]]\n", " | Returns an iterator over module buffers, yielding both the\n", " | name of the buffer as well as the buffer itself.\n", " | \n", " | Args:\n", " | prefix (str): prefix to prepend to all buffer names.\n", " | recurse (bool, optional): if True, then yields buffers of this module\n", " | and all submodules. Otherwise, yields only buffers that\n", " | are direct members of this module. Defaults to True.\n", " | remove_duplicate (bool, optional): whether to remove the duplicated buffers in the result. Defaults to True.\n", " | \n", " | Yields:\n", " | (str, torch.Tensor): Tuple containing the name and buffer\n", " | \n", " | Example::\n", " | \n", " | >>> # xdoctest: +SKIP(\"undefined vars\")\n", " | >>> for name, buf in self.named_buffers():\n", " | >>> if name in ['running_var']:\n", " | >>> print(buf.size())\n", " | \n", " | named_children(self) -> Iterator[Tuple[str, ForwardRef('Module')]]\n", " | Returns an iterator over immediate children modules, yielding both\n", " | the name of the module as well as the module itself.\n", " | \n", " | Yields:\n", " | (str, Module): Tuple containing a name and child module\n", " | \n", " | Example::\n", " | \n", " | >>> # xdoctest: +SKIP(\"undefined vars\")\n", " | >>> for name, module in model.named_children():\n", " | >>> if name in ['conv4', 'conv5']:\n", " | >>> print(module)\n", " | \n", " | named_modules(self, memo: Optional[Set[ForwardRef('Module')]] = None, prefix: str = '', remove_duplicate: bool = True)\n", " | Returns an iterator over all modules in the network, yielding\n", " | both the name of the module as well as the module itself.\n", " | \n", " | Args:\n", " | memo: a memo to store the set of modules already added to the result\n", " | prefix: a prefix that will be added to the name of the module\n", " | remove_duplicate: whether to remove the duplicated module instances in the result\n", " | or not\n", " | \n", " | Yields:\n", " | (str, Module): Tuple of name and module\n", " | \n", " | Note:\n", " | Duplicate modules are returned only once. In the following\n", " | example, ``l`` will be returned only once.\n", " | \n", " | Example::\n", " | \n", " | >>> l = nn.Linear(2, 2)\n", " | >>> net = nn.Sequential(l, l)\n", " | >>> for idx, m in enumerate(net.named_modules()):\n", " | ... print(idx, '->', m)\n", " | \n", " | 0 -> ('', Sequential(\n", " | (0): Linear(in_features=2, out_features=2, bias=True)\n", " | (1): Linear(in_features=2, out_features=2, bias=True)\n", " | ))\n", " | 1 -> ('0', Linear(in_features=2, out_features=2, bias=True))\n", " | \n", " | named_parameters(self, prefix: str = '', recurse: bool = True, remove_duplicate: bool = True) -> Iterator[Tuple[str, torch.nn.parameter.Parameter]]\n", " | Returns an iterator over module parameters, yielding both the\n", " | name of the parameter as well as the parameter itself.\n", " | \n", " | Args:\n", " | prefix (str): prefix to prepend to all parameter names.\n", " | recurse (bool): if True, then yields parameters of this module\n", " | and all submodules. Otherwise, yields only parameters that\n", " | are direct members of this module.\n", " | remove_duplicate (bool, optional): whether to remove the duplicated\n", " | parameters in the result. Defaults to True.\n", " | \n", " | Yields:\n", " | (str, Parameter): Tuple containing the name and parameter\n", " | \n", " | Example::\n", " | \n", " | >>> # xdoctest: +SKIP(\"undefined vars\")\n", " | >>> for name, param in self.named_parameters():\n", " | >>> if name in ['bias']:\n", " | >>> print(param.size())\n", " | \n", " | parameters(self, recurse: bool = True) -> Iterator[torch.nn.parameter.Parameter]\n", " | Returns an iterator over module parameters.\n", " | \n", " | This is typically passed to an optimizer.\n", " | \n", " | Args:\n", " | recurse (bool): if True, then yields parameters of this module\n", " | and all submodules. Otherwise, yields only parameters that\n", " | are direct members of this module.\n", " | \n", " | Yields:\n", " | Parameter: module parameter\n", " | \n", " | Example::\n", " | \n", " | >>> # xdoctest: +SKIP(\"undefined vars\")\n", " | >>> for param in model.parameters():\n", " | >>> print(type(param), param.size())\n", " | (20L,)\n", " | (20L, 1L, 5L, 5L)\n", " | \n", " | register_backward_hook(self, hook: Callable[[ForwardRef('Module'), Union[Tuple[torch.Tensor, ...], torch.Tensor], Union[Tuple[torch.Tensor, ...], torch.Tensor]], Union[NoneType, Tuple[torch.Tensor, ...], torch.Tensor]]) -> torch.utils.hooks.RemovableHandle\n", " | Registers a backward hook on the module.\n", " | \n", " | This function is deprecated in favor of :meth:`~torch.nn.Module.register_full_backward_hook` and\n", " | the behavior of this function will change in future versions.\n", " | \n", " | Returns:\n", " | :class:`torch.utils.hooks.RemovableHandle`:\n", " | a handle that can be used to remove the added hook by calling\n", " | ``handle.remove()``\n", " | \n", " | register_buffer(self, name: str, tensor: Optional[torch.Tensor], persistent: bool = True) -> None\n", " | Adds a buffer to the module.\n", " | \n", " | This is typically used to register a buffer that should not to be\n", " | considered a model parameter. For example, BatchNorm's ``running_mean``\n", " | is not a parameter, but is part of the module's state. Buffers, by\n", " | default, are persistent and will be saved alongside parameters. This\n", " | behavior can be changed by setting :attr:`persistent` to ``False``. The\n", " | only difference between a persistent buffer and a non-persistent buffer\n", " | is that the latter will not be a part of this module's\n", " | :attr:`state_dict`.\n", " | \n", " | Buffers can be accessed as attributes using given names.\n", " | \n", " | Args:\n", " | name (str): name of the buffer. The buffer can be accessed\n", " | from this module using the given name\n", " | tensor (Tensor or None): buffer to be registered. If ``None``, then operations\n", " | that run on buffers, such as :attr:`cuda`, are ignored. If ``None``,\n", " | the buffer is **not** included in the module's :attr:`state_dict`.\n", " | persistent (bool): whether the buffer is part of this module's\n", " | :attr:`state_dict`.\n", " | \n", " | Example::\n", " | \n", " | >>> # xdoctest: +SKIP(\"undefined vars\")\n", " | >>> self.register_buffer('running_mean', torch.zeros(num_features))\n", " | \n", " | register_forward_hook(self, hook: Union[Callable[[~T, Tuple[Any, ...], Any], Optional[Any]], Callable[[~T, Tuple[Any, ...], Dict[str, Any], Any], Optional[Any]]], *, prepend: bool = False, with_kwargs: bool = False) -> torch.utils.hooks.RemovableHandle\n", " | Registers a forward hook on the module.\n", " | \n", " | The hook will be called every time after :func:`forward` has computed an output.\n", " | \n", " | If ``with_kwargs`` is ``False`` or not specified, the input contains only\n", " | the positional arguments given to the module. Keyword arguments won't be\n", " | passed to the hooks and only to the ``forward``. The hook can modify the\n", " | output. It can modify the input inplace but it will not have effect on\n", " | forward since this is called after :func:`forward` is called. The hook\n", " | should have the following signature::\n", " | \n", " | hook(module, args, output) -> None or modified output\n", " | \n", " | If ``with_kwargs`` is ``True``, the forward hook will be passed the\n", " | ``kwargs`` given to the forward function and be expected to return the\n", " | output possibly modified. The hook should have the following signature::\n", " | \n", " | hook(module, args, kwargs, output) -> None or modified output\n", " | \n", " | Args:\n", " | hook (Callable): The user defined hook to be registered.\n", " | prepend (bool): If ``True``, the provided ``hook`` will be fired\n", " | before all existing ``forward`` hooks on this\n", " | :class:`torch.nn.modules.Module`. Otherwise, the provided\n", " | ``hook`` will be fired after all existing ``forward`` hooks on\n", " | this :class:`torch.nn.modules.Module`. Note that global\n", " | ``forward`` hooks registered with\n", " | :func:`register_module_forward_hook` will fire before all hooks\n", " | registered by this method.\n", " | Default: ``False``\n", " | with_kwargs (bool): If ``True``, the ``hook`` will be passed the\n", " | kwargs given to the forward function.\n", " | Default: ``False``\n", " | \n", " | Returns:\n", " | :class:`torch.utils.hooks.RemovableHandle`:\n", " | a handle that can be used to remove the added hook by calling\n", " | ``handle.remove()``\n", " | \n", " | register_forward_pre_hook(self, hook: Union[Callable[[~T, Tuple[Any, ...]], Optional[Any]], Callable[[~T, Tuple[Any, ...], Dict[str, Any]], Optional[Tuple[Any, Dict[str, Any]]]]], *, prepend: bool = False, with_kwargs: bool = False) -> torch.utils.hooks.RemovableHandle\n", " | Registers a forward pre-hook on the module.\n", " | \n", " | The hook will be called every time before :func:`forward` is invoked.\n", " | \n", " | \n", " | If ``with_kwargs`` is false or not specified, the input contains only\n", " | the positional arguments given to the module. Keyword arguments won't be\n", " | passed to the hooks and only to the ``forward``. The hook can modify the\n", " | input. User can either return a tuple or a single modified value in the\n", " | hook. We will wrap the value into a tuple if a single value is returned\n", " | (unless that value is already a tuple). The hook should have the\n", " | following signature::\n", " | \n", " | hook(module, args) -> None or modified input\n", " | \n", " | If ``with_kwargs`` is true, the forward pre-hook will be passed the\n", " | kwargs given to the forward function. And if the hook modifies the\n", " | input, both the args and kwargs should be returned. The hook should have\n", " | the following signature::\n", " | \n", " | hook(module, args, kwargs) -> None or a tuple of modified input and kwargs\n", " | \n", " | Args:\n", " | hook (Callable): The user defined hook to be registered.\n", " | prepend (bool): If true, the provided ``hook`` will be fired before\n", " | all existing ``forward_pre`` hooks on this\n", " | :class:`torch.nn.modules.Module`. Otherwise, the provided\n", " | ``hook`` will be fired after all existing ``forward_pre`` hooks\n", " | on this :class:`torch.nn.modules.Module`. Note that global\n", " | ``forward_pre`` hooks registered with\n", " | :func:`register_module_forward_pre_hook` will fire before all\n", " | hooks registered by this method.\n", " | Default: ``False``\n", " | with_kwargs (bool): If true, the ``hook`` will be passed the kwargs\n", " | given to the forward function.\n", " | Default: ``False``\n", " | \n", " | Returns:\n", " | :class:`torch.utils.hooks.RemovableHandle`:\n", " | a handle that can be used to remove the added hook by calling\n", " | ``handle.remove()``\n", " | \n", " | register_full_backward_hook(self, hook: Callable[[ForwardRef('Module'), Union[Tuple[torch.Tensor, ...], torch.Tensor], Union[Tuple[torch.Tensor, ...], torch.Tensor]], Union[NoneType, Tuple[torch.Tensor, ...], torch.Tensor]], prepend: bool = False) -> torch.utils.hooks.RemovableHandle\n", " | Registers a backward hook on the module.\n", " | \n", " | The hook will be called every time the gradients with respect to a module\n", " | are computed, i.e. the hook will execute if and only if the gradients with\n", " | respect to module outputs are computed. The hook should have the following\n", " | signature::\n", " | \n", " | hook(module, grad_input, grad_output) -> tuple(Tensor) or None\n", " | \n", " | The :attr:`grad_input` and :attr:`grad_output` are tuples that contain the gradients\n", " | with respect to the inputs and outputs respectively. The hook should\n", " | not modify its arguments, but it can optionally return a new gradient with\n", " | respect to the input that will be used in place of :attr:`grad_input` in\n", " | subsequent computations. :attr:`grad_input` will only correspond to the inputs given\n", " | as positional arguments and all kwarg arguments are ignored. Entries\n", " | in :attr:`grad_input` and :attr:`grad_output` will be ``None`` for all non-Tensor\n", " | arguments.\n", " | \n", " | For technical reasons, when this hook is applied to a Module, its forward function will\n", " | receive a view of each Tensor passed to the Module. Similarly the caller will receive a view\n", " | of each Tensor returned by the Module's forward function.\n", " | \n", " | .. warning ::\n", " | Modifying inputs or outputs inplace is not allowed when using backward hooks and\n", " | will raise an error.\n", " | \n", " | Args:\n", " | hook (Callable): The user-defined hook to be registered.\n", " | prepend (bool): If true, the provided ``hook`` will be fired before\n", " | all existing ``backward`` hooks on this\n", " | :class:`torch.nn.modules.Module`. Otherwise, the provided\n", " | ``hook`` will be fired after all existing ``backward`` hooks on\n", " | this :class:`torch.nn.modules.Module`. Note that global\n", " | ``backward`` hooks registered with\n", " | :func:`register_module_full_backward_hook` will fire before\n", " | all hooks registered by this method.\n", " | \n", " | Returns:\n", " | :class:`torch.utils.hooks.RemovableHandle`:\n", " | a handle that can be used to remove the added hook by calling\n", " | ``handle.remove()``\n", " | \n", " | register_full_backward_pre_hook(self, hook: Callable[[ForwardRef('Module'), Union[Tuple[torch.Tensor, ...], torch.Tensor]], Union[NoneType, Tuple[torch.Tensor, ...], torch.Tensor]], prepend: bool = False) -> torch.utils.hooks.RemovableHandle\n", " | Registers a backward pre-hook on the module.\n", " | \n", " | The hook will be called every time the gradients for the module are computed.\n", " | The hook should have the following signature::\n", " | \n", " | hook(module, grad_output) -> Tensor or None\n", " | \n", " | The :attr:`grad_output` is a tuple. The hook should\n", " | not modify its arguments, but it can optionally return a new gradient with\n", " | respect to the output that will be used in place of :attr:`grad_output` in\n", " | subsequent computations. Entries in :attr:`grad_output` will be ``None`` for\n", " | all non-Tensor arguments.\n", " | \n", " | For technical reasons, when this hook is applied to a Module, its forward function will\n", " | receive a view of each Tensor passed to the Module. Similarly the caller will receive a view\n", " | of each Tensor returned by the Module's forward function.\n", " | \n", " | .. warning ::\n", " | Modifying inputs inplace is not allowed when using backward hooks and\n", " | will raise an error.\n", " | \n", " | Args:\n", " | hook (Callable): The user-defined hook to be registered.\n", " | prepend (bool): If true, the provided ``hook`` will be fired before\n", " | all existing ``backward_pre`` hooks on this\n", " | :class:`torch.nn.modules.Module`. Otherwise, the provided\n", " | ``hook`` will be fired after all existing ``backward_pre`` hooks\n", " | on this :class:`torch.nn.modules.Module`. Note that global\n", " | ``backward_pre`` hooks registered with\n", " | :func:`register_module_full_backward_pre_hook` will fire before\n", " | all hooks registered by this method.\n", " | \n", " | Returns:\n", " | :class:`torch.utils.hooks.RemovableHandle`:\n", " | a handle that can be used to remove the added hook by calling\n", " | ``handle.remove()``\n", " | \n", " | register_load_state_dict_post_hook(self, hook)\n", " | Registers a post hook to be run after module's ``load_state_dict``\n", " | is called.\n", " | \n", " | It should have the following signature::\n", " | hook(module, incompatible_keys) -> None\n", " | \n", " | The ``module`` argument is the current module that this hook is registered\n", " | on, and the ``incompatible_keys`` argument is a ``NamedTuple`` consisting\n", " | of attributes ``missing_keys`` and ``unexpected_keys``. ``missing_keys``\n", " | is a ``list`` of ``str`` containing the missing keys and\n", " | ``unexpected_keys`` is a ``list`` of ``str`` containing the unexpected keys.\n", " | \n", " | The given incompatible_keys can be modified inplace if needed.\n", " | \n", " | Note that the checks performed when calling :func:`load_state_dict` with\n", " | ``strict=True`` are affected by modifications the hook makes to\n", " | ``missing_keys`` or ``unexpected_keys``, as expected. Additions to either\n", " | set of keys will result in an error being thrown when ``strict=True``, and\n", " | clearing out both missing and unexpected keys will avoid an error.\n", " | \n", " | Returns:\n", " | :class:`torch.utils.hooks.RemovableHandle`:\n", " | a handle that can be used to remove the added hook by calling\n", " | ``handle.remove()``\n", " | \n", " | register_module(self, name: str, module: Optional[ForwardRef('Module')]) -> None\n", " | Alias for :func:`add_module`.\n", " | \n", " | register_parameter(self, name: str, param: Optional[torch.nn.parameter.Parameter]) -> None\n", " | Adds a parameter to the module.\n", " | \n", " | The parameter can be accessed as an attribute using given name.\n", " | \n", " | Args:\n", " | name (str): name of the parameter. The parameter can be accessed\n", " | from this module using the given name\n", " | param (Parameter or None): parameter to be added to the module. If\n", " | ``None``, then operations that run on parameters, such as :attr:`cuda`,\n", " | are ignored. If ``None``, the parameter is **not** included in the\n", " | module's :attr:`state_dict`.\n", " | \n", " | register_state_dict_pre_hook(self, hook)\n", " | These hooks will be called with arguments: ``self``, ``prefix``,\n", " | and ``keep_vars`` before calling ``state_dict`` on ``self``. The registered\n", " | hooks can be used to perform pre-processing before the ``state_dict``\n", " | call is made.\n", " | \n", " | requires_grad_(self: ~T, requires_grad: bool = True) -> ~T\n", " | Change if autograd should record operations on parameters in this\n", " | module.\n", " | \n", " | This method sets the parameters' :attr:`requires_grad` attributes\n", " | in-place.\n", " | \n", " | This method is helpful for freezing part of the module for finetuning\n", " | or training parts of a model individually (e.g., GAN training).\n", " | \n", " | See :ref:`locally-disable-grad-doc` for a comparison between\n", " | `.requires_grad_()` and several similar mechanisms that may be confused with it.\n", " | \n", " | Args:\n", " | requires_grad (bool): whether autograd should record operations on\n", " | parameters in this module. Default: ``True``.\n", " | \n", " | Returns:\n", " | Module: self\n", " | \n", " | set_extra_state(self, state: Any)\n", " | This function is called from :func:`load_state_dict` to handle any extra state\n", " | found within the `state_dict`. Implement this function and a corresponding\n", " | :func:`get_extra_state` for your module if you need to store extra state within its\n", " | `state_dict`.\n", " | \n", " | Args:\n", " | state (dict): Extra state from the `state_dict`\n", " | \n", " | share_memory(self: ~T) -> ~T\n", " | See :meth:`torch.Tensor.share_memory_`\n", " | \n", " | state_dict(self, *args, destination=None, prefix='', keep_vars=False)\n", " | Returns a dictionary containing references to the whole state of the module.\n", " | \n", " | Both parameters and persistent buffers (e.g. running averages) are\n", " | included. Keys are corresponding parameter and buffer names.\n", " | Parameters and buffers set to ``None`` are not included.\n", " | \n", " | .. note::\n", " | The returned object is a shallow copy. It contains references\n", " | to the module's parameters and buffers.\n", " | \n", " | .. warning::\n", " | Currently ``state_dict()`` also accepts positional arguments for\n", " | ``destination``, ``prefix`` and ``keep_vars`` in order. However,\n", " | this is being deprecated and keyword arguments will be enforced in\n", " | future releases.\n", " | \n", " | .. warning::\n", " | Please avoid the use of argument ``destination`` as it is not\n", " | designed for end-users.\n", " | \n", " | Args:\n", " | destination (dict, optional): If provided, the state of module will\n", " | be updated into the dict and the same object is returned.\n", " | Otherwise, an ``OrderedDict`` will be created and returned.\n", " | Default: ``None``.\n", " | prefix (str, optional): a prefix added to parameter and buffer\n", " | names to compose the keys in state_dict. Default: ``''``.\n", " | keep_vars (bool, optional): by default the :class:`~torch.Tensor` s\n", " | returned in the state dict are detached from autograd. If it's\n", " | set to ``True``, detaching will not be performed.\n", " | Default: ``False``.\n", " | \n", " | Returns:\n", " | dict:\n", " | a dictionary containing a whole state of the module\n", " | \n", " | Example::\n", " | \n", " | >>> # xdoctest: +SKIP(\"undefined vars\")\n", " | >>> module.state_dict().keys()\n", " | ['bias', 'weight']\n", " | \n", " | to(self, *args, **kwargs)\n", " | Moves and/or casts the parameters and buffers.\n", " | \n", " | This can be called as\n", " | \n", " | .. function:: to(device=None, dtype=None, non_blocking=False)\n", " | :noindex:\n", " | \n", " | .. function:: to(dtype, non_blocking=False)\n", " | :noindex:\n", " | \n", " | .. function:: to(tensor, non_blocking=False)\n", " | :noindex:\n", " | \n", " | .. function:: to(memory_format=torch.channels_last)\n", " | :noindex:\n", " | \n", " | Its signature is similar to :meth:`torch.Tensor.to`, but only accepts\n", " | floating point or complex :attr:`dtype`\\ s. In addition, this method will\n", " | only cast the floating point or complex parameters and buffers to :attr:`dtype`\n", " | (if given). The integral parameters and buffers will be moved\n", " | :attr:`device`, if that is given, but with dtypes unchanged. When\n", " | :attr:`non_blocking` is set, it tries to convert/move asynchronously\n", " | with respect to the host if possible, e.g., moving CPU Tensors with\n", " | pinned memory to CUDA devices.\n", " | \n", " | See below for examples.\n", " | \n", " | .. note::\n", " | This method modifies the module in-place.\n", " | \n", " | Args:\n", " | device (:class:`torch.device`): the desired device of the parameters\n", " | and buffers in this module\n", " | dtype (:class:`torch.dtype`): the desired floating point or complex dtype of\n", " | the parameters and buffers in this module\n", " | tensor (torch.Tensor): Tensor whose dtype and device are the desired\n", " | dtype and device for all parameters and buffers in this module\n", " | memory_format (:class:`torch.memory_format`): the desired memory\n", " | format for 4D parameters and buffers in this module (keyword\n", " | only argument)\n", " | \n", " | Returns:\n", " | Module: self\n", " | \n", " | Examples::\n", " | \n", " | >>> # xdoctest: +IGNORE_WANT(\"non-deterministic\")\n", " | >>> linear = nn.Linear(2, 2)\n", " | >>> linear.weight\n", " | Parameter containing:\n", " | tensor([[ 0.1913, -0.3420],\n", " | [-0.5113, -0.2325]])\n", " | >>> linear.to(torch.double)\n", " | Linear(in_features=2, out_features=2, bias=True)\n", " | >>> linear.weight\n", " | Parameter containing:\n", " | tensor([[ 0.1913, -0.3420],\n", " | [-0.5113, -0.2325]], dtype=torch.float64)\n", " | >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_CUDA1)\n", " | >>> gpu1 = torch.device(\"cuda:1\")\n", " | >>> linear.to(gpu1, dtype=torch.half, non_blocking=True)\n", " | Linear(in_features=2, out_features=2, bias=True)\n", " | >>> linear.weight\n", " | Parameter containing:\n", " | tensor([[ 0.1914, -0.3420],\n", " | [-0.5112, -0.2324]], dtype=torch.float16, device='cuda:1')\n", " | >>> cpu = torch.device(\"cpu\")\n", " | >>> linear.to(cpu)\n", " | Linear(in_features=2, out_features=2, bias=True)\n", " | >>> linear.weight\n", " | Parameter containing:\n", " | tensor([[ 0.1914, -0.3420],\n", " | [-0.5112, -0.2324]], dtype=torch.float16)\n", " | \n", " | >>> linear = nn.Linear(2, 2, bias=None).to(torch.cdouble)\n", " | >>> linear.weight\n", " | Parameter containing:\n", " | tensor([[ 0.3741+0.j, 0.2382+0.j],\n", " | [ 0.5593+0.j, -0.4443+0.j]], dtype=torch.complex128)\n", " | >>> linear(torch.ones(3, 2, dtype=torch.cdouble))\n", " | tensor([[0.6122+0.j, 0.1150+0.j],\n", " | [0.6122+0.j, 0.1150+0.j],\n", " | [0.6122+0.j, 0.1150+0.j]], dtype=torch.complex128)\n", " | \n", " | to_empty(self: ~T, *, device: Union[str, torch.device]) -> ~T\n", " | Moves the parameters and buffers to the specified device without copying storage.\n", " | \n", " | Args:\n", " | device (:class:`torch.device`): The desired device of the parameters\n", " | and buffers in this module.\n", " | \n", " | Returns:\n", " | Module: self\n", " | \n", " | train(self: ~T, mode: bool = True) -> ~T\n", " | Sets the module in training mode.\n", " | \n", " | This has any effect only on certain modules. See documentations of\n", " | particular modules for details of their behaviors in training/evaluation\n", " | mode, if they are affected, e.g. :class:`Dropout`, :class:`BatchNorm`,\n", " | etc.\n", " | \n", " | Args:\n", " | mode (bool): whether to set training mode (``True``) or evaluation\n", " | mode (``False``). Default: ``True``.\n", " | \n", " | Returns:\n", " | Module: self\n", " | \n", " | type(self: ~T, dst_type: Union[torch.dtype, str]) -> ~T\n", " | Casts all parameters and buffers to :attr:`dst_type`.\n", " | \n", " | .. note::\n", " | This method modifies the module in-place.\n", " | \n", " | Args:\n", " | dst_type (type or string): the desired type\n", " | \n", " | Returns:\n", " | Module: self\n", " | \n", " | xpu(self: ~T, device: Union[int, torch.device, NoneType] = None) -> ~T\n", " | Moves all model parameters and buffers to the XPU.\n", " | \n", " | This also makes associated parameters and buffers different objects. So\n", " | it should be called before constructing optimizer if the module will\n", " | live on XPU while being optimized.\n", " | \n", " | .. note::\n", " | This method modifies the module in-place.\n", " | \n", " | Arguments:\n", " | device (int, optional): if specified, all parameters will be\n", " | copied to that device\n", " | \n", " | Returns:\n", " | Module: self\n", " | \n", " | zero_grad(self, set_to_none: bool = True) -> None\n", " | Sets gradients of all model parameters to zero. See similar function\n", " | under :class:`torch.optim.Optimizer` for more context.\n", " | \n", " | Args:\n", " | set_to_none (bool): instead of setting to zero, set the grads to None.\n", " | See :meth:`torch.optim.Optimizer.zero_grad` for details.\n", " | \n", " | ----------------------------------------------------------------------\n", " | Data descriptors inherited from torch.nn.modules.module.Module:\n", " | \n", " | __dict__\n", " | dictionary for instance variables (if defined)\n", " | \n", " | __weakref__\n", " | list of weak references to the object (if defined)\n", " | \n", " | ----------------------------------------------------------------------\n", " | Data and other attributes inherited from torch.nn.modules.module.Module:\n", " | \n", " | T_destination = ~T_destination\n", " | \n", " | call_super_init = False\n", " | \n", " | dump_patches = False\n", "\n" ] } ], "source": [ "help(eva02_model)" ] }, { "cell_type": "markdown", "id": "2f5ac1a7-6f1b-4417-8a67-1b2e32d385dd", "metadata": {}, "source": [ "# DETR" ] }, { "cell_type": "code", "execution_count": 33, "id": "5c3ade1b-18ea-4368-abd9-53be1fdfb610", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[2023-08-28 01:51:14,033] [INFO] [real_accelerator.py:133:get_accelerator] Setting ds_accelerator to cuda (auto detect)\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "The `max_size` parameter is deprecated and will be removed in v4.26. Please specify in `size['longest_edge'] instead`.\n" ] } ], "source": [ "from transformers import DetrImageProcessor, DetrForObjectDetection\n", "import torch\n", "from PIL import Image\n", "import requests\n", "\n", "url = \"http://images.cocodataset.org/val2017/000000039769.jpg\"\n", "image = Image.open(requests.get(url, stream=True).raw)\n", "\n", "processor = DetrImageProcessor.from_pretrained(\"facebook/detr-resnet-50\", cache_dir='/fsx/proj-fmri/shared/cache')\n", "model = DetrForObjectDetection.from_pretrained(\"facebook/detr-resnet-50\", cache_dir='/fsx/proj-fmri/shared/cache')" ] }, { "cell_type": "code", "execution_count": 34, "id": "1d5aa2d7-4868-4751-8d90-7c52be028cd9", "metadata": {}, "outputs": [], "source": [ "inputs = processor(images=image, return_tensors=\"pt\")\n", "outputs = model(**inputs)" ] }, { "cell_type": "code", "execution_count": 35, "id": "ae6bafc6-cee4-4e59-b7ba-12efc2a65b74", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Detected remote with confidence 0.998 at location [40.16, 70.81, 175.55, 117.98]\n", "Detected remote with confidence 0.996 at location [333.24, 72.55, 368.33, 187.66]\n", "Detected couch with confidence 0.995 at location [-0.02, 1.15, 639.73, 473.76]\n", "Detected cat with confidence 0.999 at location [13.24, 52.05, 314.02, 470.93]\n", "Detected cat with confidence 0.999 at location [345.4, 23.85, 640.37, 368.72]\n" ] } ], "source": [ "# convert outputs (bounding boxes and class logits) to COCO API\n", "# let's only keep detections with score > 0.9\n", "target_sizes = torch.tensor([image.size[::-1]])\n", "results = processor.post_process_object_detection(outputs, target_sizes=target_sizes, threshold=0.9)[0]\n", "\n", "for score, label, box in zip(results[\"scores\"], results[\"labels\"], results[\"boxes\"]):\n", " box = [round(i, 2) for i in box.tolist()]\n", " print(\n", " f\"Detected {model.config.id2label[label.item()]} with confidence \"\n", " f\"{round(score.item(), 3)} at location {box}\"\n", " )" ] }, { "cell_type": "code", "execution_count": 36, "id": "6dcc5934-79d4-4062-8b32-e42b3ebcdc0f", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "DetrImageProcessor {\n", " \"do_normalize\": true,\n", " \"do_pad\": true,\n", " \"do_rescale\": true,\n", " \"do_resize\": true,\n", " \"feature_extractor_type\": \"DetrFeatureExtractor\",\n", " \"format\": \"coco_detection\",\n", " \"image_mean\": [\n", " 0.485,\n", " 0.456,\n", " 0.406\n", " ],\n", " \"image_processor_type\": \"DetrImageProcessor\",\n", " \"image_std\": [\n", " 0.229,\n", " 0.224,\n", " 0.225\n", " ],\n", " \"resample\": 2,\n", " \"rescale_factor\": 0.00392156862745098,\n", " \"size\": {\n", " \"longest_edge\": 1333,\n", " \"shortest_edge\": 800\n", " }\n", "}" ] }, "execution_count": 36, "metadata": {}, "output_type": "execute_result" } ], "source": [ "processor" ] }, { "cell_type": "markdown", "id": "db1d89cc-b432-473e-af69-d81c435ac731", "metadata": {}, "source": [ "# CLIPSeg" ] }, { "cell_type": "code", "execution_count": 37, "id": "15db14d1-ee4d-4429-9286-054c4498293b", "metadata": {}, "outputs": [], "source": [ "from transformers import CLIPSegProcessor, CLIPSegForImageSegmentation\n", "\n", "processor = CLIPSegProcessor.from_pretrained(\"CIDAS/clipseg-rd16\",cache_dir='/fsx/proj-fmri/shared/cache')\n", "model = CLIPSegForImageSegmentation.from_pretrained(\"CIDAS/clipseg-rd16\",cache_dir='/fsx/proj-fmri/shared/cache')" ] }, { "cell_type": "code", "execution_count": 38, "id": "4aa225d4-5a3b-4dbb-ae57-dea2872ff492", "metadata": {}, "outputs": [ { "ename": "AttributeError", "evalue": "'JpegImageFile' object has no attribute 'shape'", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)", "Cell \u001b[0;32mIn[38], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mimage\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mshape\u001b[49m\n", "\u001b[0;31mAttributeError\u001b[0m: 'JpegImageFile' object has no attribute 'shape'" ] } ], "source": [ "image.shape" ] }, { "cell_type": "code", "execution_count": null, "id": "ad7e2daf-0c7c-4fec-b29e-9ba47a037c6b", "metadata": {}, "outputs": [], "source": [ "from PIL import Image\n", "import requests\n", "import h5py\n", "\n", "# url = \"https://unsplash.com/photos/8Nc_oQsc2qQ/download?ixid=MnwxMjA3fDB8MXxhbGx8fHx8fHx8fHwxNjcxMjAwNzI0&force=true&w=640\"\n", "# image = Image.open(requests.get(url, stream=True).raw)\n", "\n", "image_path = \"/fsx/proj-fmri/shared/mindeyev2_dataset/coco_images_224_float16.hdf5\"\n", "with h5py.File(image_path, 'r') as file:\n", " image = file['images'][0]\n", "image = np.moveaxis(image, 0, -1).astype(np.float32)\n", "plt.imshow(image)\n", "\n", "prompts = [\"person\",\"animal\",\"object\",\"background\"]\n", "import torch\n", "\n", "# Rescale to [0, 255]\n", "array = (image * 255).astype(np.uint8)\n", "\n", "# Convert to PIL image\n", "image = Image.fromarray(array)\n", "\n", "inputs = processor(text=prompts, images=[image] * len(prompts), padding=\"max_length\", return_tensors=\"pt\")\n", "# predict\n", "with torch.no_grad():\n", " outputs = model(**inputs)\n", "preds = outputs.logits.unsqueeze(1)\n", "print(preds.shape)" ] }, { "cell_type": "code", "execution_count": null, "id": "131eb5b7-2f16-4a79-8402-edc1a1d8c348", "metadata": {}, "outputs": [], "source": [ "preds = ((preds[0] + preds[1] + preds[2] + preds[-1].max() - preds[-1]) / 4)[None]\n", "preds.shape" ] }, { "cell_type": "code", "execution_count": null, "id": "e2bf99e7-064d-4c22-997f-aa1a35dbab82", "metadata": {}, "outputs": [], "source": [ "_, ax = plt.subplots(1, len(prompts) + 1, figsize=(3*(len(prompts) + 1), 4))\n", "[a.axis('off') for a in ax.flatten()]\n", "ax[0].imshow(image)\n", "[ax[i+1].imshow(torch.sigmoid(preds[i][0])) for i in range(1)];\n", "# [ax[i+1].text(0, -15, prompt) for i, prompt in enumerate(prompts)];" ] }, { "cell_type": "code", "execution_count": null, "id": "b58b926f-a2b2-423b-b367-18808cf6b4f7", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.8" } }, "nbformat": 4, "nbformat_minor": 5 }