{"action":"created","comment":{"url":"https://api.github.com/repos/allenai/allennlp/pulls/comments/189009480","pull_request_review_id":121104538,"id":189009480,"diff_hunk":"@@ -0,0 +1,95 @@\n+\"\"\"\n+A ``Module`` that takes two matrices as input and returns a matrix of attentions.\n+\"\"\"\n+import math\n+\n+import torch\n+\n+from allennlp.nn import util\n+from allennlp.nn.activations import Activation\n+from torch.nn import Parameter\n+\n+from allennlp.common import Params\n+from allennlp.modules.matrix_attention import MatrixAttention\n+from overrides import overrides\n+\n+\n+@MatrixAttention.register(\"linear_matrix_attention\")\n+class LinearMatrixAttention(MatrixAttention):\n+    \"\"\"\n+    This similarity function performs a dot product between a vector of weights and some\n+    combination of the two input vectors, followed by an (optional) activation function.  The\n+    combination used is configurable.\n+\n+    If the two vectors are ``x`` and ``y``, we allow the following kinds of combinations: ``x``,\n+    ``y``, ``x*y``, ``x+y``, ``x-y``, ``x/y``, where each of those binary operations is performed\n+    elementwise.  You can list as many combinations as you want, comma separated.  For example, you\n+    might give ``x,y,x*y`` as the ``combination`` parameter to this class.  The computed similarity\n+    function would then be ``w^T [x; y; x*y] + b``, where ``w`` is a vector of weights, ``b`` is a\n+    bias parameter, and ``[;]`` is vector concatenation.\n+\n+    Note that if you want a bilinear similarity function with a diagonal weight matrix W, where the\n+    similarity function is computed as `x * w * y + b` (with `w` the diagonal of `W`), you can\n+    accomplish that with this class by using \"x*y\" for `combination`.\n+\n+    Parameters\n+    ----------\n+    tensor_1_dim : ``int``\n+        The dimension of the first tensor, ``x``, described above.  This is ``x.size()[-1]`` - the\n+        length of the vector that will go into the similarity computation.  We need this so we can\n+        build weight vectors correctly.\n+    tensor_2_dim : ``int``\n+        The dimension of the second tensor, ``y``, described above.  This is ``y.size()[-1]`` - the\n+        length of the vector that will go into the similarity computation.  We need this so we can\n+        build weight vectors correctly.\n+    combination : ``str``, optional (default=\"x,y\")\n+        Described above.\n+    activation : ``Activation``, optional (default=linear (i.e. no activation))\n+        An activation function applied after the ``w^T * [x;y] + b`` calculation.  Default is no\n+        activation.\n+    \"\"\"\n+\n+    def __init__(self,\n+                 tensor_1_dim: int,\n+                 tensor_2_dim: int,\n+                 combination: str = 'x,y',\n+                 activation: Activation = Activation.by_name('linear')()) -> None:\n+        super(LinearMatrixAttention, self).__init__()\n+        self._combination = combination\n+        combined_dim = util.get_combined_dim(combination, [tensor_1_dim, tensor_2_dim])\n+        self._weight_vector = Parameter(torch.Tensor(combined_dim))\n+        self._bias = Parameter(torch.Tensor(1))\n+        self._activation = activation\n+        self.reset_parameters()\n+\n+    def reset_parameters(self):\n+        std = math.sqrt(6 / (self._weight_vector.size(0) + 1))\n+        self._weight_vector.data.uniform_(-std, std)\n+        self._bias.data.fill_(0)\n+\n+    @overrides\n+    def forward(self, tensor_1: torch.Tensor, tensor_2: torch.Tensor) -> torch.Tensor:\n+        tiled_matrix_1 = tensor_1.unsqueeze(2).expand(tensor_1.size()[0],","path":"allennlp/modules/linear_matrix_attention.py","position":72,"original_position":72,"commit_id":"2081d0f57ba54ec2ef96cc761d1be540ac8538d1","original_commit_id":"2081d0f57ba54ec2ef96cc761d1be540ac8538d1","user":{"login":"murphp15","id":9319000,"avatar_url":"https://avatars1.githubusercontent.com/u/9319000?v=4","gravatar_id":"","url":"https://api.github.com/users/murphp15","html_url":"https://github.com/murphp15","followers_url":"https://api.github.com/users/murphp15/followers","following_url":"https://api.github.com/users/murphp15/following{/other_user}","gists_url":"https://api.github.com/users/murphp15/gists{/gist_id}","starred_url":"https://api.github.com/users/murphp15/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/murphp15/subscriptions","organizations_url":"https://api.github.com/users/murphp15/orgs","repos_url":"https://api.github.com/users/murphp15/repos","events_url":"https://api.github.com/users/murphp15/events{/privacy}","received_events_url":"https://api.github.com/users/murphp15/received_events","type":"User","site_admin":false},"body":"should block? Did you mean should *not* block? ","created_at":"2018-05-17T15:50:07Z","updated_at":"2018-05-17T15:50:07Z","html_url":"https://github.com/allenai/allennlp/pull/1235#discussion_r189009480","pull_request_url":"https://api.github.com/repos/allenai/allennlp/pulls/1235","author_association":"CONTRIBUTOR","_links":{"self":{"href":"https://api.github.com/repos/allenai/allennlp/pulls/comments/189009480"},"html":{"href":"https://github.com/allenai/allennlp/pull/1235#discussion_r189009480"},"pull_request":{"href":"https://api.github.com/repos/allenai/allennlp/pulls/1235"}},"in_reply_to_id":189007832},"pull_request":{"url":"https://api.github.com/repos/allenai/allennlp/pulls/1235","id":188755491,"html_url":"https://github.com/allenai/allennlp/pull/1235","diff_url":"https://github.com/allenai/allennlp/pull/1235.diff","patch_url":"https://github.com/allenai/allennlp/pull/1235.patch","issue_url":"https://api.github.com/repos/allenai/allennlp/issues/1235","number":1235,"state":"open","locked":false,"title":"[WIP] Improve performance of attention modules","user":{"login":"murphp15","id":9319000,"avatar_url":"https://avatars1.githubusercontent.com/u/9319000?v=4","gravatar_id":"","url":"https://api.github.com/users/murphp15","html_url":"https://github.com/murphp15","followers_url":"https://api.github.com/users/murphp15/followers","following_url":"https://api.github.com/users/murphp15/following{/other_user}","gists_url":"https://api.github.com/users/murphp15/gists{/gist_id}","starred_url":"https://api.github.com/users/murphp15/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/murphp15/subscriptions","organizations_url":"https://api.github.com/users/murphp15/orgs","repos_url":"https://api.github.com/users/murphp15/repos","events_url":"https://api.github.com/users/murphp15/events{/privacy}","received_events_url":"https://api.github.com/users/murphp15/received_events","type":"User","site_admin":false},"body":"This is an initial POC for getting feedback before proceeding with the entire refactor.\r\nIn this PR\r\n1. I added implementations of MatrixAttention. there is one for dot product, one for cosine product, etc.\r\n2. There is an implementation of MatrixAttention called LegacyMatrixAttention used to support backwards compatibility and use already trained models.\r\n3. There is a class called AttentionMatrixFactory which can be used to create an instance of MatrixAttention. it will determine whether to use the legacy impl or the newer impl based on the name.\r\n\r\nLet me know what you guys think, all feedback welcome. ","created_at":"2018-05-17T14:26:01Z","updated_at":"2018-05-17T15:50:07Z","closed_at":null,"merged_at":null,"merge_commit_sha":"9536b0d9d868e687bec808ae29a32f4615573334","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/allenai/allennlp/pulls/1235/commits","review_comments_url":"https://api.github.com/repos/allenai/allennlp/pulls/1235/comments","review_comment_url":"https://api.github.com/repos/allenai/allennlp/pulls/comments{/number}","comments_url":"https://api.github.com/repos/allenai/allennlp/issues/1235/comments","statuses_url":"https://api.github.com/repos/allenai/allennlp/statuses/2081d0f57ba54ec2ef96cc761d1be540ac8538d1","head":{"label":"murphp15:feature/improve-performance-of-attention","ref":"feature/improve-performance-of-attention","sha":"2081d0f57ba54ec2ef96cc761d1be540ac8538d1","user":{"login":"murphp15","id":9319000,"avatar_url":"https://avatars1.githubusercontent.com/u/9319000?v=4","gravatar_id":"","url":"https://api.github.com/users/murphp15","html_url":"https://github.com/murphp15","followers_url":"https://api.github.com/users/murphp15/followers","following_url":"https://api.github.com/users/murphp15/following{/other_user}","gists_url":"https://api.github.com/users/murphp15/gists{/gist_id}","starred_url":"https://api.github.com/users/murphp15/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/murphp15/subscriptions","organizations_url":"https://api.github.com/users/murphp15/orgs","repos_url":"https://api.github.com/users/murphp15/repos","events_url":"https://api.github.com/users/murphp15/events{/privacy}","received_events_url":"https://api.github.com/users/murphp15/received_events","type":"User","site_admin":false},"repo":{"id":133045454,"name":"allennlp","full_name":"murphp15/allennlp","owner":{"login":"murphp15","id":9319000,"avatar_url":"https://avatars1.githubusercontent.com/u/9319000?v=4","gravatar_id":"","url":"https://api.github.com/users/murphp15","html_url":"https://github.com/murphp15","followers_url":"https://api.github.com/users/murphp15/followers","following_url":"https://api.github.com/users/murphp15/following{/other_user}","gists_url":"https://api.github.com/users/murphp15/gists{/gist_id}","starred_url":"https://api.github.com/users/murphp15/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/murphp15/subscriptions","organizations_url":"https://api.github.com/users/murphp15/orgs","repos_url":"https://api.github.com/users/murphp15/repos","events_url":"https://api.github.com/users/murphp15/events{/privacy}","received_events_url":"https://api.github.com/users/murphp15/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/murphp15/allennlp","description":"An open-source NLP research library, built on PyTorch.","fork":true,"url":"https://api.github.com/repos/murphp15/allennlp","forks_url":"https://api.github.com/repos/murphp15/allennlp/forks","keys_url":"https://api.github.com/repos/murphp15/allennlp/keys{/key_id}","collaborators_url":"https://api.github.com/repos/murphp15/allennlp/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/murphp15/allennlp/teams","hooks_url":"https://api.github.com/repos/murphp15/allennlp/hooks","issue_events_url":"https://api.github.com/repos/murphp15/allennlp/issues/events{/number}","events_url":"https://api.github.com/repos/murphp15/allennlp/events","assignees_url":"https://api.github.com/repos/murphp15/allennlp/assignees{/user}","branches_url":"https://api.github.com/repos/murphp15/allennlp/branches{/branch}","tags_url":"https://api.github.com/repos/murphp15/allennlp/tags","blobs_url":"https://api.github.com/repos/murphp15/allennlp/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/murphp15/allennlp/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/murphp15/allennlp/git/refs{/sha}","trees_url":"https://api.github.com/repos/murphp15/allennlp/git/trees{/sha}","statuses_url":"https://api.github.com/repos/murphp15/allennlp/statuses/{sha}","languages_url":"https://api.github.com/repos/murphp15/allennlp/languages","stargazers_url":"https://api.github.com/repos/murphp15/allennlp/stargazers","contributors_url":"https://api.github.com/repos/murphp15/allennlp/contributors","subscribers_url":"https://api.github.com/repos/murphp15/allennlp/subscribers","subscription_url":"https://api.github.com/repos/murphp15/allennlp/subscription","commits_url":"https://api.github.com/repos/murphp15/allennlp/commits{/sha}","git_commits_url":"https://api.github.com/repos/murphp15/allennlp/git/commits{/sha}","comments_url":"https://api.github.com/repos/murphp15/allennlp/comments{/number}","issue_comment_url":"https://api.github.com/repos/murphp15/allennlp/issues/comments{/number}","contents_url":"https://api.github.com/repos/murphp15/allennlp/contents/{+path}","compare_url":"https://api.github.com/repos/murphp15/allennlp/compare/{base}...{head}","merges_url":"https://api.github.com/repos/murphp15/allennlp/merges","archive_url":"https://api.github.com/repos/murphp15/allennlp/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/murphp15/allennlp/downloads","issues_url":"https://api.github.com/repos/murphp15/allennlp/issues{/number}","pulls_url":"https://api.github.com/repos/murphp15/allennlp/pulls{/number}","milestones_url":"https://api.github.com/repos/murphp15/allennlp/milestones{/number}","notifications_url":"https://api.github.com/repos/murphp15/allennlp/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/murphp15/allennlp/labels{/name}","releases_url":"https://api.github.com/repos/murphp15/allennlp/releases{/id}","deployments_url":"https://api.github.com/repos/murphp15/allennlp/deployments","created_at":"2018-05-11T13:48:26Z","updated_at":"2018-05-14T11:52:58Z","pushed_at":"2018-05-17T14:25:54Z","git_url":"git://github.com/murphp15/allennlp.git","ssh_url":"git@github.com:murphp15/allennlp.git","clone_url":"https://github.com/murphp15/allennlp.git","svn_url":"https://github.com/murphp15/allennlp","homepage":"http://www.allennlp.org/","size":11076,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":0,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0"},"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"allenai:master","ref":"master","sha":"76a5a80a163c1661fcd51dd4455cb3a30d1d9855","user":{"login":"allenai","id":5667695,"avatar_url":"https://avatars3.githubusercontent.com/u/5667695?v=4","gravatar_id":"","url":"https://api.github.com/users/allenai","html_url":"https://github.com/allenai","followers_url":"https://api.github.com/users/allenai/followers","following_url":"https://api.github.com/users/allenai/following{/other_user}","gists_url":"https://api.github.com/users/allenai/gists{/gist_id}","starred_url":"https://api.github.com/users/allenai/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/allenai/subscriptions","organizations_url":"https://api.github.com/users/allenai/orgs","repos_url":"https://api.github.com/users/allenai/repos","events_url":"https://api.github.com/users/allenai/events{/privacy}","received_events_url":"https://api.github.com/users/allenai/received_events","type":"Organization","site_admin":false},"repo":{"id":91356408,"name":"allennlp","full_name":"allenai/allennlp","owner":{"login":"allenai","id":5667695,"avatar_url":"https://avatars3.githubusercontent.com/u/5667695?v=4","gravatar_id":"","url":"https://api.github.com/users/allenai","html_url":"https://github.com/allenai","followers_url":"https://api.github.com/users/allenai/followers","following_url":"https://api.github.com/users/allenai/following{/other_user}","gists_url":"https://api.github.com/users/allenai/gists{/gist_id}","starred_url":"https://api.github.com/users/allenai/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/allenai/subscriptions","organizations_url":"https://api.github.com/users/allenai/orgs","repos_url":"https://api.github.com/users/allenai/repos","events_url":"https://api.github.com/users/allenai/events{/privacy}","received_events_url":"https://api.github.com/users/allenai/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/allenai/allennlp","description":"An open-source NLP research library, built on PyTorch.","fork":false,"url":"https://api.github.com/repos/allenai/allennlp","forks_url":"https://api.github.com/repos/allenai/allennlp/forks","keys_url":"https://api.github.com/repos/allenai/allennlp/keys{/key_id}","collaborators_url":"https://api.github.com/repos/allenai/allennlp/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/allenai/allennlp/teams","hooks_url":"https://api.github.com/repos/allenai/allennlp/hooks","issue_events_url":"https://api.github.com/repos/allenai/allennlp/issues/events{/number}","events_url":"https://api.github.com/repos/allenai/allennlp/events","assignees_url":"https://api.github.com/repos/allenai/allennlp/assignees{/user}","branches_url":"https://api.github.com/repos/allenai/allennlp/branches{/branch}","tags_url":"https://api.github.com/repos/allenai/allennlp/tags","blobs_url":"https://api.github.com/repos/allenai/allennlp/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/allenai/allennlp/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/allenai/allennlp/git/refs{/sha}","trees_url":"https://api.github.com/repos/allenai/allennlp/git/trees{/sha}","statuses_url":"https://api.github.com/repos/allenai/allennlp/statuses/{sha}","languages_url":"https://api.github.com/repos/allenai/allennlp/languages","stargazers_url":"https://api.github.com/repos/allenai/allennlp/stargazers","contributors_url":"https://api.github.com/repos/allenai/allennlp/contributors","subscribers_url":"https://api.github.com/repos/allenai/allennlp/subscribers","subscription_url":"https://api.github.com/repos/allenai/allennlp/subscription","commits_url":"https://api.github.com/repos/allenai/allennlp/commits{/sha}","git_commits_url":"https://api.github.com/repos/allenai/allennlp/git/commits{/sha}","comments_url":"https://api.github.com/repos/allenai/allennlp/comments{/number}","issue_comment_url":"https://api.github.com/repos/allenai/allennlp/issues/comments{/number}","contents_url":"https://api.github.com/repos/allenai/allennlp/contents/{+path}","compare_url":"https://api.github.com/repos/allenai/allennlp/compare/{base}...{head}","merges_url":"https://api.github.com/repos/allenai/allennlp/merges","archive_url":"https://api.github.com/repos/allenai/allennlp/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/allenai/allennlp/downloads","issues_url":"https://api.github.com/repos/allenai/allennlp/issues{/number}","pulls_url":"https://api.github.com/repos/allenai/allennlp/pulls{/number}","milestones_url":"https://api.github.com/repos/allenai/allennlp/milestones{/number}","notifications_url":"https://api.github.com/repos/allenai/allennlp/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/allenai/allennlp/labels{/name}","releases_url":"https://api.github.com/repos/allenai/allennlp/releases{/id}","deployments_url":"https://api.github.com/repos/allenai/allennlp/deployments","created_at":"2017-05-15T15:52:41Z","updated_at":"2018-05-17T13:22:04Z","pushed_at":"2018-05-17T15:01:55Z","git_url":"git://github.com/allenai/allennlp.git","ssh_url":"git@github.com:allenai/allennlp.git","clone_url":"https://github.com/allenai/allennlp.git","svn_url":"https://github.com/allenai/allennlp","homepage":"http://www.allennlp.org/","size":11091,"stargazers_count":2076,"watchers_count":2076,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":312,"mirror_url":null,"archived":false,"open_issues_count":91,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0"},"forks":312,"open_issues":91,"watchers":2076,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/allenai/allennlp/pulls/1235"},"html":{"href":"https://github.com/allenai/allennlp/pull/1235"},"issue":{"href":"https://api.github.com/repos/allenai/allennlp/issues/1235"},"comments":{"href":"https://api.github.com/repos/allenai/allennlp/issues/1235/comments"},"review_comments":{"href":"https://api.github.com/repos/allenai/allennlp/pulls/1235/comments"},"review_comment":{"href":"https://api.github.com/repos/allenai/allennlp/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/allenai/allennlp/pulls/1235/commits"},"statuses":{"href":"https://api.github.com/repos/allenai/allennlp/statuses/2081d0f57ba54ec2ef96cc761d1be540ac8538d1"}},"author_association":"CONTRIBUTOR"}}
{"actor":{"display_login":"murphp15"}}