{"action":"created","comment":{"url":"https://api.github.com/repos/facebookresearch/ParlAI/pulls/comments/213723234","pull_request_review_id":150611637,"id":213723234,"node_id":"MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDIxMzcyMzIzNA==","diff_hunk":"@@ -599,49 +551,81 @@ def __init__(self, attn_type, hidden_size, emb_size, bidirectional=False,\n                 # equivalent to dot if attn is identity\n                 self.attn = nn.Linear(hsz, hszXdirs, bias=False)\n \n-    def forward(self, xes, hidden, enc_out, attn_mask=None):\n+    def forward(self, xes, hidden, attn_params):\n+        \"\"\"Compute attention over attn_params given input and hidden states.\n+\n+        :param xes:         input state. will be combined with applied\n+                            attention.\n+        :param hidden:      hidden state from model. will be used to select\n+                            states to attend to in from the attn_params.\n+        :param attn_params: tuple of encoder output states and a mask showing\n+                            which input indices are nonzero.\n+\n+        :returns: output, attn_weights\n+                  output is a new state of same size as input state `xes`.\n+                  attn_weights are the weights given to each state in the\n+                  encoder outputs.\n+        \"\"\"\n         if self.attention == 'none':\n+            # do nothing, no attention\n             return xes\n \n         if type(hidden) == tuple:\n             # for lstms use the \"hidden\" state not the cell state\n             hidden = hidden[0]\n         last_hidden = hidden[-1]  # select hidden state from last RNN layer\n \n+        enc_out, attn_mask = attn_params\n+        bsz, seqlen, hszXnumdir = enc_out.size()\n+        numlayersXnumdir = last_hidden.size(1)\n+\n         if self.attention == 'local':\n-            if enc_out.size(1) > self.max_length:\n-                offset = enc_out.size(1) - self.max_length\n-                enc_out = enc_out.narrow(1, offset, self.max_length)\n+            # local attention weights aren't based on encoder states\n             h_merged = torch.cat((xes.squeeze(1), last_hidden), 1)\n             attn_weights = F.softmax(self.attn(h_merged), dim=1)\n-            if attn_weights.size(1) > enc_out.size(1):\n-                attn_weights = attn_weights.narrow(1, 0, enc_out.size(1))\n+\n+            # adjust state sizes to the fixed window size\n+            if seqlen > self.max_length:\n+                offset = seqlen - self.max_length\n+                enc_out = enc_out.narrow(1, offset, self.max_length)\n+                seqlen = self.max_length\n+            if attn_weights.size(1) > seqlen:\n+                attn_weights = attn_weights.narrow(1, 0, seqlen)\n         else:\n             hid = last_hidden.unsqueeze(1)\n             if self.attention == 'concat':\n-                hid = hid.expand(last_hidden.size(0),\n-                                 enc_out.size(1),\n-                                 last_hidden.size(1))\n+                # concat hidden state and encoder outputs\n+                hid = hid.expand(bsz, seqlen, numlayersXnumdir)\n                 h_merged = torch.cat((enc_out, hid), 2)\n+                # then do linear combination of them with activation\n                 active = F.tanh(self.attn(h_merged))\n                 attn_w_premask = self.attn_v(active).squeeze(2)\n             elif self.attention == 'dot':\n-                if hid.size(2) != enc_out.size(2):\n+                # dot product between hidden and encoder outputs\n+                if numlayersXnumdir != hszXnumdir:\n                     # enc_out has two directions, so double hid\n                     hid = torch.cat([hid, hid], 2)\n-                attn_w_premask = (\n-                    torch.bmm(hid, enc_out.transpose(1, 2)).squeeze(1))\n+                enc_t = enc_out.transpose(1, 2)\n+                attn_w_premask = torch.bmm(hid, enc_t).squeeze(1)\n             elif self.attention == 'general':\n+                # before doing dot product, transform hidden state with linear\n+                # same as dot if linear is identity\n                 hid = self.attn(hid)\n-                attn_w_premask = (\n-                    torch.bmm(hid, enc_out.transpose(1, 2)).squeeze(1))\n-            # calculate activation scores\n+                enc_t = enc_out.transpose(1, 2)\n+                attn_w_premask = torch.bmm(hid, enc_t).squeeze(1)\n+\n+            # calculate activation scores, apply mask if needed\n             if attn_mask is not None:\n                 # remove activation from NULL symbols\n+                # TODO: is this the best operation?\n                 attn_w_premask -= (1 - attn_mask) * 1e20","path":"parlai/agents/seq2seq/modules.py","position":1056,"original_position":1056,"commit_id":"2e1efddfb5091980a992917618ce09b29f1792b7","original_commit_id":"2e1efddfb5091980a992917618ce09b29f1792b7","user":{"login":"stephenroller","id":31896,"node_id":"MDQ6VXNlcjMxODk2","avatar_url":"https://avatars2.githubusercontent.com/u/31896?v=4","gravatar_id":"","url":"https://api.github.com/users/stephenroller","html_url":"https://github.com/stephenroller","followers_url":"https://api.github.com/users/stephenroller/followers","following_url":"https://api.github.com/users/stephenroller/following{/other_user}","gists_url":"https://api.github.com/users/stephenroller/gists{/gist_id}","starred_url":"https://api.github.com/users/stephenroller/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/stephenroller/subscriptions","organizations_url":"https://api.github.com/users/stephenroller/orgs","repos_url":"https://api.github.com/users/stephenroller/repos","events_url":"https://api.github.com/users/stephenroller/events{/privacy}","received_events_url":"https://api.github.com/users/stephenroller/received_events","type":"User","site_admin":false},"body":"Could do `th.masked_fill_(attn_w_premask, -1e20)`. That's what I've seen in fairseq.\r\n\r\nMaybe make that `1e20` as a private constant, like `_ALMOST_INFINITY`? Don't think it matters.","created_at":"2018-08-29T15:22:24Z","updated_at":"2018-08-29T15:32:14Z","html_url":"https://github.com/facebookresearch/ParlAI/pull/1119#discussion_r213723234","pull_request_url":"https://api.github.com/repos/facebookresearch/ParlAI/pulls/1119","author_association":"CONTRIBUTOR","_links":{"self":{"href":"https://api.github.com/repos/facebookresearch/ParlAI/pulls/comments/213723234"},"html":{"href":"https://github.com/facebookresearch/ParlAI/pull/1119#discussion_r213723234"},"pull_request":{"href":"https://api.github.com/repos/facebookresearch/ParlAI/pulls/1119"}}},"pull_request":{"url":"https://api.github.com/repos/facebookresearch/ParlAI/pulls/1119","id":211588034,"node_id":"MDExOlB1bGxSZXF1ZXN0MjExNTg4MDM0","html_url":"https://github.com/facebookresearch/ParlAI/pull/1119","diff_url":"https://github.com/facebookresearch/ParlAI/pull/1119.diff","patch_url":"https://github.com/facebookresearch/ParlAI/pull/1119.patch","issue_url":"https://api.github.com/repos/facebookresearch/ParlAI/issues/1119","number":1119,"state":"open","locked":false,"title":"new version of seq2seq","user":{"login":"alexholdenmiller","id":1141354,"node_id":"MDQ6VXNlcjExNDEzNTQ=","avatar_url":"https://avatars1.githubusercontent.com/u/1141354?v=4","gravatar_id":"","url":"https://api.github.com/users/alexholdenmiller","html_url":"https://github.com/alexholdenmiller","followers_url":"https://api.github.com/users/alexholdenmiller/followers","following_url":"https://api.github.com/users/alexholdenmiller/following{/other_user}","gists_url":"https://api.github.com/users/alexholdenmiller/gists{/gist_id}","starred_url":"https://api.github.com/users/alexholdenmiller/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/alexholdenmiller/subscriptions","organizations_url":"https://api.github.com/users/alexholdenmiller/orgs","repos_url":"https://api.github.com/users/alexholdenmiller/repos","events_url":"https://api.github.com/users/alexholdenmiller/events{/privacy}","received_events_url":"https://api.github.com/users/alexholdenmiller/received_events","type":"User","site_admin":false},"body":"- primary change is massive updates to further modularize and clean out modules.py\r\n\r\n- successfully trains & evals & ranks on babi:task10k:1\r\n- successfully trains & evals & ranks on convai2","created_at":"2018-08-28T23:34:23Z","updated_at":"2018-08-29T15:32:13Z","closed_at":null,"merged_at":null,"merge_commit_sha":"ec2313e819d993347e5d2e8cd8aa0c5fce9b04c6","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[{"id":590886701,"node_id":"MDU6TGFiZWw1OTA4ODY3MDE=","url":"https://api.github.com/repos/facebookresearch/ParlAI/labels/CLA%20Signed","name":"CLA Signed","color":"009900","default":false}],"milestone":null,"commits_url":"https://api.github.com/repos/facebookresearch/ParlAI/pulls/1119/commits","review_comments_url":"https://api.github.com/repos/facebookresearch/ParlAI/pulls/1119/comments","review_comment_url":"https://api.github.com/repos/facebookresearch/ParlAI/pulls/comments{/number}","comments_url":"https://api.github.com/repos/facebookresearch/ParlAI/issues/1119/comments","statuses_url":"https://api.github.com/repos/facebookresearch/ParlAI/statuses/2e1efddfb5091980a992917618ce09b29f1792b7","head":{"label":"facebookresearch:break_s2s","ref":"break_s2s","sha":"2e1efddfb5091980a992917618ce09b29f1792b7","user":{"login":"facebookresearch","id":16943930,"node_id":"MDEyOk9yZ2FuaXphdGlvbjE2OTQzOTMw","avatar_url":"https://avatars0.githubusercontent.com/u/16943930?v=4","gravatar_id":"","url":"https://api.github.com/users/facebookresearch","html_url":"https://github.com/facebookresearch","followers_url":"https://api.github.com/users/facebookresearch/followers","following_url":"https://api.github.com/users/facebookresearch/following{/other_user}","gists_url":"https://api.github.com/users/facebookresearch/gists{/gist_id}","starred_url":"https://api.github.com/users/facebookresearch/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/facebookresearch/subscriptions","organizations_url":"https://api.github.com/users/facebookresearch/orgs","repos_url":"https://api.github.com/users/facebookresearch/repos","events_url":"https://api.github.com/users/facebookresearch/events{/privacy}","received_events_url":"https://api.github.com/users/facebookresearch/received_events","type":"Organization","site_admin":false},"repo":{"id":89266735,"node_id":"MDEwOlJlcG9zaXRvcnk4OTI2NjczNQ==","name":"ParlAI","full_name":"facebookresearch/ParlAI","owner":{"login":"facebookresearch","id":16943930,"node_id":"MDEyOk9yZ2FuaXphdGlvbjE2OTQzOTMw","avatar_url":"https://avatars0.githubusercontent.com/u/16943930?v=4","gravatar_id":"","url":"https://api.github.com/users/facebookresearch","html_url":"https://github.com/facebookresearch","followers_url":"https://api.github.com/users/facebookresearch/followers","following_url":"https://api.github.com/users/facebookresearch/following{/other_user}","gists_url":"https://api.github.com/users/facebookresearch/gists{/gist_id}","starred_url":"https://api.github.com/users/facebookresearch/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/facebookresearch/subscriptions","organizations_url":"https://api.github.com/users/facebookresearch/orgs","repos_url":"https://api.github.com/users/facebookresearch/repos","events_url":"https://api.github.com/users/facebookresearch/events{/privacy}","received_events_url":"https://api.github.com/users/facebookresearch/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/facebookresearch/ParlAI","description":"A framework for training and evaluating AI models on a variety of openly available dialog datasets.","fork":false,"url":"https://api.github.com/repos/facebookresearch/ParlAI","forks_url":"https://api.github.com/repos/facebookresearch/ParlAI/forks","keys_url":"https://api.github.com/repos/facebookresearch/ParlAI/keys{/key_id}","collaborators_url":"https://api.github.com/repos/facebookresearch/ParlAI/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/facebookresearch/ParlAI/teams","hooks_url":"https://api.github.com/repos/facebookresearch/ParlAI/hooks","issue_events_url":"https://api.github.com/repos/facebookresearch/ParlAI/issues/events{/number}","events_url":"https://api.github.com/repos/facebookresearch/ParlAI/events","assignees_url":"https://api.github.com/repos/facebookresearch/ParlAI/assignees{/user}","branches_url":"https://api.github.com/repos/facebookresearch/ParlAI/branches{/branch}","tags_url":"https://api.github.com/repos/facebookresearch/ParlAI/tags","blobs_url":"https://api.github.com/repos/facebookresearch/ParlAI/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/facebookresearch/ParlAI/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/facebookresearch/ParlAI/git/refs{/sha}","trees_url":"https://api.github.com/repos/facebookresearch/ParlAI/git/trees{/sha}","statuses_url":"https://api.github.com/repos/facebookresearch/ParlAI/statuses/{sha}","languages_url":"https://api.github.com/repos/facebookresearch/ParlAI/languages","stargazers_url":"https://api.github.com/repos/facebookresearch/ParlAI/stargazers","contributors_url":"https://api.github.com/repos/facebookresearch/ParlAI/contributors","subscribers_url":"https://api.github.com/repos/facebookresearch/ParlAI/subscribers","subscription_url":"https://api.github.com/repos/facebookresearch/ParlAI/subscription","commits_url":"https://api.github.com/repos/facebookresearch/ParlAI/commits{/sha}","git_commits_url":"https://api.github.com/repos/facebookresearch/ParlAI/git/commits{/sha}","comments_url":"https://api.github.com/repos/facebookresearch/ParlAI/comments{/number}","issue_comment_url":"https://api.github.com/repos/facebookresearch/ParlAI/issues/comments{/number}","contents_url":"https://api.github.com/repos/facebookresearch/ParlAI/contents/{+path}","compare_url":"https://api.github.com/repos/facebookresearch/ParlAI/compare/{base}...{head}","merges_url":"https://api.github.com/repos/facebookresearch/ParlAI/merges","archive_url":"https://api.github.com/repos/facebookresearch/ParlAI/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/facebookresearch/ParlAI/downloads","issues_url":"https://api.github.com/repos/facebookresearch/ParlAI/issues{/number}","pulls_url":"https://api.github.com/repos/facebookresearch/ParlAI/pulls{/number}","milestones_url":"https://api.github.com/repos/facebookresearch/ParlAI/milestones{/number}","notifications_url":"https://api.github.com/repos/facebookresearch/ParlAI/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/facebookresearch/ParlAI/labels{/name}","releases_url":"https://api.github.com/repos/facebookresearch/ParlAI/releases{/id}","deployments_url":"https://api.github.com/repos/facebookresearch/ParlAI/deployments","created_at":"2017-04-24T17:10:44Z","updated_at":"2018-08-29T15:15:58Z","pushed_at":"2018-08-29T15:05:47Z","git_url":"git://github.com/facebookresearch/ParlAI.git","ssh_url":"git@github.com:facebookresearch/ParlAI.git","clone_url":"https://github.com/facebookresearch/ParlAI.git","svn_url":"https://github.com/facebookresearch/ParlAI","homepage":"parl.ai","size":12173,"stargazers_count":3608,"watchers_count":3608,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":639,"mirror_url":null,"archived":false,"open_issues_count":25,"license":{"key":"other","name":"Other","spdx_id":null,"url":null,"node_id":"MDc6TGljZW5zZTA="},"forks":639,"open_issues":25,"watchers":3608,"default_branch":"master"}},"base":{"label":"facebookresearch:s2s_migrate","ref":"s2s_migrate","sha":"3bb94410e11f0025e6523639005f7921b6214805","user":{"login":"facebookresearch","id":16943930,"node_id":"MDEyOk9yZ2FuaXphdGlvbjE2OTQzOTMw","avatar_url":"https://avatars0.githubusercontent.com/u/16943930?v=4","gravatar_id":"","url":"https://api.github.com/users/facebookresearch","html_url":"https://github.com/facebookresearch","followers_url":"https://api.github.com/users/facebookresearch/followers","following_url":"https://api.github.com/users/facebookresearch/following{/other_user}","gists_url":"https://api.github.com/users/facebookresearch/gists{/gist_id}","starred_url":"https://api.github.com/users/facebookresearch/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/facebookresearch/subscriptions","organizations_url":"https://api.github.com/users/facebookresearch/orgs","repos_url":"https://api.github.com/users/facebookresearch/repos","events_url":"https://api.github.com/users/facebookresearch/events{/privacy}","received_events_url":"https://api.github.com/users/facebookresearch/received_events","type":"Organization","site_admin":false},"repo":{"id":89266735,"node_id":"MDEwOlJlcG9zaXRvcnk4OTI2NjczNQ==","name":"ParlAI","full_name":"facebookresearch/ParlAI","owner":{"login":"facebookresearch","id":16943930,"node_id":"MDEyOk9yZ2FuaXphdGlvbjE2OTQzOTMw","avatar_url":"https://avatars0.githubusercontent.com/u/16943930?v=4","gravatar_id":"","url":"https://api.github.com/users/facebookresearch","html_url":"https://github.com/facebookresearch","followers_url":"https://api.github.com/users/facebookresearch/followers","following_url":"https://api.github.com/users/facebookresearch/following{/other_user}","gists_url":"https://api.github.com/users/facebookresearch/gists{/gist_id}","starred_url":"https://api.github.com/users/facebookresearch/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/facebookresearch/subscriptions","organizations_url":"https://api.github.com/users/facebookresearch/orgs","repos_url":"https://api.github.com/users/facebookresearch/repos","events_url":"https://api.github.com/users/facebookresearch/events{/privacy}","received_events_url":"https://api.github.com/users/facebookresearch/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/facebookresearch/ParlAI","description":"A framework for training and evaluating AI models on a variety of openly available dialog datasets.","fork":false,"url":"https://api.github.com/repos/facebookresearch/ParlAI","forks_url":"https://api.github.com/repos/facebookresearch/ParlAI/forks","keys_url":"https://api.github.com/repos/facebookresearch/ParlAI/keys{/key_id}","collaborators_url":"https://api.github.com/repos/facebookresearch/ParlAI/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/facebookresearch/ParlAI/teams","hooks_url":"https://api.github.com/repos/facebookresearch/ParlAI/hooks","issue_events_url":"https://api.github.com/repos/facebookresearch/ParlAI/issues/events{/number}","events_url":"https://api.github.com/repos/facebookresearch/ParlAI/events","assignees_url":"https://api.github.com/repos/facebookresearch/ParlAI/assignees{/user}","branches_url":"https://api.github.com/repos/facebookresearch/ParlAI/branches{/branch}","tags_url":"https://api.github.com/repos/facebookresearch/ParlAI/tags","blobs_url":"https://api.github.com/repos/facebookresearch/ParlAI/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/facebookresearch/ParlAI/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/facebookresearch/ParlAI/git/refs{/sha}","trees_url":"https://api.github.com/repos/facebookresearch/ParlAI/git/trees{/sha}","statuses_url":"https://api.github.com/repos/facebookresearch/ParlAI/statuses/{sha}","languages_url":"https://api.github.com/repos/facebookresearch/ParlAI/languages","stargazers_url":"https://api.github.com/repos/facebookresearch/ParlAI/stargazers","contributors_url":"https://api.github.com/repos/facebookresearch/ParlAI/contributors","subscribers_url":"https://api.github.com/repos/facebookresearch/ParlAI/subscribers","subscription_url":"https://api.github.com/repos/facebookresearch/ParlAI/subscription","commits_url":"https://api.github.com/repos/facebookresearch/ParlAI/commits{/sha}","git_commits_url":"https://api.github.com/repos/facebookresearch/ParlAI/git/commits{/sha}","comments_url":"https://api.github.com/repos/facebookresearch/ParlAI/comments{/number}","issue_comment_url":"https://api.github.com/repos/facebookresearch/ParlAI/issues/comments{/number}","contents_url":"https://api.github.com/repos/facebookresearch/ParlAI/contents/{+path}","compare_url":"https://api.github.com/repos/facebookresearch/ParlAI/compare/{base}...{head}","merges_url":"https://api.github.com/repos/facebookresearch/ParlAI/merges","archive_url":"https://api.github.com/repos/facebookresearch/ParlAI/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/facebookresearch/ParlAI/downloads","issues_url":"https://api.github.com/repos/facebookresearch/ParlAI/issues{/number}","pulls_url":"https://api.github.com/repos/facebookresearch/ParlAI/pulls{/number}","milestones_url":"https://api.github.com/repos/facebookresearch/ParlAI/milestones{/number}","notifications_url":"https://api.github.com/repos/facebookresearch/ParlAI/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/facebookresearch/ParlAI/labels{/name}","releases_url":"https://api.github.com/repos/facebookresearch/ParlAI/releases{/id}","deployments_url":"https://api.github.com/repos/facebookresearch/ParlAI/deployments","created_at":"2017-04-24T17:10:44Z","updated_at":"2018-08-29T15:15:58Z","pushed_at":"2018-08-29T15:05:47Z","git_url":"git://github.com/facebookresearch/ParlAI.git","ssh_url":"git@github.com:facebookresearch/ParlAI.git","clone_url":"https://github.com/facebookresearch/ParlAI.git","svn_url":"https://github.com/facebookresearch/ParlAI","homepage":"parl.ai","size":12173,"stargazers_count":3608,"watchers_count":3608,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":639,"mirror_url":null,"archived":false,"open_issues_count":25,"license":{"key":"other","name":"Other","spdx_id":null,"url":null,"node_id":"MDc6TGljZW5zZTA="},"forks":639,"open_issues":25,"watchers":3608,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/facebookresearch/ParlAI/pulls/1119"},"html":{"href":"https://github.com/facebookresearch/ParlAI/pull/1119"},"issue":{"href":"https://api.github.com/repos/facebookresearch/ParlAI/issues/1119"},"comments":{"href":"https://api.github.com/repos/facebookresearch/ParlAI/issues/1119/comments"},"review_comments":{"href":"https://api.github.com/repos/facebookresearch/ParlAI/pulls/1119/comments"},"review_comment":{"href":"https://api.github.com/repos/facebookresearch/ParlAI/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/facebookresearch/ParlAI/pulls/1119/commits"},"statuses":{"href":"https://api.github.com/repos/facebookresearch/ParlAI/statuses/2e1efddfb5091980a992917618ce09b29f1792b7"}},"author_association":"MEMBER"}}
{"actor":{"display_login":"stephenroller"}}