{"action":"created","comment":{"url":"https://api.github.com/repos/allenai/allennlp/pulls/comments/235507578","pull_request_review_id":177377164,"id":235507578,"node_id":"MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDIzNTUwNzU3OA==","diff_hunk":"@@ -0,0 +1,111 @@\n+\"\"\"\n+A ``TokenEmbedder`` which uses one of the BERT models\n+(https://github.com/google-research/bert)\n+to produce embeddings.\n+\n+At its core it uses Hugging Face's PyTorch implementation\n+(https://github.com/huggingface/pytorch-pretrained-BERT),\n+so thanks to them!\n+\"\"\"\n+import logging\n+\n+import torch\n+\n+from pytorch_pretrained_bert.modeling import BertModel\n+\n+from allennlp.modules.token_embedders.token_embedder import TokenEmbedder\n+from allennlp.nn import util\n+\n+logger = logging.getLogger(__name__)\n+\n+\n+class BertEmbedder(TokenEmbedder):\n+    \"\"\"\n+    A ``TokenEmbedder`` that produces BERT embeddings for your tokens.\n+    Should be paired with a ``BertIndexer``, which produces wordpiece ids.\n+\n+    Most likely you probably want to use ``PretrainedBertEmbedder``\n+    for one of the named pretrained models, not this base class.\n+\n+    Parameters\n+    ----------\n+    bert_model: ``BertModel``\n+        The BERT model being wrapped.\n+    \"\"\"\n+    def __init__(self,\n+                 bert_model: BertModel) -> None:\n+        super().__init__()\n+        self.bert_model = bert_model\n+        self.output_dim = bert_model.config.hidden_size\n+\n+    def get_output_dim(self) -> int:\n+        return self.output_dim\n+\n+    def forward(self,\n+                input_ids: torch.LongTensor,\n+                offsets: torch.LongTensor = None,\n+                token_type_ids: torch.LongTensor = None) -> torch.Tensor:\n+        \"\"\"\n+        Parameters\n+        ----------\n+        input_ids: ``torch.LongTensor``\n+            The wordpiece ids for each input sentence.\n+        offsets: ``torch.LongTensor``, optional (default = None)\n+            The BERT embeddings are one per wordpiece. However it's possible/likely\n+            you might want one per original token. In that case, ``offsets``\n+            should represent the indices of the last wordpiece for each original token.\n+\n+            That is, if you had the sentence \"Definitely not\", and if the corresponding\n+            wordpieces were [\"Def\", \"##in\", \"##ite\", \"##ly\", \"not\"], then the input_ids\n+            would be 5 wordpiece ids, and the offsets would be [3, 4]. If offsets are\n+            provided, the returned tensor will contain the _last_ wordpiece embedding\n+            for each token, and (in particular) will contain one embedding per token.\n+            If offsets are not provided, the entire tensor of wordpiece embeddings\n+            will be returned.\n+        token_type_ids: ``torch.LongTensor``, optional (default = None)\n+            If an input consists of two sentences (as in the BERT paper),\n+            tokens from the first sentence should have type 0 and tokens from\n+            the second sentence should have type 1.  If you don't provide this\n+            (the default BertIndexer doesn't) then it's assumed to be all 0s.","path":"allennlp/modules/token_embedders/bert_token_embedder.py","position":69,"original_position":69,"commit_id":"3a4c0bcf77483e5a3d635f85c80579037bdf09e9","original_commit_id":"3a4c0bcf77483e5a3d635f85c80579037bdf09e9","user":{"login":"matt-gardner","id":3291951,"node_id":"MDQ6VXNlcjMyOTE5NTE=","avatar_url":"https://avatars2.githubusercontent.com/u/3291951?v=4","gravatar_id":"","url":"https://api.github.com/users/matt-gardner","html_url":"https://github.com/matt-gardner","followers_url":"https://api.github.com/users/matt-gardner/followers","following_url":"https://api.github.com/users/matt-gardner/following{/other_user}","gists_url":"https://api.github.com/users/matt-gardner/gists{/gist_id}","starred_url":"https://api.github.com/users/matt-gardner/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/matt-gardner/subscriptions","organizations_url":"https://api.github.com/users/matt-gardner/orgs","repos_url":"https://api.github.com/users/matt-gardner/repos","events_url":"https://api.github.com/users/matt-gardner/events{/privacy}","received_events_url":"https://api.github.com/users/matt-gardner/received_events","type":"User","site_admin":false},"body":"Is there a way to modify the indexer to provide this?  It's fine to do it in another PR, but I'd at least open an issue to track adding that.  Seems pretty important if you want to use this for SQuAD.","created_at":"2018-11-21T19:03:10Z","updated_at":"2018-11-21T19:09:58Z","html_url":"https://github.com/allenai/allennlp/pull/2067#discussion_r235507578","pull_request_url":"https://api.github.com/repos/allenai/allennlp/pulls/2067","author_association":"MEMBER","_links":{"self":{"href":"https://api.github.com/repos/allenai/allennlp/pulls/comments/235507578"},"html":{"href":"https://github.com/allenai/allennlp/pull/2067#discussion_r235507578"},"pull_request":{"href":"https://api.github.com/repos/allenai/allennlp/pulls/2067"}}},"pull_request":{"url":"https://api.github.com/repos/allenai/allennlp/pulls/2067","id":231687252,"node_id":"MDExOlB1bGxSZXF1ZXN0MjMxNjg3MjUy","html_url":"https://github.com/allenai/allennlp/pull/2067","diff_url":"https://github.com/allenai/allennlp/pull/2067.diff","patch_url":"https://github.com/allenai/allennlp/pull/2067.patch","issue_url":"https://api.github.com/repos/allenai/allennlp/issues/2067","number":2067,"state":"open","locked":false,"title":"add BERT token embedder ","user":{"login":"joelgrus","id":1308313,"node_id":"MDQ6VXNlcjEzMDgzMTM=","avatar_url":"https://avatars1.githubusercontent.com/u/1308313?v=4","gravatar_id":"","url":"https://api.github.com/users/joelgrus","html_url":"https://github.com/joelgrus","followers_url":"https://api.github.com/users/joelgrus/followers","following_url":"https://api.github.com/users/joelgrus/following{/other_user}","gists_url":"https://api.github.com/users/joelgrus/gists{/gist_id}","starred_url":"https://api.github.com/users/joelgrus/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/joelgrus/subscriptions","organizations_url":"https://api.github.com/users/joelgrus/orgs","repos_url":"https://api.github.com/users/joelgrus/repos","events_url":"https://api.github.com/users/joelgrus/events{/privacy}","received_events_url":"https://api.github.com/users/joelgrus/received_events","type":"User","site_admin":false},"body":"this is ready for review. in addition to the included unit tests, I trained two NER models using these embeddings (unfortunately, I realized this morning, I used the _uncased_ BERT model, which seems like a bad idea for NER)\r\n\r\n(1) only BERT embeddings: https://beaker-internal.allenai.org/ex/ex_rnk3mcplnpjz/tasks\r\n(2) BERT embeddings + character embeddings: https://beaker-internal.allenai.org/ex/ex_nrq8d5vw5cb2/tasks\r\n\r\n(apologies to non-AI2 people for the beaker-internal links)\r\n\r\nas discussed offline, because of the positional encodings the BERT embedding has a max sequence length and will crash if you feed it longer sequences. this implementation simply truncates longer sequences and logs a warning. I left a TODO to come up with something better.","created_at":"2018-11-16T23:58:05Z","updated_at":"2018-11-21T19:09:58Z","closed_at":null,"merged_at":null,"merge_commit_sha":"0b74ad7222330d40232ac7cfce2839d8adcbacce","assignee":null,"assignees":[],"requested_reviewers":[{"login":"DeNeutoy","id":16001974,"node_id":"MDQ6VXNlcjE2MDAxOTc0","avatar_url":"https://avatars1.githubusercontent.com/u/16001974?v=4","gravatar_id":"","url":"https://api.github.com/users/DeNeutoy","html_url":"https://github.com/DeNeutoy","followers_url":"https://api.github.com/users/DeNeutoy/followers","following_url":"https://api.github.com/users/DeNeutoy/following{/other_user}","gists_url":"https://api.github.com/users/DeNeutoy/gists{/gist_id}","starred_url":"https://api.github.com/users/DeNeutoy/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/DeNeutoy/subscriptions","organizations_url":"https://api.github.com/users/DeNeutoy/orgs","repos_url":"https://api.github.com/users/DeNeutoy/repos","events_url":"https://api.github.com/users/DeNeutoy/events{/privacy}","received_events_url":"https://api.github.com/users/DeNeutoy/received_events","type":"User","site_admin":false}],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/allenai/allennlp/pulls/2067/commits","review_comments_url":"https://api.github.com/repos/allenai/allennlp/pulls/2067/comments","review_comment_url":"https://api.github.com/repos/allenai/allennlp/pulls/comments{/number}","comments_url":"https://api.github.com/repos/allenai/allennlp/issues/2067/comments","statuses_url":"https://api.github.com/repos/allenai/allennlp/statuses/3a4c0bcf77483e5a3d635f85c80579037bdf09e9","head":{"label":"joelgrus:bert","ref":"bert","sha":"3a4c0bcf77483e5a3d635f85c80579037bdf09e9","user":{"login":"joelgrus","id":1308313,"node_id":"MDQ6VXNlcjEzMDgzMTM=","avatar_url":"https://avatars1.githubusercontent.com/u/1308313?v=4","gravatar_id":"","url":"https://api.github.com/users/joelgrus","html_url":"https://github.com/joelgrus","followers_url":"https://api.github.com/users/joelgrus/followers","following_url":"https://api.github.com/users/joelgrus/following{/other_user}","gists_url":"https://api.github.com/users/joelgrus/gists{/gist_id}","starred_url":"https://api.github.com/users/joelgrus/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/joelgrus/subscriptions","organizations_url":"https://api.github.com/users/joelgrus/orgs","repos_url":"https://api.github.com/users/joelgrus/repos","events_url":"https://api.github.com/users/joelgrus/events{/privacy}","received_events_url":"https://api.github.com/users/joelgrus/received_events","type":"User","site_admin":false},"repo":{"id":114700287,"node_id":"MDEwOlJlcG9zaXRvcnkxMTQ3MDAyODc=","name":"allennlp","full_name":"joelgrus/allennlp","private":false,"owner":{"login":"joelgrus","id":1308313,"node_id":"MDQ6VXNlcjEzMDgzMTM=","avatar_url":"https://avatars1.githubusercontent.com/u/1308313?v=4","gravatar_id":"","url":"https://api.github.com/users/joelgrus","html_url":"https://github.com/joelgrus","followers_url":"https://api.github.com/users/joelgrus/followers","following_url":"https://api.github.com/users/joelgrus/following{/other_user}","gists_url":"https://api.github.com/users/joelgrus/gists{/gist_id}","starred_url":"https://api.github.com/users/joelgrus/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/joelgrus/subscriptions","organizations_url":"https://api.github.com/users/joelgrus/orgs","repos_url":"https://api.github.com/users/joelgrus/repos","events_url":"https://api.github.com/users/joelgrus/events{/privacy}","received_events_url":"https://api.github.com/users/joelgrus/received_events","type":"User","site_admin":false},"html_url":"https://github.com/joelgrus/allennlp","description":"An open-source NLP research library, built on PyTorch.","fork":true,"url":"https://api.github.com/repos/joelgrus/allennlp","forks_url":"https://api.github.com/repos/joelgrus/allennlp/forks","keys_url":"https://api.github.com/repos/joelgrus/allennlp/keys{/key_id}","collaborators_url":"https://api.github.com/repos/joelgrus/allennlp/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/joelgrus/allennlp/teams","hooks_url":"https://api.github.com/repos/joelgrus/allennlp/hooks","issue_events_url":"https://api.github.com/repos/joelgrus/allennlp/issues/events{/number}","events_url":"https://api.github.com/repos/joelgrus/allennlp/events","assignees_url":"https://api.github.com/repos/joelgrus/allennlp/assignees{/user}","branches_url":"https://api.github.com/repos/joelgrus/allennlp/branches{/branch}","tags_url":"https://api.github.com/repos/joelgrus/allennlp/tags","blobs_url":"https://api.github.com/repos/joelgrus/allennlp/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/joelgrus/allennlp/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/joelgrus/allennlp/git/refs{/sha}","trees_url":"https://api.github.com/repos/joelgrus/allennlp/git/trees{/sha}","statuses_url":"https://api.github.com/repos/joelgrus/allennlp/statuses/{sha}","languages_url":"https://api.github.com/repos/joelgrus/allennlp/languages","stargazers_url":"https://api.github.com/repos/joelgrus/allennlp/stargazers","contributors_url":"https://api.github.com/repos/joelgrus/allennlp/contributors","subscribers_url":"https://api.github.com/repos/joelgrus/allennlp/subscribers","subscription_url":"https://api.github.com/repos/joelgrus/allennlp/subscription","commits_url":"https://api.github.com/repos/joelgrus/allennlp/commits{/sha}","git_commits_url":"https://api.github.com/repos/joelgrus/allennlp/git/commits{/sha}","comments_url":"https://api.github.com/repos/joelgrus/allennlp/comments{/number}","issue_comment_url":"https://api.github.com/repos/joelgrus/allennlp/issues/comments{/number}","contents_url":"https://api.github.com/repos/joelgrus/allennlp/contents/{+path}","compare_url":"https://api.github.com/repos/joelgrus/allennlp/compare/{base}...{head}","merges_url":"https://api.github.com/repos/joelgrus/allennlp/merges","archive_url":"https://api.github.com/repos/joelgrus/allennlp/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/joelgrus/allennlp/downloads","issues_url":"https://api.github.com/repos/joelgrus/allennlp/issues{/number}","pulls_url":"https://api.github.com/repos/joelgrus/allennlp/pulls{/number}","milestones_url":"https://api.github.com/repos/joelgrus/allennlp/milestones{/number}","notifications_url":"https://api.github.com/repos/joelgrus/allennlp/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/joelgrus/allennlp/labels{/name}","releases_url":"https://api.github.com/repos/joelgrus/allennlp/releases{/id}","deployments_url":"https://api.github.com/repos/joelgrus/allennlp/deployments","created_at":"2017-12-19T00:12:40Z","updated_at":"2018-11-15T21:21:30Z","pushed_at":"2018-11-21T15:12:49Z","git_url":"git://github.com/joelgrus/allennlp.git","ssh_url":"git@github.com:joelgrus/allennlp.git","clone_url":"https://github.com/joelgrus/allennlp.git","svn_url":"https://github.com/joelgrus/allennlp","homepage":"http://www.allennlp.org/","size":55449,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":0,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0","node_id":"MDc6TGljZW5zZTI="},"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"allenai:master","ref":"master","sha":"6b16222b72596ebe7b809dea26cc253195e02590","user":{"login":"allenai","id":5667695,"node_id":"MDEyOk9yZ2FuaXphdGlvbjU2Njc2OTU=","avatar_url":"https://avatars3.githubusercontent.com/u/5667695?v=4","gravatar_id":"","url":"https://api.github.com/users/allenai","html_url":"https://github.com/allenai","followers_url":"https://api.github.com/users/allenai/followers","following_url":"https://api.github.com/users/allenai/following{/other_user}","gists_url":"https://api.github.com/users/allenai/gists{/gist_id}","starred_url":"https://api.github.com/users/allenai/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/allenai/subscriptions","organizations_url":"https://api.github.com/users/allenai/orgs","repos_url":"https://api.github.com/users/allenai/repos","events_url":"https://api.github.com/users/allenai/events{/privacy}","received_events_url":"https://api.github.com/users/allenai/received_events","type":"Organization","site_admin":false},"repo":{"id":91356408,"node_id":"MDEwOlJlcG9zaXRvcnk5MTM1NjQwOA==","name":"allennlp","full_name":"allenai/allennlp","private":false,"owner":{"login":"allenai","id":5667695,"node_id":"MDEyOk9yZ2FuaXphdGlvbjU2Njc2OTU=","avatar_url":"https://avatars3.githubusercontent.com/u/5667695?v=4","gravatar_id":"","url":"https://api.github.com/users/allenai","html_url":"https://github.com/allenai","followers_url":"https://api.github.com/users/allenai/followers","following_url":"https://api.github.com/users/allenai/following{/other_user}","gists_url":"https://api.github.com/users/allenai/gists{/gist_id}","starred_url":"https://api.github.com/users/allenai/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/allenai/subscriptions","organizations_url":"https://api.github.com/users/allenai/orgs","repos_url":"https://api.github.com/users/allenai/repos","events_url":"https://api.github.com/users/allenai/events{/privacy}","received_events_url":"https://api.github.com/users/allenai/received_events","type":"Organization","site_admin":false},"html_url":"https://github.com/allenai/allennlp","description":"An open-source NLP research library, built on PyTorch.","fork":false,"url":"https://api.github.com/repos/allenai/allennlp","forks_url":"https://api.github.com/repos/allenai/allennlp/forks","keys_url":"https://api.github.com/repos/allenai/allennlp/keys{/key_id}","collaborators_url":"https://api.github.com/repos/allenai/allennlp/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/allenai/allennlp/teams","hooks_url":"https://api.github.com/repos/allenai/allennlp/hooks","issue_events_url":"https://api.github.com/repos/allenai/allennlp/issues/events{/number}","events_url":"https://api.github.com/repos/allenai/allennlp/events","assignees_url":"https://api.github.com/repos/allenai/allennlp/assignees{/user}","branches_url":"https://api.github.com/repos/allenai/allennlp/branches{/branch}","tags_url":"https://api.github.com/repos/allenai/allennlp/tags","blobs_url":"https://api.github.com/repos/allenai/allennlp/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/allenai/allennlp/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/allenai/allennlp/git/refs{/sha}","trees_url":"https://api.github.com/repos/allenai/allennlp/git/trees{/sha}","statuses_url":"https://api.github.com/repos/allenai/allennlp/statuses/{sha}","languages_url":"https://api.github.com/repos/allenai/allennlp/languages","stargazers_url":"https://api.github.com/repos/allenai/allennlp/stargazers","contributors_url":"https://api.github.com/repos/allenai/allennlp/contributors","subscribers_url":"https://api.github.com/repos/allenai/allennlp/subscribers","subscription_url":"https://api.github.com/repos/allenai/allennlp/subscription","commits_url":"https://api.github.com/repos/allenai/allennlp/commits{/sha}","git_commits_url":"https://api.github.com/repos/allenai/allennlp/git/commits{/sha}","comments_url":"https://api.github.com/repos/allenai/allennlp/comments{/number}","issue_comment_url":"https://api.github.com/repos/allenai/allennlp/issues/comments{/number}","contents_url":"https://api.github.com/repos/allenai/allennlp/contents/{+path}","compare_url":"https://api.github.com/repos/allenai/allennlp/compare/{base}...{head}","merges_url":"https://api.github.com/repos/allenai/allennlp/merges","archive_url":"https://api.github.com/repos/allenai/allennlp/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/allenai/allennlp/downloads","issues_url":"https://api.github.com/repos/allenai/allennlp/issues{/number}","pulls_url":"https://api.github.com/repos/allenai/allennlp/pulls{/number}","milestones_url":"https://api.github.com/repos/allenai/allennlp/milestones{/number}","notifications_url":"https://api.github.com/repos/allenai/allennlp/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/allenai/allennlp/labels{/name}","releases_url":"https://api.github.com/repos/allenai/allennlp/releases{/id}","deployments_url":"https://api.github.com/repos/allenai/allennlp/deployments","created_at":"2017-05-15T15:52:41Z","updated_at":"2018-11-21T18:56:10Z","pushed_at":"2018-11-21T15:20:29Z","git_url":"git://github.com/allenai/allennlp.git","ssh_url":"git@github.com:allenai/allennlp.git","clone_url":"https://github.com/allenai/allennlp.git","svn_url":"https://github.com/allenai/allennlp","homepage":"http://www.allennlp.org/","size":53798,"stargazers_count":4259,"watchers_count":4259,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":773,"mirror_url":null,"archived":false,"open_issues_count":93,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0","node_id":"MDc6TGljZW5zZTI="},"forks":773,"open_issues":93,"watchers":4259,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/allenai/allennlp/pulls/2067"},"html":{"href":"https://github.com/allenai/allennlp/pull/2067"},"issue":{"href":"https://api.github.com/repos/allenai/allennlp/issues/2067"},"comments":{"href":"https://api.github.com/repos/allenai/allennlp/issues/2067/comments"},"review_comments":{"href":"https://api.github.com/repos/allenai/allennlp/pulls/2067/comments"},"review_comment":{"href":"https://api.github.com/repos/allenai/allennlp/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/allenai/allennlp/pulls/2067/commits"},"statuses":{"href":"https://api.github.com/repos/allenai/allennlp/statuses/3a4c0bcf77483e5a3d635f85c80579037bdf09e9"}},"author_association":"CONTRIBUTOR"}}
{"actor":{"display_login":"matt-gardner"}}