{"action":"created","comment":{"url":"https://api.github.com/repos/pytorch/pytorch/pulls/comments/225002312","pull_request_review_id":164508207,"id":225002312,"node_id":"MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDIyNTAwMjMxMg==","diff_hunk":"@@ -274,16 +270,27 @@ def declkey(decl):\n                 ])\n                 decl['has_tensor_options'] = True\n \n-    jit_decls = sort_decls(jit_decls)\n-    for decl in jit_decls:\n-        ops.append(OPERATOR.substitute(signature=signature(decl),\n-                                       op=emit_decl_variant(decl)))\n-\n-    # Sort the generated snippets to ensure that the generation is deterministic\n-    env = {\n-        'constructors': ops,\n-    }\n-    write(out, 'register_aten_ops.cpp', REGISTER_ATEN_OPS_CPP, env)\n+    # Group and sort the generated snippets to ensure that the\n+    # generation is deterministic\n+    jit_decl_groups = sort_decls(jit_decls)\n+\n+    # NOTE: see the comment at the top of the register_aten_ops.cpp\n+    # template regarding sharding of the generated files.\n+    num_shards = 3\n+    shards = [[] for _ in range(num_shards)]\n+\n+    # ops are assigned arbitrarily but stably to a file based on hash\n+    for group in jit_decl_groups:\n+        x = sum(ord(c) for c in group[0]['name']) % num_shards","path":"tools/jit/gen_jit_dispatch.py","position":38,"original_position":38,"commit_id":"db57af7bb99f7c96e39500e5857aab04310623ba","original_commit_id":"db57af7bb99f7c96e39500e5857aab04310623ba","user":{"login":"anderspapitto","id":1388690,"node_id":"MDQ6VXNlcjEzODg2OTA=","avatar_url":"https://avatars3.githubusercontent.com/u/1388690?v=4","gravatar_id":"","url":"https://api.github.com/users/anderspapitto","html_url":"https://github.com/anderspapitto","followers_url":"https://api.github.com/users/anderspapitto/followers","following_url":"https://api.github.com/users/anderspapitto/following{/other_user}","gists_url":"https://api.github.com/users/anderspapitto/gists{/gist_id}","starred_url":"https://api.github.com/users/anderspapitto/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/anderspapitto/subscriptions","organizations_url":"https://api.github.com/users/anderspapitto/orgs","repos_url":"https://api.github.com/users/anderspapitto/repos","events_url":"https://api.github.com/users/anderspapitto/events{/privacy}","received_events_url":"https://api.github.com/users/anderspapitto/received_events","type":"User","site_admin":false},"body":"well this already is a copy-paste of the other site :)\r\n\r\nI could switch to just calling hash(). It looks like it's deterministic across interpreter startups (I wasn't sure about this, but I've just tested it). It may vary between python versions, or maybe even between machines or environments? Not sure about this, or if it matters even if true","created_at":"2018-10-14T17:58:58Z","updated_at":"2018-10-14T17:58:58Z","html_url":"https://github.com/pytorch/pytorch/pull/12615#discussion_r225002312","pull_request_url":"https://api.github.com/repos/pytorch/pytorch/pulls/12615","author_association":"MEMBER","_links":{"self":{"href":"https://api.github.com/repos/pytorch/pytorch/pulls/comments/225002312"},"html":{"href":"https://github.com/pytorch/pytorch/pull/12615#discussion_r225002312"},"pull_request":{"href":"https://api.github.com/repos/pytorch/pytorch/pulls/12615"}},"in_reply_to_id":224973807},"pull_request":{"url":"https://api.github.com/repos/pytorch/pytorch/pulls/12615","id":222573642,"node_id":"MDExOlB1bGxSZXF1ZXN0MjIyNTczNjQy","html_url":"https://github.com/pytorch/pytorch/pull/12615","diff_url":"https://github.com/pytorch/pytorch/pull/12615.diff","patch_url":"https://github.com/pytorch/pytorch/pull/12615.patch","issue_url":"https://api.github.com/repos/pytorch/pytorch/issues/12615","number":12615,"state":"open","locked":false,"title":"split register_aten_ops.cpp into shards","user":{"login":"anderspapitto","id":1388690,"node_id":"MDQ6VXNlcjEzODg2OTA=","avatar_url":"https://avatars3.githubusercontent.com/u/1388690?v=4","gravatar_id":"","url":"https://api.github.com/users/anderspapitto","html_url":"https://github.com/anderspapitto","followers_url":"https://api.github.com/users/anderspapitto/followers","following_url":"https://api.github.com/users/anderspapitto/following{/other_user}","gists_url":"https://api.github.com/users/anderspapitto/gists{/gist_id}","starred_url":"https://api.github.com/users/anderspapitto/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/anderspapitto/subscriptions","organizations_url":"https://api.github.com/users/anderspapitto/orgs","repos_url":"https://api.github.com/users/anderspapitto/repos","events_url":"https://api.github.com/users/anderspapitto/events{/privacy}","received_events_url":"https://api.github.com/users/anderspapitto/received_events","type":"User","site_admin":false},"body":"after an analogous breakup of VariableType.cpp, the generated\r\nregister_aten_ops.cpp is now the slowest-to-compile file in a typical\r\nincremental rebuild by a wide margin. Therefore, give it the same\r\ntreatment - the generated code is split across several files to allow\r\nparallel compilation.\r\n\r\nNote that the existing code takes some care to arrange that overloads\r\nof the same op name are given in a particular order. This diff\r\npreserves that behavior, by treating all overloads of the same name as\r\na single indivisible unit, and sharding based on these groups rather\r\nthan on individual constructors.\r\n\r\n","created_at":"2018-10-12T20:09:46Z","updated_at":"2018-10-14T17:58:58Z","closed_at":null,"merged_at":null,"merge_commit_sha":"861b8aa859fb6953c4e0fec0514523d55db557ef","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/pytorch/pytorch/pulls/12615/commits","review_comments_url":"https://api.github.com/repos/pytorch/pytorch/pulls/12615/comments","review_comment_url":"https://api.github.com/repos/pytorch/pytorch/pulls/comments{/number}","comments_url":"https://api.github.com/repos/pytorch/pytorch/issues/12615/comments","statuses_url":"https://api.github.com/repos/pytorch/pytorch/statuses/db57af7bb99f7c96e39500e5857aab04310623ba","head":{"label":"anderspapitto:split-register-operators","ref":"split-register-operators","sha":"db57af7bb99f7c96e39500e5857aab04310623ba","user":{"login":"anderspapitto","id":1388690,"node_id":"MDQ6VXNlcjEzODg2OTA=","avatar_url":"https://avatars3.githubusercontent.com/u/1388690?v=4","gravatar_id":"","url":"https://api.github.com/users/anderspapitto","html_url":"https://github.com/anderspapitto","followers_url":"https://api.github.com/users/anderspapitto/followers","following_url":"https://api.github.com/users/anderspapitto/following{/other_user}","gists_url":"https://api.github.com/users/anderspapitto/gists{/gist_id}","starred_url":"https://api.github.com/users/anderspapitto/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/anderspapitto/subscriptions","organizations_url":"https://api.github.com/users/anderspapitto/orgs","repos_url":"https://api.github.com/users/anderspapitto/repos","events_url":"https://api.github.com/users/anderspapitto/events{/privacy}","received_events_url":"https://api.github.com/users/anderspapitto/received_events","type":"User","site_admin":false},"repo":{"id":108888522,"node_id":"MDEwOlJlcG9zaXRvcnkxMDg4ODg1MjI=","name":"pytorch","full_name":"anderspapitto/pytorch","private":false,"owner":{"login":"anderspapitto","id":1388690,"node_id":"MDQ6VXNlcjEzODg2OTA=","avatar_url":"https://avatars3.githubusercontent.com/u/1388690?v=4","gravatar_id":"","url":"https://api.github.com/users/anderspapitto","html_url":"https://github.com/anderspapitto","followers_url":"https://api.github.com/users/anderspapitto/followers","following_url":"https://api.github.com/users/anderspapitto/following{/other_user}","gists_url":"https://api.github.com/users/anderspapitto/gists{/gist_id}","starred_url":"https://api.github.com/users/anderspapitto/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/anderspapitto/subscriptions","organizations_url":"https://api.github.com/users/anderspapitto/orgs","repos_url":"https://api.github.com/users/anderspapitto/repos","events_url":"https://api.github.com/users/anderspapitto/events{/privacy}","received_events_url":"https://api.github.com/users/anderspapitto/received_events","type":"User","site_admin":false},"html_url":"https://github.com/anderspapitto/pytorch","description":"Tensors and Dynamic neural networks in Python  with strong GPU acceleration","fork":true,"url":"https://api.github.com/repos/anderspapitto/pytorch","forks_url":"https://api.github.com/repos/anderspapitto/pytorch/forks","keys_url":"https://api.github.com/repos/anderspapitto/pytorch/keys{/key_id}","collaborators_url":"https://api.github.com/repos/anderspapitto/pytorch/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/anderspapitto/pytorch/teams","hooks_url":"https://api.github.com/repos/anderspapitto/pytorch/hooks","issue_events_url":"https://api.github.com/repos/anderspapitto/pytorch/issues/events{/number}","events_url":"https://api.github.com/repos/anderspapitto/pytorch/events","assignees_url":"https://api.github.com/repos/anderspapitto/pytorch/assignees{/user}","branches_url":"https://api.github.com/repos/anderspapitto/pytorch/branches{/branch}","tags_url":"https://api.github.com/repos/anderspapitto/pytorch/tags","blobs_url":"https://api.github.com/repos/anderspapitto/pytorch/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/anderspapitto/pytorch/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/anderspapitto/pytorch/git/refs{/sha}","trees_url":"https://api.github.com/repos/anderspapitto/pytorch/git/trees{/sha}","statuses_url":"https://api.github.com/repos/anderspapitto/pytorch/statuses/{sha}","languages_url":"https://api.github.com/repos/anderspapitto/pytorch/languages","stargazers_url":"https://api.github.com/repos/anderspapitto/pytorch/stargazers","contributors_url":"https://api.github.com/repos/anderspapitto/pytorch/contributors","subscribers_url":"https://api.github.com/repos/anderspapitto/pytorch/subscribers","subscription_url":"https://api.github.com/repos/anderspapitto/pytorch/subscription","commits_url":"https://api.github.com/repos/anderspapitto/pytorch/commits{/sha}","git_commits_url":"https://api.github.com/repos/anderspapitto/pytorch/git/commits{/sha}","comments_url":"https://api.github.com/repos/anderspapitto/pytorch/comments{/number}","issue_comment_url":"https://api.github.com/repos/anderspapitto/pytorch/issues/comments{/number}","contents_url":"https://api.github.com/repos/anderspapitto/pytorch/contents/{+path}","compare_url":"https://api.github.com/repos/anderspapitto/pytorch/compare/{base}...{head}","merges_url":"https://api.github.com/repos/anderspapitto/pytorch/merges","archive_url":"https://api.github.com/repos/anderspapitto/pytorch/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/anderspapitto/pytorch/downloads","issues_url":"https://api.github.com/repos/anderspapitto/pytorch/issues{/number}","pulls_url":"https://api.github.com/repos/anderspapitto/pytorch/pulls{/number}","milestones_url":"https://api.github.com/repos/anderspapitto/pytorch/milestones{/number}","notifications_url":"https://api.github.com/repos/anderspapitto/pytorch/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/anderspapitto/pytorch/labels{/name}","releases_url":"https://api.github.com/repos/anderspapitto/pytorch/releases{/id}","deployments_url":"https://api.github.com/repos/anderspapitto/pytorch/deployments","created_at":"2017-10-30T18:01:07Z","updated_at":"2018-09-17T18:27:19Z","pushed_at":"2018-10-12T21:58:12Z","git_url":"git://github.com/anderspapitto/pytorch.git","ssh_url":"git@github.com:anderspapitto/pytorch.git","clone_url":"https://github.com/anderspapitto/pytorch.git","svn_url":"https://github.com/anderspapitto/pytorch","homepage":"http://pytorch.org","size":77100,"stargazers_count":0,"watchers_count":0,"language":"C++","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":0,"license":{"key":"other","name":"Other","spdx_id":"NOASSERTION","url":null,"node_id":"MDc6TGljZW5zZTA="},"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"pytorch:master","ref":"master","sha":"17ab3bd502c41bb51e868059afc59e92bfe8af2a","user":{"login":"pytorch","id":21003710,"node_id":"MDEyOk9yZ2FuaXphdGlvbjIxMDAzNzEw","avatar_url":"https://avatars3.githubusercontent.com/u/21003710?v=4","gravatar_id":"","url":"https://api.github.com/users/pytorch","html_url":"https://github.com/pytorch","followers_url":"https://api.github.com/users/pytorch/followers","following_url":"https://api.github.com/users/pytorch/following{/other_user}","gists_url":"https://api.github.com/users/pytorch/gists{/gist_id}","starred_url":"https://api.github.com/users/pytorch/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/pytorch/subscriptions","organizations_url":"https://api.github.com/users/pytorch/orgs","repos_url":"https://api.github.com/users/pytorch/repos","events_url":"https://api.github.com/users/pytorch/events{/privacy}","received_events_url":"https://api.github.com/users/pytorch/received_events","type":"Organization","site_admin":false},"repo":{"id":65600975,"node_id":"MDEwOlJlcG9zaXRvcnk2NTYwMDk3NQ==","name":"pytorch","full_name":"pytorch/pytorch","private":false,"owner":{"login":"pytorch","id":21003710,"node_id":"MDEyOk9yZ2FuaXphdGlvbjIxMDAzNzEw","avatar_url":"https://avatars3.githubusercontent.com/u/21003710?v=4","gravatar_id":"","url":"https://api.github.com/users/pytorch","html_url":"https://github.com/pytorch","followers_url":"https://api.github.com/users/pytorch/followers","following_url":"https://api.github.com/users/pytorch/following{/other_user}","gists_url":"https://api.github.com/users/pytorch/gists{/gist_id}","starred_url":"https://api.github.com/users/pytorch/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/pytorch/subscriptions","organizations_url":"https://api.github.com/users/pytorch/orgs","repos_url":"https://api.github.com/users/pytorch/repos","events_url":"https://api.github.com/users/pytorch/events{/privacy}","received_events_url":"https://api.github.com/users/pytorch/received_events","type":"Organization","site_admin":false},"html_url":"https://github.com/pytorch/pytorch","description":"Tensors and Dynamic neural networks in Python  with strong GPU acceleration","fork":false,"url":"https://api.github.com/repos/pytorch/pytorch","forks_url":"https://api.github.com/repos/pytorch/pytorch/forks","keys_url":"https://api.github.com/repos/pytorch/pytorch/keys{/key_id}","collaborators_url":"https://api.github.com/repos/pytorch/pytorch/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/pytorch/pytorch/teams","hooks_url":"https://api.github.com/repos/pytorch/pytorch/hooks","issue_events_url":"https://api.github.com/repos/pytorch/pytorch/issues/events{/number}","events_url":"https://api.github.com/repos/pytorch/pytorch/events","assignees_url":"https://api.github.com/repos/pytorch/pytorch/assignees{/user}","branches_url":"https://api.github.com/repos/pytorch/pytorch/branches{/branch}","tags_url":"https://api.github.com/repos/pytorch/pytorch/tags","blobs_url":"https://api.github.com/repos/pytorch/pytorch/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/pytorch/pytorch/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/pytorch/pytorch/git/refs{/sha}","trees_url":"https://api.github.com/repos/pytorch/pytorch/git/trees{/sha}","statuses_url":"https://api.github.com/repos/pytorch/pytorch/statuses/{sha}","languages_url":"https://api.github.com/repos/pytorch/pytorch/languages","stargazers_url":"https://api.github.com/repos/pytorch/pytorch/stargazers","contributors_url":"https://api.github.com/repos/pytorch/pytorch/contributors","subscribers_url":"https://api.github.com/repos/pytorch/pytorch/subscribers","subscription_url":"https://api.github.com/repos/pytorch/pytorch/subscription","commits_url":"https://api.github.com/repos/pytorch/pytorch/commits{/sha}","git_commits_url":"https://api.github.com/repos/pytorch/pytorch/git/commits{/sha}","comments_url":"https://api.github.com/repos/pytorch/pytorch/comments{/number}","issue_comment_url":"https://api.github.com/repos/pytorch/pytorch/issues/comments{/number}","contents_url":"https://api.github.com/repos/pytorch/pytorch/contents/{+path}","compare_url":"https://api.github.com/repos/pytorch/pytorch/compare/{base}...{head}","merges_url":"https://api.github.com/repos/pytorch/pytorch/merges","archive_url":"https://api.github.com/repos/pytorch/pytorch/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/pytorch/pytorch/downloads","issues_url":"https://api.github.com/repos/pytorch/pytorch/issues{/number}","pulls_url":"https://api.github.com/repos/pytorch/pytorch/pulls{/number}","milestones_url":"https://api.github.com/repos/pytorch/pytorch/milestones{/number}","notifications_url":"https://api.github.com/repos/pytorch/pytorch/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/pytorch/pytorch/labels{/name}","releases_url":"https://api.github.com/repos/pytorch/pytorch/releases{/id}","deployments_url":"https://api.github.com/repos/pytorch/pytorch/deployments","created_at":"2016-08-13T05:26:41Z","updated_at":"2018-10-14T17:07:18Z","pushed_at":"2018-10-14T15:46:53Z","git_url":"git://github.com/pytorch/pytorch.git","ssh_url":"git@github.com:pytorch/pytorch.git","clone_url":"https://github.com/pytorch/pytorch.git","svn_url":"https://github.com/pytorch/pytorch","homepage":"http://pytorch.org","size":79294,"stargazers_count":19902,"watchers_count":19902,"language":"C++","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":4721,"mirror_url":null,"archived":false,"open_issues_count":1888,"license":{"key":"other","name":"Other","spdx_id":"NOASSERTION","url":null,"node_id":"MDc6TGljZW5zZTA="},"forks":4721,"open_issues":1888,"watchers":19902,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/pytorch/pytorch/pulls/12615"},"html":{"href":"https://github.com/pytorch/pytorch/pull/12615"},"issue":{"href":"https://api.github.com/repos/pytorch/pytorch/issues/12615"},"comments":{"href":"https://api.github.com/repos/pytorch/pytorch/issues/12615/comments"},"review_comments":{"href":"https://api.github.com/repos/pytorch/pytorch/pulls/12615/comments"},"review_comment":{"href":"https://api.github.com/repos/pytorch/pytorch/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/pytorch/pytorch/pulls/12615/commits"},"statuses":{"href":"https://api.github.com/repos/pytorch/pytorch/statuses/db57af7bb99f7c96e39500e5857aab04310623ba"}},"author_association":"MEMBER"}}
{"actor":{"display_login":"anderspapitto"}}