{"action":"created","comment":{"url":"https://api.github.com/repos/pytorch/pytorch/pulls/comments/227467292","pull_request_review_id":167515261,"id":227467292,"node_id":"MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDIyNzQ2NzI5Mg==","diff_hunk":"@@ -14,48 +10,24 @@ namespace data {\n namespace samplers {\n \n /// A `Sampler` that returns random indices.\n-class RandomSampler : public Sampler {\n+class RandomSampler : public Sampler<> {\n  public:\n   /// Constructs a `RandomSampler` with a size and dtype for the stored indices.\n   ///\n   /// The constructor will eagerly allocate all required indices, which is the\n   /// sequence `0 ... size - 1`. `index_dtype` is the data type of the stored\n   /// indices. You can change it to influence memory usage.\n-  explicit RandomSampler(int64_t size, Dtype index_dtype = torch::kInt64)\n-      : indices_(torch::randperm(size, index_dtype)) {}\n+  explicit RandomSampler(int64_t size, Dtype index_dtype = torch::kInt64);\n \n   /// Resets the `RandomSampler` to a new set of indices.\n-  void reset() override {\n-    // This allocates a new chunk of memory every time (just FYI). It should be\n-    // amortized over the entire epoch hopefully.\n-    indices_ = torch::randperm(indices_.numel(), indices_.options());\n-    index_ = 0;\n-  }\n+  void reset() override;\n \n   /// Returns the next batch of indices.\n-  optional<std::vector<size_t>> next(size_t batch_size) override {\n-    AT_ASSERT(index_ <= indices_.numel());\n-    const size_t remaining_indices = indices_.numel() - index_;\n-    if (remaining_indices == 0) {\n-      return nullopt;\n-    }\n-    std::vector<size_t> index_batch(std::min(batch_size, remaining_indices));\n-    auto slice = indices_.slice(/*dim=*/0, index_, index_ + index_batch.size());\n-    // You may want to store your indices with 32-bit or less, but here we need\n-    // to upcast to 64-bit. A batch itself won't hold too many indices, so that\n-    // should be ok. Note that if this indeed results in a type promotion, there\n-    // will be two allocations: one for the upcast slice, and one for the\n-    // returned `index_batch` vector.\n-    slice = slice.to(torch::kInt64);\n-    const auto* data = slice.data<int64_t>();\n-    std::copy(data, data + index_batch.size(), index_batch.begin());\n-    index_ += index_batch.size();\n-    return index_batch;\n-  }\n+  optional<std::vector<size_t>> next(size_t batch_size) override;","path":"torch/csrc/api/include/torch/data/samplers/random.h","position":57,"original_position":57,"commit_id":"63f50dbac5a6771ff40424f0ed2265d37cd11605","original_commit_id":"63f50dbac5a6771ff40424f0ed2265d37cd11605","user":{"login":"goldsborough","id":6429851,"node_id":"MDQ6VXNlcjY0Mjk4NTE=","avatar_url":"https://avatars3.githubusercontent.com/u/6429851?v=4","gravatar_id":"","url":"https://api.github.com/users/goldsborough","html_url":"https://github.com/goldsborough","followers_url":"https://api.github.com/users/goldsborough/followers","following_url":"https://api.github.com/users/goldsborough/following{/other_user}","gists_url":"https://api.github.com/users/goldsborough/gists{/gist_id}","starred_url":"https://api.github.com/users/goldsborough/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/goldsborough/subscriptions","organizations_url":"https://api.github.com/users/goldsborough/orgs","repos_url":"https://api.github.com/users/goldsborough/repos","events_url":"https://api.github.com/users/goldsborough/events{/privacy}","received_events_url":"https://api.github.com/users/goldsborough/received_events","type":"User","site_admin":false},"body":"Yep it works fine","created_at":"2018-10-23T16:21:01Z","updated_at":"2018-10-23T16:21:01Z","html_url":"https://github.com/pytorch/pytorch/pull/12960#discussion_r227467292","pull_request_url":"https://api.github.com/repos/pytorch/pytorch/pulls/12960","author_association":"CONTRIBUTOR","_links":{"self":{"href":"https://api.github.com/repos/pytorch/pytorch/pulls/comments/227467292"},"html":{"href":"https://github.com/pytorch/pytorch/pull/12960#discussion_r227467292"},"pull_request":{"href":"https://api.github.com/repos/pytorch/pytorch/pulls/12960"}},"in_reply_to_id":227198838},"pull_request":{"url":"https://api.github.com/repos/pytorch/pytorch/pulls/12960","id":224854727,"node_id":"MDExOlB1bGxSZXF1ZXN0MjI0ODU0NzI3","html_url":"https://github.com/pytorch/pytorch/pull/12960","diff_url":"https://github.com/pytorch/pytorch/pull/12960.diff","patch_url":"https://github.com/pytorch/pytorch/pull/12960.patch","issue_url":"https://api.github.com/repos/pytorch/pytorch/issues/12960","number":12960,"state":"open","locked":false,"title":"[C++ Frontend] Remove size() from BatchDataset and templatize IndexType","user":{"login":"goldsborough","id":6429851,"node_id":"MDQ6VXNlcjY0Mjk4NTE=","avatar_url":"https://avatars3.githubusercontent.com/u/6429851?v=4","gravatar_id":"","url":"https://api.github.com/users/goldsborough","html_url":"https://github.com/goldsborough","followers_url":"https://api.github.com/users/goldsborough/followers","following_url":"https://api.github.com/users/goldsborough/following{/other_user}","gists_url":"https://api.github.com/users/goldsborough/gists{/gist_id}","starred_url":"https://api.github.com/users/goldsborough/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/goldsborough/subscriptions","organizations_url":"https://api.github.com/users/goldsborough/orgs","repos_url":"https://api.github.com/users/goldsborough/repos","events_url":"https://api.github.com/users/goldsborough/events{/privacy}","received_events_url":"https://api.github.com/users/goldsborough/received_events","type":"User","site_admin":false},"body":"This PR brings to changes to the recently landed C++ Frontend dataloader:\r\n\r\n1. Removes the `size()` method from `BatchDataset`. This makes it cleaner to implement unsized (\"infinite stream\") datasets. The method was not used much beyond initial configuration.\r\n2. Makes the index type of a dataset a template parameter of `BatchDataset` and `Sampler`. This essentially allows custom index types instead of only `vector<size_t>`. This greatly improves flexibility.\r\n\r\nSee the `InfiniteStreamDataset` and `TestIndex` datasets in the tests for what this enables.\r\n\r\nSome additional minor updates and code movements too.\r\n\r\n@apaszke @SsnL ","created_at":"2018-10-23T00:27:34Z","updated_at":"2018-10-23T16:21:01Z","closed_at":null,"merged_at":null,"merge_commit_sha":"621f98e78279fdc2c217cf94076d5f174b57e04f","assignee":null,"assignees":[],"requested_reviewers":[{"login":"ebetica","id":3605224,"node_id":"MDQ6VXNlcjM2MDUyMjQ=","avatar_url":"https://avatars3.githubusercontent.com/u/3605224?v=4","gravatar_id":"","url":"https://api.github.com/users/ebetica","html_url":"https://github.com/ebetica","followers_url":"https://api.github.com/users/ebetica/followers","following_url":"https://api.github.com/users/ebetica/following{/other_user}","gists_url":"https://api.github.com/users/ebetica/gists{/gist_id}","starred_url":"https://api.github.com/users/ebetica/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ebetica/subscriptions","organizations_url":"https://api.github.com/users/ebetica/orgs","repos_url":"https://api.github.com/users/ebetica/repos","events_url":"https://api.github.com/users/ebetica/events{/privacy}","received_events_url":"https://api.github.com/users/ebetica/received_events","type":"User","site_admin":false}],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/pytorch/pytorch/pulls/12960/commits","review_comments_url":"https://api.github.com/repos/pytorch/pytorch/pulls/12960/comments","review_comment_url":"https://api.github.com/repos/pytorch/pytorch/pulls/comments{/number}","comments_url":"https://api.github.com/repos/pytorch/pytorch/issues/12960/comments","statuses_url":"https://api.github.com/repos/pytorch/pytorch/statuses/63f50dbac5a6771ff40424f0ed2265d37cd11605","head":{"label":"goldsborough:cpp-data-loader-improved","ref":"cpp-data-loader-improved","sha":"63f50dbac5a6771ff40424f0ed2265d37cd11605","user":{"login":"goldsborough","id":6429851,"node_id":"MDQ6VXNlcjY0Mjk4NTE=","avatar_url":"https://avatars3.githubusercontent.com/u/6429851?v=4","gravatar_id":"","url":"https://api.github.com/users/goldsborough","html_url":"https://github.com/goldsborough","followers_url":"https://api.github.com/users/goldsborough/followers","following_url":"https://api.github.com/users/goldsborough/following{/other_user}","gists_url":"https://api.github.com/users/goldsborough/gists{/gist_id}","starred_url":"https://api.github.com/users/goldsborough/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/goldsborough/subscriptions","organizations_url":"https://api.github.com/users/goldsborough/orgs","repos_url":"https://api.github.com/users/goldsborough/repos","events_url":"https://api.github.com/users/goldsborough/events{/privacy}","received_events_url":"https://api.github.com/users/goldsborough/received_events","type":"User","site_admin":false},"repo":{"id":115096378,"node_id":"MDEwOlJlcG9zaXRvcnkxMTUwOTYzNzg=","name":"pytorch","full_name":"goldsborough/pytorch","private":false,"owner":{"login":"goldsborough","id":6429851,"node_id":"MDQ6VXNlcjY0Mjk4NTE=","avatar_url":"https://avatars3.githubusercontent.com/u/6429851?v=4","gravatar_id":"","url":"https://api.github.com/users/goldsborough","html_url":"https://github.com/goldsborough","followers_url":"https://api.github.com/users/goldsborough/followers","following_url":"https://api.github.com/users/goldsborough/following{/other_user}","gists_url":"https://api.github.com/users/goldsborough/gists{/gist_id}","starred_url":"https://api.github.com/users/goldsborough/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/goldsborough/subscriptions","organizations_url":"https://api.github.com/users/goldsborough/orgs","repos_url":"https://api.github.com/users/goldsborough/repos","events_url":"https://api.github.com/users/goldsborough/events{/privacy}","received_events_url":"https://api.github.com/users/goldsborough/received_events","type":"User","site_admin":false},"html_url":"https://github.com/goldsborough/pytorch","description":"Tensors and Dynamic neural networks in Python  with strong GPU acceleration","fork":true,"url":"https://api.github.com/repos/goldsborough/pytorch","forks_url":"https://api.github.com/repos/goldsborough/pytorch/forks","keys_url":"https://api.github.com/repos/goldsborough/pytorch/keys{/key_id}","collaborators_url":"https://api.github.com/repos/goldsborough/pytorch/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/goldsborough/pytorch/teams","hooks_url":"https://api.github.com/repos/goldsborough/pytorch/hooks","issue_events_url":"https://api.github.com/repos/goldsborough/pytorch/issues/events{/number}","events_url":"https://api.github.com/repos/goldsborough/pytorch/events","assignees_url":"https://api.github.com/repos/goldsborough/pytorch/assignees{/user}","branches_url":"https://api.github.com/repos/goldsborough/pytorch/branches{/branch}","tags_url":"https://api.github.com/repos/goldsborough/pytorch/tags","blobs_url":"https://api.github.com/repos/goldsborough/pytorch/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/goldsborough/pytorch/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/goldsborough/pytorch/git/refs{/sha}","trees_url":"https://api.github.com/repos/goldsborough/pytorch/git/trees{/sha}","statuses_url":"https://api.github.com/repos/goldsborough/pytorch/statuses/{sha}","languages_url":"https://api.github.com/repos/goldsborough/pytorch/languages","stargazers_url":"https://api.github.com/repos/goldsborough/pytorch/stargazers","contributors_url":"https://api.github.com/repos/goldsborough/pytorch/contributors","subscribers_url":"https://api.github.com/repos/goldsborough/pytorch/subscribers","subscription_url":"https://api.github.com/repos/goldsborough/pytorch/subscription","commits_url":"https://api.github.com/repos/goldsborough/pytorch/commits{/sha}","git_commits_url":"https://api.github.com/repos/goldsborough/pytorch/git/commits{/sha}","comments_url":"https://api.github.com/repos/goldsborough/pytorch/comments{/number}","issue_comment_url":"https://api.github.com/repos/goldsborough/pytorch/issues/comments{/number}","contents_url":"https://api.github.com/repos/goldsborough/pytorch/contents/{+path}","compare_url":"https://api.github.com/repos/goldsborough/pytorch/compare/{base}...{head}","merges_url":"https://api.github.com/repos/goldsborough/pytorch/merges","archive_url":"https://api.github.com/repos/goldsborough/pytorch/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/goldsborough/pytorch/downloads","issues_url":"https://api.github.com/repos/goldsborough/pytorch/issues{/number}","pulls_url":"https://api.github.com/repos/goldsborough/pytorch/pulls{/number}","milestones_url":"https://api.github.com/repos/goldsborough/pytorch/milestones{/number}","notifications_url":"https://api.github.com/repos/goldsborough/pytorch/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/goldsborough/pytorch/labels{/name}","releases_url":"https://api.github.com/repos/goldsborough/pytorch/releases{/id}","deployments_url":"https://api.github.com/repos/goldsborough/pytorch/deployments","created_at":"2017-12-22T08:49:57Z","updated_at":"2018-10-23T00:54:15Z","pushed_at":"2018-10-23T16:07:22Z","git_url":"git://github.com/goldsborough/pytorch.git","ssh_url":"git@github.com:goldsborough/pytorch.git","clone_url":"https://github.com/goldsborough/pytorch.git","svn_url":"https://github.com/goldsborough/pytorch","homepage":"http://pytorch.org","size":80312,"stargazers_count":3,"watchers_count":3,"language":"C++","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":0,"license":{"key":"other","name":"Other","spdx_id":"NOASSERTION","url":null,"node_id":"MDc6TGljZW5zZTA="},"forks":0,"open_issues":0,"watchers":3,"default_branch":"master"}},"base":{"label":"pytorch:master","ref":"master","sha":"da2da551706ccdf7e6918e8f44560e502b352ce4","user":{"login":"pytorch","id":21003710,"node_id":"MDEyOk9yZ2FuaXphdGlvbjIxMDAzNzEw","avatar_url":"https://avatars3.githubusercontent.com/u/21003710?v=4","gravatar_id":"","url":"https://api.github.com/users/pytorch","html_url":"https://github.com/pytorch","followers_url":"https://api.github.com/users/pytorch/followers","following_url":"https://api.github.com/users/pytorch/following{/other_user}","gists_url":"https://api.github.com/users/pytorch/gists{/gist_id}","starred_url":"https://api.github.com/users/pytorch/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/pytorch/subscriptions","organizations_url":"https://api.github.com/users/pytorch/orgs","repos_url":"https://api.github.com/users/pytorch/repos","events_url":"https://api.github.com/users/pytorch/events{/privacy}","received_events_url":"https://api.github.com/users/pytorch/received_events","type":"Organization","site_admin":false},"repo":{"id":65600975,"node_id":"MDEwOlJlcG9zaXRvcnk2NTYwMDk3NQ==","name":"pytorch","full_name":"pytorch/pytorch","private":false,"owner":{"login":"pytorch","id":21003710,"node_id":"MDEyOk9yZ2FuaXphdGlvbjIxMDAzNzEw","avatar_url":"https://avatars3.githubusercontent.com/u/21003710?v=4","gravatar_id":"","url":"https://api.github.com/users/pytorch","html_url":"https://github.com/pytorch","followers_url":"https://api.github.com/users/pytorch/followers","following_url":"https://api.github.com/users/pytorch/following{/other_user}","gists_url":"https://api.github.com/users/pytorch/gists{/gist_id}","starred_url":"https://api.github.com/users/pytorch/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/pytorch/subscriptions","organizations_url":"https://api.github.com/users/pytorch/orgs","repos_url":"https://api.github.com/users/pytorch/repos","events_url":"https://api.github.com/users/pytorch/events{/privacy}","received_events_url":"https://api.github.com/users/pytorch/received_events","type":"Organization","site_admin":false},"html_url":"https://github.com/pytorch/pytorch","description":"Tensors and Dynamic neural networks in Python  with strong GPU acceleration","fork":false,"url":"https://api.github.com/repos/pytorch/pytorch","forks_url":"https://api.github.com/repos/pytorch/pytorch/forks","keys_url":"https://api.github.com/repos/pytorch/pytorch/keys{/key_id}","collaborators_url":"https://api.github.com/repos/pytorch/pytorch/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/pytorch/pytorch/teams","hooks_url":"https://api.github.com/repos/pytorch/pytorch/hooks","issue_events_url":"https://api.github.com/repos/pytorch/pytorch/issues/events{/number}","events_url":"https://api.github.com/repos/pytorch/pytorch/events","assignees_url":"https://api.github.com/repos/pytorch/pytorch/assignees{/user}","branches_url":"https://api.github.com/repos/pytorch/pytorch/branches{/branch}","tags_url":"https://api.github.com/repos/pytorch/pytorch/tags","blobs_url":"https://api.github.com/repos/pytorch/pytorch/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/pytorch/pytorch/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/pytorch/pytorch/git/refs{/sha}","trees_url":"https://api.github.com/repos/pytorch/pytorch/git/trees{/sha}","statuses_url":"https://api.github.com/repos/pytorch/pytorch/statuses/{sha}","languages_url":"https://api.github.com/repos/pytorch/pytorch/languages","stargazers_url":"https://api.github.com/repos/pytorch/pytorch/stargazers","contributors_url":"https://api.github.com/repos/pytorch/pytorch/contributors","subscribers_url":"https://api.github.com/repos/pytorch/pytorch/subscribers","subscription_url":"https://api.github.com/repos/pytorch/pytorch/subscription","commits_url":"https://api.github.com/repos/pytorch/pytorch/commits{/sha}","git_commits_url":"https://api.github.com/repos/pytorch/pytorch/git/commits{/sha}","comments_url":"https://api.github.com/repos/pytorch/pytorch/comments{/number}","issue_comment_url":"https://api.github.com/repos/pytorch/pytorch/issues/comments{/number}","contents_url":"https://api.github.com/repos/pytorch/pytorch/contents/{+path}","compare_url":"https://api.github.com/repos/pytorch/pytorch/compare/{base}...{head}","merges_url":"https://api.github.com/repos/pytorch/pytorch/merges","archive_url":"https://api.github.com/repos/pytorch/pytorch/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/pytorch/pytorch/downloads","issues_url":"https://api.github.com/repos/pytorch/pytorch/issues{/number}","pulls_url":"https://api.github.com/repos/pytorch/pytorch/pulls{/number}","milestones_url":"https://api.github.com/repos/pytorch/pytorch/milestones{/number}","notifications_url":"https://api.github.com/repos/pytorch/pytorch/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/pytorch/pytorch/labels{/name}","releases_url":"https://api.github.com/repos/pytorch/pytorch/releases{/id}","deployments_url":"https://api.github.com/repos/pytorch/pytorch/deployments","created_at":"2016-08-13T05:26:41Z","updated_at":"2018-10-23T16:19:46Z","pushed_at":"2018-10-23T16:18:18Z","git_url":"git://github.com/pytorch/pytorch.git","ssh_url":"git@github.com:pytorch/pytorch.git","clone_url":"https://github.com/pytorch/pytorch.git","svn_url":"https://github.com/pytorch/pytorch","homepage":"http://pytorch.org","size":81076,"stargazers_count":20302,"watchers_count":20302,"language":"C++","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":4812,"mirror_url":null,"archived":false,"open_issues_count":1964,"license":{"key":"other","name":"Other","spdx_id":"NOASSERTION","url":null,"node_id":"MDc6TGljZW5zZTA="},"forks":4812,"open_issues":1964,"watchers":20302,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/pytorch/pytorch/pulls/12960"},"html":{"href":"https://github.com/pytorch/pytorch/pull/12960"},"issue":{"href":"https://api.github.com/repos/pytorch/pytorch/issues/12960"},"comments":{"href":"https://api.github.com/repos/pytorch/pytorch/issues/12960/comments"},"review_comments":{"href":"https://api.github.com/repos/pytorch/pytorch/pulls/12960/comments"},"review_comment":{"href":"https://api.github.com/repos/pytorch/pytorch/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/pytorch/pytorch/pulls/12960/commits"},"statuses":{"href":"https://api.github.com/repos/pytorch/pytorch/statuses/63f50dbac5a6771ff40424f0ed2265d37cd11605"}},"author_association":"CONTRIBUTOR"}}
{"actor":{"display_login":"goldsborough"}}