{"action":"created","comment":{"url":"https://api.github.com/repos/husky-team/husky/pulls/comments/87358686","pull_request_review_id":7973036,"id":87358686,"diff_hunk":"@@ -0,0 +1,146 @@\n+// Copyright 2016 Husky Team\r\n+//\r\n+// Licensed under the Apache License, Version 2.0 (the \"License\");\r\n+// you may not use this file except in compliance with the License.\r\n+// You may obtain a copy of the License at\r\n+//\r\n+// http://www.apache.org/licenses/LICENSE-2.0\r\n+//\r\n+// Unless required by applicable law or agreed to in writing, software\r\n+// distributed under the License is distributed on an \"AS IS\" BASIS,\r\n+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\r\n+// See the License for the specific language governing permissions and\r\n+// limitations under the License.\r\n+\r\n+#pragma once\r\n+\r\n+#include <atomic>\r\n+#include <string>\r\n+#include <vector>\r\n+\r\n+#include \"base/serialization.hpp\"\r\n+#include \"base/thread_support.hpp\"\r\n+#include \"zmq_helpers.hpp\"\r\n+\r\n+namespace husky {\r\n+\r\n+template <typename CollectT>\r\n+class Shuffler {\r\n+   public:\r\n+    Shuffler() {}\r\n+\r\n+    Shuffler(Shuffler&& s) : collection_(s.collection_) { s.collection_ = nullptr; }\r\n+\r\n+    CollectT& storage() {\r\n+        if (collection_ == nullptr)\r\n+            collection_ = new CollectT();\r\n+        return *collection_;\r\n+    }\r\n+\r\n+    virtual ~Shuffler() {\r\n+        if (collection_ != nullptr)\r\n+            delete collection_;\r\n+        collection_ = nullptr;\r\n+    }\r\n+\r\n+   protected:\r\n+    CollectT* collection_ = nullptr;\r\n+};\r\n+\r\n+template <typename CellT>\r\n+class ShuffleCombiner {\r\n+   protected:\r\n+    typedef std::vector<CellT> CollectT;\r\n+\r\n+   public:\r\n+    ShuffleCombiner() {}\r\n+\r\n+    ShuffleCombiner(ShuffleCombiner&& sc) {\r\n+        messages_ = std::move(sc.messages_);\r\n+        publisher_ = sc.publisher_;\r\n+        subscriber_ = sc.subscriber_;\r\n+        num_units_ = sc.num_units_;\r\n+        num_local_workers_ = sc.num_local_workers_;\r\n+        channel_id_ = sc.channel_id_;\r\n+        local_id_ = sc.local_id_;\r\n+        sc.num_units_ = 0;\r\n+        sc.num_local_workers_ = 0;\r\n+        sc.channel_id_ = -1;\r\n+        sc.local_id_ = -1;\r\n+        sc.publisher_ = nullptr;\r\n+        sc.subscriber_ = nullptr;\r\n+    }\r\n+\r\n+    ~ShuffleCombiner() {\r\n+        messages_.clear();\r\n+        delete publisher_;\r\n+        delete subscriber_;\r\n+    }\r\n+\r\n+    void init(int num_global_threads, int num_local_threads, int channel_id, int local_id) {\r\n+        num_units_ = num_global_threads;\r\n+        num_local_workers_ = num_local_threads;\r\n+        channel_id_ = channel_id;\r\n+        local_id_ = local_id;\r\n+        messages_.resize(num_units_);\r\n+        publisher_ = channel_sc_socket_map_[channel_id][local_id_];\r\n+        subscriber_ = channel_sc_socket_map_[channel_id][local_id_ + num_local_workers_];\r\n+    }\r\n+\r\n+    void send_shuffler_buffer() { husky::zmq_send_int32(publisher_, local_id_, ZMQ_NOBLOCK); }\r\n+\r\n+    int access_next() {\r\n+        int worker_id = husky::zmq_recv_int32(subscriber_);\r\n+        return worker_id;\r\n+    }\r\n+\r\n+    CollectT& storage(unsigned idx) { return messages_[idx].storage(); }\r\n+\r\n+    static void init_sockets(int num_local_workers, int channel_id, zmq::context_t& context) {\r\n+        if (channel_sc_socket_map_.find(channel_id) == channel_sc_socket_map_.end()) {\r","path":"core/shuffle_combiner.hpp","position":100,"original_position":100,"commit_id":"f5a714dae389310b7849cd6c12d220192c8fee26","original_commit_id":"f5a714dae389310b7849cd6c12d220192c8fee26","user":{"login":"zzxx-husky","id":16422228,"avatar_url":"https://avatars.githubusercontent.com/u/16422228?v=3","gravatar_id":"","url":"https://api.github.com/users/zzxx-husky","html_url":"https://github.com/zzxx-husky","followers_url":"https://api.github.com/users/zzxx-husky/followers","following_url":"https://api.github.com/users/zzxx-husky/following{/other_user}","gists_url":"https://api.github.com/users/zzxx-husky/gists{/gist_id}","starred_url":"https://api.github.com/users/zzxx-husky/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/zzxx-husky/subscriptions","organizations_url":"https://api.github.com/users/zzxx-husky/orgs","repos_url":"https://api.github.com/users/zzxx-husky/repos","events_url":"https://api.github.com/users/zzxx-husky/events{/privacy}","received_events_url":"https://api.github.com/users/zzxx-husky/received_events","type":"User","site_admin":false},"body":"Maybe you change to share the two sockets among different channels in the future.","created_at":"2016-11-10T09:42:47Z","updated_at":"2016-11-10T09:42:47Z","html_url":"https://github.com/husky-team/husky/pull/124#discussion_r87358686","pull_request_url":"https://api.github.com/repos/husky-team/husky/pulls/124","_links":{"self":{"href":"https://api.github.com/repos/husky-team/husky/pulls/comments/87358686"},"html":{"href":"https://github.com/husky-team/husky/pull/124#discussion_r87358686"},"pull_request":{"href":"https://api.github.com/repos/husky-team/husky/pulls/124"}}},"pull_request":{"url":"https://api.github.com/repos/husky-team/husky/pulls/124","id":93107465,"html_url":"https://github.com/husky-team/husky/pull/124","diff_url":"https://github.com/husky-team/husky/pull/124.diff","patch_url":"https://github.com/husky-team/husky/pull/124.patch","issue_url":"https://api.github.com/repos/husky-team/husky/issues/124","number":124,"state":"open","locked":false,"title":"[Core] An access_next method in ShuffleCombiner","user":{"login":"xwkuang5","id":12459119,"avatar_url":"https://avatars.githubusercontent.com/u/12459119?v=3","gravatar_id":"","url":"https://api.github.com/users/xwkuang5","html_url":"https://github.com/xwkuang5","followers_url":"https://api.github.com/users/xwkuang5/followers","following_url":"https://api.github.com/users/xwkuang5/following{/other_user}","gists_url":"https://api.github.com/users/xwkuang5/gists{/gist_id}","starred_url":"https://api.github.com/users/xwkuang5/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/xwkuang5/subscriptions","organizations_url":"https://api.github.com/users/xwkuang5/orgs","repos_url":"https://api.github.com/users/xwkuang5/repos","events_url":"https://api.github.com/users/xwkuang5/events{/privacy}","received_events_url":"https://api.github.com/users/xwkuang5/received_events","type":"User","site_admin":false},"body":"issue #49\r\nImplement shuffle combiner with ZeroMQ publisher-subscriber pattern.\r\nEach worker will be both publisher and subscriber at the same time.","created_at":"2016-11-10T08:37:23Z","updated_at":"2016-11-10T09:42:47Z","closed_at":null,"merged_at":null,"merge_commit_sha":"216fe5ddf6c5be2a1a5187364a4f7741a958e93d","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/husky-team/husky/pulls/124/commits","review_comments_url":"https://api.github.com/repos/husky-team/husky/pulls/124/comments","review_comment_url":"https://api.github.com/repos/husky-team/husky/pulls/comments{/number}","comments_url":"https://api.github.com/repos/husky-team/husky/issues/124/comments","statuses_url":"https://api.github.com/repos/husky-team/husky/statuses/f5a714dae389310b7849cd6c12d220192c8fee26","head":{"label":"xwkuang5:dev","ref":"dev","sha":"f5a714dae389310b7849cd6c12d220192c8fee26","user":{"login":"xwkuang5","id":12459119,"avatar_url":"https://avatars.githubusercontent.com/u/12459119?v=3","gravatar_id":"","url":"https://api.github.com/users/xwkuang5","html_url":"https://github.com/xwkuang5","followers_url":"https://api.github.com/users/xwkuang5/followers","following_url":"https://api.github.com/users/xwkuang5/following{/other_user}","gists_url":"https://api.github.com/users/xwkuang5/gists{/gist_id}","starred_url":"https://api.github.com/users/xwkuang5/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/xwkuang5/subscriptions","organizations_url":"https://api.github.com/users/xwkuang5/orgs","repos_url":"https://api.github.com/users/xwkuang5/repos","events_url":"https://api.github.com/users/xwkuang5/events{/privacy}","received_events_url":"https://api.github.com/users/xwkuang5/received_events","type":"User","site_admin":false},"repo":{"id":67996244,"name":"husky","full_name":"xwkuang5/husky","owner":{"login":"xwkuang5","id":12459119,"avatar_url":"https://avatars.githubusercontent.com/u/12459119?v=3","gravatar_id":"","url":"https://api.github.com/users/xwkuang5","html_url":"https://github.com/xwkuang5","followers_url":"https://api.github.com/users/xwkuang5/followers","following_url":"https://api.github.com/users/xwkuang5/following{/other_user}","gists_url":"https://api.github.com/users/xwkuang5/gists{/gist_id}","starred_url":"https://api.github.com/users/xwkuang5/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/xwkuang5/subscriptions","organizations_url":"https://api.github.com/users/xwkuang5/orgs","repos_url":"https://api.github.com/users/xwkuang5/repos","events_url":"https://api.github.com/users/xwkuang5/events{/privacy}","received_events_url":"https://api.github.com/users/xwkuang5/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/xwkuang5/husky","description":"A more expressive and most importantly, more efficient system for distributed data analytics.","fork":true,"url":"https://api.github.com/repos/xwkuang5/husky","forks_url":"https://api.github.com/repos/xwkuang5/husky/forks","keys_url":"https://api.github.com/repos/xwkuang5/husky/keys{/key_id}","collaborators_url":"https://api.github.com/repos/xwkuang5/husky/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/xwkuang5/husky/teams","hooks_url":"https://api.github.com/repos/xwkuang5/husky/hooks","issue_events_url":"https://api.github.com/repos/xwkuang5/husky/issues/events{/number}","events_url":"https://api.github.com/repos/xwkuang5/husky/events","assignees_url":"https://api.github.com/repos/xwkuang5/husky/assignees{/user}","branches_url":"https://api.github.com/repos/xwkuang5/husky/branches{/branch}","tags_url":"https://api.github.com/repos/xwkuang5/husky/tags","blobs_url":"https://api.github.com/repos/xwkuang5/husky/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/xwkuang5/husky/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/xwkuang5/husky/git/refs{/sha}","trees_url":"https://api.github.com/repos/xwkuang5/husky/git/trees{/sha}","statuses_url":"https://api.github.com/repos/xwkuang5/husky/statuses/{sha}","languages_url":"https://api.github.com/repos/xwkuang5/husky/languages","stargazers_url":"https://api.github.com/repos/xwkuang5/husky/stargazers","contributors_url":"https://api.github.com/repos/xwkuang5/husky/contributors","subscribers_url":"https://api.github.com/repos/xwkuang5/husky/subscribers","subscription_url":"https://api.github.com/repos/xwkuang5/husky/subscription","commits_url":"https://api.github.com/repos/xwkuang5/husky/commits{/sha}","git_commits_url":"https://api.github.com/repos/xwkuang5/husky/git/commits{/sha}","comments_url":"https://api.github.com/repos/xwkuang5/husky/comments{/number}","issue_comment_url":"https://api.github.com/repos/xwkuang5/husky/issues/comments{/number}","contents_url":"https://api.github.com/repos/xwkuang5/husky/contents/{+path}","compare_url":"https://api.github.com/repos/xwkuang5/husky/compare/{base}...{head}","merges_url":"https://api.github.com/repos/xwkuang5/husky/merges","archive_url":"https://api.github.com/repos/xwkuang5/husky/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/xwkuang5/husky/downloads","issues_url":"https://api.github.com/repos/xwkuang5/husky/issues{/number}","pulls_url":"https://api.github.com/repos/xwkuang5/husky/pulls{/number}","milestones_url":"https://api.github.com/repos/xwkuang5/husky/milestones{/number}","notifications_url":"https://api.github.com/repos/xwkuang5/husky/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/xwkuang5/husky/labels{/name}","releases_url":"https://api.github.com/repos/xwkuang5/husky/releases{/id}","deployments_url":"https://api.github.com/repos/xwkuang5/husky/deployments","created_at":"2016-09-12T09:39:13Z","updated_at":"2016-09-12T09:39:15Z","pushed_at":"2016-11-10T09:07:49Z","git_url":"git://github.com/xwkuang5/husky.git","ssh_url":"git@github.com:xwkuang5/husky.git","clone_url":"https://github.com/xwkuang5/husky.git","svn_url":"https://github.com/xwkuang5/husky","homepage":"http://www.husky-project.com/","size":529,"stargazers_count":0,"watchers_count":0,"language":"C++","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"husky-team:master","ref":"master","sha":"0566619ce075b69e74fb979f2eb5eb672283643c","user":{"login":"husky-team","id":21353188,"avatar_url":"https://avatars.githubusercontent.com/u/21353188?v=3","gravatar_id":"","url":"https://api.github.com/users/husky-team","html_url":"https://github.com/husky-team","followers_url":"https://api.github.com/users/husky-team/followers","following_url":"https://api.github.com/users/husky-team/following{/other_user}","gists_url":"https://api.github.com/users/husky-team/gists{/gist_id}","starred_url":"https://api.github.com/users/husky-team/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/husky-team/subscriptions","organizations_url":"https://api.github.com/users/husky-team/orgs","repos_url":"https://api.github.com/users/husky-team/repos","events_url":"https://api.github.com/users/husky-team/events{/privacy}","received_events_url":"https://api.github.com/users/husky-team/received_events","type":"Organization","site_admin":false},"repo":{"id":67001593,"name":"husky","full_name":"husky-team/husky","owner":{"login":"husky-team","id":21353188,"avatar_url":"https://avatars.githubusercontent.com/u/21353188?v=3","gravatar_id":"","url":"https://api.github.com/users/husky-team","html_url":"https://github.com/husky-team","followers_url":"https://api.github.com/users/husky-team/followers","following_url":"https://api.github.com/users/husky-team/following{/other_user}","gists_url":"https://api.github.com/users/husky-team/gists{/gist_id}","starred_url":"https://api.github.com/users/husky-team/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/husky-team/subscriptions","organizations_url":"https://api.github.com/users/husky-team/orgs","repos_url":"https://api.github.com/users/husky-team/repos","events_url":"https://api.github.com/users/husky-team/events{/privacy}","received_events_url":"https://api.github.com/users/husky-team/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/husky-team/husky","description":"A more expressive and most importantly, more efficient system for distributed data analytics.","fork":false,"url":"https://api.github.com/repos/husky-team/husky","forks_url":"https://api.github.com/repos/husky-team/husky/forks","keys_url":"https://api.github.com/repos/husky-team/husky/keys{/key_id}","collaborators_url":"https://api.github.com/repos/husky-team/husky/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/husky-team/husky/teams","hooks_url":"https://api.github.com/repos/husky-team/husky/hooks","issue_events_url":"https://api.github.com/repos/husky-team/husky/issues/events{/number}","events_url":"https://api.github.com/repos/husky-team/husky/events","assignees_url":"https://api.github.com/repos/husky-team/husky/assignees{/user}","branches_url":"https://api.github.com/repos/husky-team/husky/branches{/branch}","tags_url":"https://api.github.com/repos/husky-team/husky/tags","blobs_url":"https://api.github.com/repos/husky-team/husky/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/husky-team/husky/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/husky-team/husky/git/refs{/sha}","trees_url":"https://api.github.com/repos/husky-team/husky/git/trees{/sha}","statuses_url":"https://api.github.com/repos/husky-team/husky/statuses/{sha}","languages_url":"https://api.github.com/repos/husky-team/husky/languages","stargazers_url":"https://api.github.com/repos/husky-team/husky/stargazers","contributors_url":"https://api.github.com/repos/husky-team/husky/contributors","subscribers_url":"https://api.github.com/repos/husky-team/husky/subscribers","subscription_url":"https://api.github.com/repos/husky-team/husky/subscription","commits_url":"https://api.github.com/repos/husky-team/husky/commits{/sha}","git_commits_url":"https://api.github.com/repos/husky-team/husky/git/commits{/sha}","comments_url":"https://api.github.com/repos/husky-team/husky/comments{/number}","issue_comment_url":"https://api.github.com/repos/husky-team/husky/issues/comments{/number}","contents_url":"https://api.github.com/repos/husky-team/husky/contents/{+path}","compare_url":"https://api.github.com/repos/husky-team/husky/compare/{base}...{head}","merges_url":"https://api.github.com/repos/husky-team/husky/merges","archive_url":"https://api.github.com/repos/husky-team/husky/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/husky-team/husky/downloads","issues_url":"https://api.github.com/repos/husky-team/husky/issues{/number}","pulls_url":"https://api.github.com/repos/husky-team/husky/pulls{/number}","milestones_url":"https://api.github.com/repos/husky-team/husky/milestones{/number}","notifications_url":"https://api.github.com/repos/husky-team/husky/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/husky-team/husky/labels{/name}","releases_url":"https://api.github.com/repos/husky-team/husky/releases{/id}","deployments_url":"https://api.github.com/repos/husky-team/husky/deployments","created_at":"2016-08-31T03:55:53Z","updated_at":"2016-11-08T17:14:16Z","pushed_at":"2016-11-10T09:07:49Z","git_url":"git://github.com/husky-team/husky.git","ssh_url":"git@github.com:husky-team/husky.git","clone_url":"https://github.com/husky-team/husky.git","svn_url":"https://github.com/husky-team/husky","homepage":"http://www.husky-project.com/","size":594,"stargazers_count":23,"watchers_count":23,"language":"C++","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":35,"mirror_url":null,"open_issues_count":12,"forks":35,"open_issues":12,"watchers":23,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/husky-team/husky/pulls/124"},"html":{"href":"https://github.com/husky-team/husky/pull/124"},"issue":{"href":"https://api.github.com/repos/husky-team/husky/issues/124"},"comments":{"href":"https://api.github.com/repos/husky-team/husky/issues/124/comments"},"review_comments":{"href":"https://api.github.com/repos/husky-team/husky/pulls/124/comments"},"review_comment":{"href":"https://api.github.com/repos/husky-team/husky/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/husky-team/husky/pulls/124/commits"},"statuses":{"href":"https://api.github.com/repos/husky-team/husky/statuses/f5a714dae389310b7849cd6c12d220192c8fee26"}}}}
{"actor":{"display_login":"zzxx-husky"}}