{"action":"created","comment":{"url":"https://api.github.com/repos/klarna/brod/pulls/comments/57526717","id":57526717,"diff_hunk":"@@ -0,0 +1,889 @@\n+%%%\n+%%%   Copyright (c) 2016 Klarna AB\n+%%%\n+%%%   Licensed under the Apache License, Version 2.0 (the \"License\");\n+%%%   you may not use this file except in compliance with the License.\n+%%%   You may obtain a copy of the License at\n+%%%\n+%%%       http://www.apache.org/licenses/LICENSE-2.0\n+%%%\n+%%%   Unless required by applicable law or agreed to in writing, software\n+%%%   distributed under the License is distributed on an \"AS IS\" BASIS,\n+%%%   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+%%%   See the License for the specific language governing permissions and\n+%%%   limitations under the License.\n+%%%\n+\n+%%%=============================================================================\n+%%% @doc\n+%%% Kafka consumer group membership controller\n+%%%\n+%%% @copyright 2016 Klarna AB\n+%%% @end\n+%%%=============================================================================\n+\n+-module(brod_group_controller).\n+-behaviour(gen_server).\n+\n+-export([ ack/5\n+        , commit_offsets/1\n+        , start_link/4\n+        ]).\n+\n+-export([ code_change/3\n+        , handle_call/3\n+        , handle_cast/2\n+        , handle_info/2\n+        , init/1\n+        , terminate/2\n+        ]).\n+\n+-include(\"brod_int.hrl\").\n+\n+-define(PARTITION_ASSIGMENT_STRATEGY_ROUNDROBIN, roundrobin). %% default\n+-type partition_assignment_strategy() :: atom().\n+\n+%% default configs\n+-define(SESSION_TIMEOUT_SECONDS, 10).\n+-define(HEARTBEAT_RATE_SECONDS, 2).\n+-define(PROTOCOL_TYPE, <<\"consumer\">>).\n+-define(MAX_REJOIN_ATTEMPTS, 5).\n+-define(REJOIN_DELAY_SECONDS, 1).\n+-define(OFFSET_COMMIT_POLICY, commit_to_kafka_v2).\n+-define(OFFSET_COMMIT_INTERVAL_SECONDS, 60).\n+%% use kfaka's offset meta-topic retention policy\n+-define(OFFSET_RETENTION_DEFAULT, -1).\n+\n+-define(ESCALATE_EC(EC), kpro_ErrorCode:is_error(EC) andalso erlang:throw(EC)).\n+\n+-define(ESCALATE(Expr), fun() ->\n+                          case Expr of\n+                            {ok, Result}    -> Result;\n+                            {error, Reason} -> throw(Reason)\n+                          end\n+                        end()).\n+\n+%% loopback commands\n+-define(LO_CMD_SEND_HB, lo_cmd_send_heartbeat).\n+-define(LO_CMD_COMMIT_OFFSETS, lo_cmd_commit_offsets).\n+-define(LO_CMD_STABILIZE(AttemptCount, Reason),\n+        {lo_cmd_stabilize, AttemptCount, Reason}).\n+\n+-type config() :: group_config().\n+-type ts() :: erlang:timestamp().\n+-type member() :: kpro_GroupMemberMetadata().\n+-type offset_commit_policy() :: brod_offset_commit_policy().\n+\n+-record(state,\n+        { client :: client()\n+        , groupId :: group_id()\n+          %% Group member ID, which should be set to empty in the first\n+          %% join group request, then a new member id is assigned by the\n+          %% group coordinator and in join group response.\n+          %% This filed may change if the member has lost connection\n+          %% to the coordinator and received 'UnknownMemberId' exception\n+          %% in response messages.\n+        , memberId = <<\"\">> :: member_id()\n+          %% State#state.memberId =:= State#state.leaderId if\n+          %% elected as group leader by the coordinator.\n+        , leaderId :: member_id()\n+          %% Generation ID is used by the group coordinator to sync state\n+          %% of the group members, e.g. kick out stale members who have not\n+          %% kept up with the latest generation ID bumps.\n+        , generationId = 0 :: integer()\n+          %% A set of topic names where the group members consumes from\n+        , topics = [] :: [topic()]\n+          %% This is the result of group coordinator discovery.\n+          %% It may change when the coordinator is down then a new one\n+          %% is elected among the kafka cluster members.\n+        , coordinator :: endpoint()\n+          %% The socket pid to the group coordinator broker.\n+          %% This socket is dedicated for group management and\n+          %% offset commit requests.\n+          %% We can not just get a payload socket from client\n+          %% because the socket might be shared with other group\n+          %% members in the same client, however group members are\n+          %% distinguished by connections to coordinator\n+        , sock_pid  :: pid()\n+          %% heartbeat reference, to discard stale responses\n+        , hb_ref :: {corr_id(), ts()}\n+          %% all group members received in the join group response\n+          %% this field is currently not used, but the binary encoded\n+          %% kpro_GroupMemberMetadata.protocolMetadata.userData field\n+          %% can be useful for 'sticky' assignments etc. in the future\n+        , members = [] :: [member()]\n+          %% Set to false before joining the group\n+          %% then set to true when sucessfully joined the group.\n+          %% This is by far only used to prevent the timer-triggered\n+          %% loopback command message sending a HeartbeatRequest to\n+          %% the group coordinator broker.\n+        , is_in_group = false :: boolean()\n+          %% The message-set subscriber which subscribes to all\n+          %% assigned topic-partitions.\n+        , subscriber :: pid()\n+          %% The offsets that has been acknowledged by the subscriber\n+          %% i.e. the offsets that are ready for commit.\n+          %% NOTE: this filed is not used if offset_commit_policy is\n+          %% 'consumer_managed'\n+        , acked_offsets = [] :: [{{topic(), partition()}, offset()}]\n+\n+          %% configs, see start_link/4 doc for details\n+        , partition_assignment_strategy  :: partition_assignment_strategy()\n+        , session_timeout_seconds        :: pos_integer()\n+        , heartbeat_rate_seconds         :: pos_integer()\n+        , max_rejoin_attempts            :: non_neg_integer()\n+        , rejoin_delay_seconds           :: non_neg_integer()\n+        , offset_retention_seconds       :: ?undef | integer()\n+        , offset_commit_policy           :: offset_commit_policy()\n+        , offset_commit_interval_seconds :: pos_integer()\n+        }).\n+\n+-define(IS_LEADER(S), (S#state.leaderId =:= S#state.memberId)).\n+\n+%%%_* APIs =====================================================================\n+\n+%% @doc To be called by group subscriber.\n+%% Client:  ClientId (or pid, but not recommended)\n+%% GroupId: Predefined globally unique (in a kafka cluster) binary string.\n+%% Topics:  Predefined set of topic names in the group.\n+%% Config:  The group controller configs in a proplist, possible entries:\n+%%  - partition_assignment_strategy  (optional, default = roundrobin)\n+%%      roudrobin: Take all topic-offset (sorted [{TopicName, Partition}] list)\n+%%                 assign one to each member in a roundrobin fashion.\n+%%      TODO: support sticky assignments\n+%%  - session_timeout_seconds (optional, default = 10)\n+%%      Time in seconds for the group coordinator broker to consider a member\n+%%      'down' if no heartbeat or any kind of requests received from a broker\n+%%      in the past N seconds.\n+%%      A group member may also consider the coordinator broker 'down' if no\n+%%      heartbeat response response received in the past N seconds.\n+%%  - heartbeat_rate_seconds (optional, default = 2)\n+%%      Time in seconds for the member to 'ping' the group coordinator.\n+%%      OBS: Care should be taken when picking the number, on one hand, we do\n+%%           not want to flush the broker with requests if we set it too low,\n+%%           on the other hand, if set it too high, it may take too long for\n+%%           the members to realise status changes of the group such as\n+%%           assignment rebalacing or group coordinator switchover etc.\n+%%  - max_rejoin_attempts (optional, default = 5)\n+%%      Maximum number of times allowd for a member to re-join the group.\n+%%      The gen_server will stop if it reached the maximum number of retres.\n+%%      OBS: 'let it crash' may not be the optimal strategy here because\n+%%           the group member id is kept in the gen_server looping state and\n+%%           it is reused when re-joining the group.\n+%%  - rejoin_delay_seconds (optional, default = 1)\n+%%      Delay in seconds before re-joining the group.\n+%%  - offset_commit_policy (optional, default = commit_to_kafka_v2)\n+%%      How/where to commit offsets, possible values:\n+%%        - commit_to_kafka_v2:\n+%%            Group controller will commit the offsets to kafka using\n+%%            version 2 OffsetCommitRequest.\n+%%        - consumer_managed:\n+%%            The subscirber (brodh_group_subscriber.erl) is responsible","path":"src/brod_group_controller.erl","position":181,"original_position":181,"commit_id":"b55f08b2d6a12d5e40bab95802e120a0ee5b9a73","original_commit_id":"b55f08b2d6a12d5e40bab95802e120a0ee5b9a73","user":{"login":"id","id":138221,"avatar_url":"https://avatars.githubusercontent.com/u/138221?v=3","gravatar_id":"","url":"https://api.github.com/users/id","html_url":"https://github.com/id","followers_url":"https://api.github.com/users/id/followers","following_url":"https://api.github.com/users/id/following{/other_user}","gists_url":"https://api.github.com/users/id/gists{/gist_id}","starred_url":"https://api.github.com/users/id/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/id/subscriptions","organizations_url":"https://api.github.com/users/id/orgs","repos_url":"https://api.github.com/users/id/repos","events_url":"https://api.github.com/users/id/events{/privacy}","received_events_url":"https://api.github.com/users/id/received_events","type":"User","site_admin":false},"body":"brod_","created_at":"2016-03-27T10:58:37Z","updated_at":"2016-03-27T10:58:37Z","html_url":"https://github.com/klarna/brod/pull/37#discussion_r57526717","pull_request_url":"https://api.github.com/repos/klarna/brod/pulls/37","_links":{"self":{"href":"https://api.github.com/repos/klarna/brod/pulls/comments/57526717"},"html":{"href":"https://github.com/klarna/brod/pull/37#discussion_r57526717"},"pull_request":{"href":"https://api.github.com/repos/klarna/brod/pulls/37"}}},"pull_request":{"url":"https://api.github.com/repos/klarna/brod/pulls/37","id":61991641,"html_url":"https://github.com/klarna/brod/pull/37","diff_url":"https://github.com/klarna/brod/pull/37.diff","patch_url":"https://github.com/klarna/brod/pull/37.patch","issue_url":"https://api.github.com/repos/klarna/brod/issues/37","number":37,"state":"open","locked":false,"title":"Consumer group","user":{"login":"zmstone","id":164324,"avatar_url":"https://avatars.githubusercontent.com/u/164324?v=3","gravatar_id":"","url":"https://api.github.com/users/zmstone","html_url":"https://github.com/zmstone","followers_url":"https://api.github.com/users/zmstone/followers","following_url":"https://api.github.com/users/zmstone/following{/other_user}","gists_url":"https://api.github.com/users/zmstone/gists{/gist_id}","starred_url":"https://api.github.com/users/zmstone/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/zmstone/subscriptions","organizations_url":"https://api.github.com/users/zmstone/orgs","repos_url":"https://api.github.com/users/zmstone/repos","events_url":"https://api.github.com/users/zmstone/events{/privacy}","received_events_url":"https://api.github.com/users/zmstone/received_events","type":"User","site_admin":false},"body":"","created_at":"2016-03-07T22:25:01Z","updated_at":"2016-03-27T10:58:37Z","closed_at":null,"merged_at":null,"merge_commit_sha":"9b3053d6ead93b4d0ffc93594d92171eb0d3c190","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/klarna/brod/pulls/37/commits","review_comments_url":"https://api.github.com/repos/klarna/brod/pulls/37/comments","review_comment_url":"https://api.github.com/repos/klarna/brod/pulls/comments{/number}","comments_url":"https://api.github.com/repos/klarna/brod/issues/37/comments","statuses_url":"https://api.github.com/repos/klarna/brod/statuses/b55f08b2d6a12d5e40bab95802e120a0ee5b9a73","head":{"label":"klarna:consumer-group","ref":"consumer-group","sha":"b55f08b2d6a12d5e40bab95802e120a0ee5b9a73","user":{"login":"klarna","id":394540,"avatar_url":"https://avatars.githubusercontent.com/u/394540?v=3","gravatar_id":"","url":"https://api.github.com/users/klarna","html_url":"https://github.com/klarna","followers_url":"https://api.github.com/users/klarna/followers","following_url":"https://api.github.com/users/klarna/following{/other_user}","gists_url":"https://api.github.com/users/klarna/gists{/gist_id}","starred_url":"https://api.github.com/users/klarna/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/klarna/subscriptions","organizations_url":"https://api.github.com/users/klarna/orgs","repos_url":"https://api.github.com/users/klarna/repos","events_url":"https://api.github.com/users/klarna/events{/privacy}","received_events_url":"https://api.github.com/users/klarna/received_events","type":"Organization","site_admin":false},"repo":{"id":31531585,"name":"brod","full_name":"klarna/brod","owner":{"login":"klarna","id":394540,"avatar_url":"https://avatars.githubusercontent.com/u/394540?v=3","gravatar_id":"","url":"https://api.github.com/users/klarna","html_url":"https://github.com/klarna","followers_url":"https://api.github.com/users/klarna/followers","following_url":"https://api.github.com/users/klarna/following{/other_user}","gists_url":"https://api.github.com/users/klarna/gists{/gist_id}","starred_url":"https://api.github.com/users/klarna/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/klarna/subscriptions","organizations_url":"https://api.github.com/users/klarna/orgs","repos_url":"https://api.github.com/users/klarna/repos","events_url":"https://api.github.com/users/klarna/events{/privacy}","received_events_url":"https://api.github.com/users/klarna/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/klarna/brod","description":"Client for Apache Kafka in Erlang","fork":false,"url":"https://api.github.com/repos/klarna/brod","forks_url":"https://api.github.com/repos/klarna/brod/forks","keys_url":"https://api.github.com/repos/klarna/brod/keys{/key_id}","collaborators_url":"https://api.github.com/repos/klarna/brod/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/klarna/brod/teams","hooks_url":"https://api.github.com/repos/klarna/brod/hooks","issue_events_url":"https://api.github.com/repos/klarna/brod/issues/events{/number}","events_url":"https://api.github.com/repos/klarna/brod/events","assignees_url":"https://api.github.com/repos/klarna/brod/assignees{/user}","branches_url":"https://api.github.com/repos/klarna/brod/branches{/branch}","tags_url":"https://api.github.com/repos/klarna/brod/tags","blobs_url":"https://api.github.com/repos/klarna/brod/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/klarna/brod/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/klarna/brod/git/refs{/sha}","trees_url":"https://api.github.com/repos/klarna/brod/git/trees{/sha}","statuses_url":"https://api.github.com/repos/klarna/brod/statuses/{sha}","languages_url":"https://api.github.com/repos/klarna/brod/languages","stargazers_url":"https://api.github.com/repos/klarna/brod/stargazers","contributors_url":"https://api.github.com/repos/klarna/brod/contributors","subscribers_url":"https://api.github.com/repos/klarna/brod/subscribers","subscription_url":"https://api.github.com/repos/klarna/brod/subscription","commits_url":"https://api.github.com/repos/klarna/brod/commits{/sha}","git_commits_url":"https://api.github.com/repos/klarna/brod/git/commits{/sha}","comments_url":"https://api.github.com/repos/klarna/brod/comments{/number}","issue_comment_url":"https://api.github.com/repos/klarna/brod/issues/comments{/number}","contents_url":"https://api.github.com/repos/klarna/brod/contents/{+path}","compare_url":"https://api.github.com/repos/klarna/brod/compare/{base}...{head}","merges_url":"https://api.github.com/repos/klarna/brod/merges","archive_url":"https://api.github.com/repos/klarna/brod/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/klarna/brod/downloads","issues_url":"https://api.github.com/repos/klarna/brod/issues{/number}","pulls_url":"https://api.github.com/repos/klarna/brod/pulls{/number}","milestones_url":"https://api.github.com/repos/klarna/brod/milestones{/number}","notifications_url":"https://api.github.com/repos/klarna/brod/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/klarna/brod/labels{/name}","releases_url":"https://api.github.com/repos/klarna/brod/releases{/id}","deployments_url":"https://api.github.com/repos/klarna/brod/deployments","created_at":"2015-03-02T09:02:18Z","updated_at":"2016-03-10T17:49:01Z","pushed_at":"2016-03-26T19:10:15Z","git_url":"git://github.com/klarna/brod.git","ssh_url":"git@github.com:klarna/brod.git","clone_url":"https://github.com/klarna/brod.git","svn_url":"https://github.com/klarna/brod","homepage":null,"size":1181,"stargazers_count":50,"watchers_count":50,"language":"Erlang","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":14,"mirror_url":null,"open_issues_count":7,"forks":14,"open_issues":7,"watchers":50,"default_branch":"master"}},"base":{"label":"klarna:2.0-dev","ref":"2.0-dev","sha":"43bfccb2532176c1123ea852342daec5b47a0a3b","user":{"login":"klarna","id":394540,"avatar_url":"https://avatars.githubusercontent.com/u/394540?v=3","gravatar_id":"","url":"https://api.github.com/users/klarna","html_url":"https://github.com/klarna","followers_url":"https://api.github.com/users/klarna/followers","following_url":"https://api.github.com/users/klarna/following{/other_user}","gists_url":"https://api.github.com/users/klarna/gists{/gist_id}","starred_url":"https://api.github.com/users/klarna/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/klarna/subscriptions","organizations_url":"https://api.github.com/users/klarna/orgs","repos_url":"https://api.github.com/users/klarna/repos","events_url":"https://api.github.com/users/klarna/events{/privacy}","received_events_url":"https://api.github.com/users/klarna/received_events","type":"Organization","site_admin":false},"repo":{"id":31531585,"name":"brod","full_name":"klarna/brod","owner":{"login":"klarna","id":394540,"avatar_url":"https://avatars.githubusercontent.com/u/394540?v=3","gravatar_id":"","url":"https://api.github.com/users/klarna","html_url":"https://github.com/klarna","followers_url":"https://api.github.com/users/klarna/followers","following_url":"https://api.github.com/users/klarna/following{/other_user}","gists_url":"https://api.github.com/users/klarna/gists{/gist_id}","starred_url":"https://api.github.com/users/klarna/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/klarna/subscriptions","organizations_url":"https://api.github.com/users/klarna/orgs","repos_url":"https://api.github.com/users/klarna/repos","events_url":"https://api.github.com/users/klarna/events{/privacy}","received_events_url":"https://api.github.com/users/klarna/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/klarna/brod","description":"Client for Apache Kafka in Erlang","fork":false,"url":"https://api.github.com/repos/klarna/brod","forks_url":"https://api.github.com/repos/klarna/brod/forks","keys_url":"https://api.github.com/repos/klarna/brod/keys{/key_id}","collaborators_url":"https://api.github.com/repos/klarna/brod/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/klarna/brod/teams","hooks_url":"https://api.github.com/repos/klarna/brod/hooks","issue_events_url":"https://api.github.com/repos/klarna/brod/issues/events{/number}","events_url":"https://api.github.com/repos/klarna/brod/events","assignees_url":"https://api.github.com/repos/klarna/brod/assignees{/user}","branches_url":"https://api.github.com/repos/klarna/brod/branches{/branch}","tags_url":"https://api.github.com/repos/klarna/brod/tags","blobs_url":"https://api.github.com/repos/klarna/brod/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/klarna/brod/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/klarna/brod/git/refs{/sha}","trees_url":"https://api.github.com/repos/klarna/brod/git/trees{/sha}","statuses_url":"https://api.github.com/repos/klarna/brod/statuses/{sha}","languages_url":"https://api.github.com/repos/klarna/brod/languages","stargazers_url":"https://api.github.com/repos/klarna/brod/stargazers","contributors_url":"https://api.github.com/repos/klarna/brod/contributors","subscribers_url":"https://api.github.com/repos/klarna/brod/subscribers","subscription_url":"https://api.github.com/repos/klarna/brod/subscription","commits_url":"https://api.github.com/repos/klarna/brod/commits{/sha}","git_commits_url":"https://api.github.com/repos/klarna/brod/git/commits{/sha}","comments_url":"https://api.github.com/repos/klarna/brod/comments{/number}","issue_comment_url":"https://api.github.com/repos/klarna/brod/issues/comments{/number}","contents_url":"https://api.github.com/repos/klarna/brod/contents/{+path}","compare_url":"https://api.github.com/repos/klarna/brod/compare/{base}...{head}","merges_url":"https://api.github.com/repos/klarna/brod/merges","archive_url":"https://api.github.com/repos/klarna/brod/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/klarna/brod/downloads","issues_url":"https://api.github.com/repos/klarna/brod/issues{/number}","pulls_url":"https://api.github.com/repos/klarna/brod/pulls{/number}","milestones_url":"https://api.github.com/repos/klarna/brod/milestones{/number}","notifications_url":"https://api.github.com/repos/klarna/brod/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/klarna/brod/labels{/name}","releases_url":"https://api.github.com/repos/klarna/brod/releases{/id}","deployments_url":"https://api.github.com/repos/klarna/brod/deployments","created_at":"2015-03-02T09:02:18Z","updated_at":"2016-03-10T17:49:01Z","pushed_at":"2016-03-26T19:10:15Z","git_url":"git://github.com/klarna/brod.git","ssh_url":"git@github.com:klarna/brod.git","clone_url":"https://github.com/klarna/brod.git","svn_url":"https://github.com/klarna/brod","homepage":null,"size":1181,"stargazers_count":50,"watchers_count":50,"language":"Erlang","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":14,"mirror_url":null,"open_issues_count":7,"forks":14,"open_issues":7,"watchers":50,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/klarna/brod/pulls/37"},"html":{"href":"https://github.com/klarna/brod/pull/37"},"issue":{"href":"https://api.github.com/repos/klarna/brod/issues/37"},"comments":{"href":"https://api.github.com/repos/klarna/brod/issues/37/comments"},"review_comments":{"href":"https://api.github.com/repos/klarna/brod/pulls/37/comments"},"review_comment":{"href":"https://api.github.com/repos/klarna/brod/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/klarna/brod/pulls/37/commits"},"statuses":{"href":"https://api.github.com/repos/klarna/brod/statuses/b55f08b2d6a12d5e40bab95802e120a0ee5b9a73"}}}}