{"action":"created","comment":{"url":"https://api.github.com/repos/aio-libs/aiokafka/pulls/comments/56334378","id":56334378,"diff_hunk":"@@ -1,375 +1,612 @@\n import asyncio\n-from collections import deque\n-\n import logging\n-import numbers\n-from itertools import zip_longest, repeat\n \n-from kafka.common import (\n-    OffsetRequest, OffsetCommitRequest, OffsetFetchRequest,\n-    UnknownTopicOrPartitionError, FetchRequest,\n-    ConsumerFetchSizeTooSmall, ConsumerNoMoreData,\n-    check_error\n-)\n+from kafka.common import OffsetAndMetadata, TopicPartition\n+from kafka.coordinator.assignors.roundrobin import RoundRobinPartitionAssignor\n+from kafka.consumer.subscription_state import SubscriptionState\n+\n+from aiokafka.client import AIOKafkaClient\n+from aiokafka.group_coordinator import GroupCoordinator\n+from aiokafka.fetcher import Fetcher\n+from aiokafka import __version__\n+\n+log = logging.getLogger(__name__)\n+\n+\n+class AIOKafkaConsumer(object):\n+    \"\"\"Consume records from a Kafka cluster.\n+\n+    The consumer will transparently handle the failure of servers in the Kafka\n+    cluster, and adapt as topic-partitions are created or migrate between\n+    brokers. It also interacts with the assigned kafka Group Coordinator node\n+    to allow multiple consumers to load balance consumption of topics (requires\n+    kafka >= 0.9.0.0).\n+\n+    Arguments:\n+        *topics (str): optional list of topics to subscribe to. If not set,\n+            call subscribe() or assign() before consuming records.\n+        bootstrap_servers: 'host[:port]' string (or list of 'host[:port]'\n+            strings) that the consumer should contact to bootstrap initial\n+            cluster metadata. This does not have to be the full node list.\n+            It just needs to have at least one broker that will respond to a\n+            Metadata API Request. Default port is 9092. If no servers are\n+            specified, will default to localhost:9092.\n+        client_id (str): a name for this client. This string is passed in\n+            each request to servers and can be used to identify specific\n+            server-side log entries that correspond to this client. Also\n+            submitted to GroupCoordinator for logging with respect to\n+            consumer group administration. Default: 'aiokafka-{version}'\n+        group_id (str or None): name of the consumer group to join for dynamic\n+            partition assignment (if enabled), and to use for fetching and\n+            committing offsets. If None, auto-partition assignment (via\n+            group coordinator) and offset commits are disabled.\n+            Default: 'aiokafka-default-group'\n+        key_deserializer (callable): Any callable that takes a\n+            raw message key and returns a deserialized key.\n+        value_deserializer (callable, optional): Any callable that takes a\n+            raw message value and returns a deserialized value.\n+        fetch_min_bytes (int): Minimum amount of data the server should\n+            return for a fetch request, otherwise wait up to\n+            fetch_max_wait_ms for more data to accumulate. Default: 1.\n+        fetch_max_wait_ms (int): The maximum amount of time in milliseconds\n+            the server will block before answering the fetch request if\n+            there isn't sufficient data to immediately satisfy the\n+            requirement given by fetch_min_bytes. Default: 500.\n+        max_partition_fetch_bytes (int): The maximum amount of data\n+            per-partition the server will return. The maximum total memory\n+            used for a request = #partitions * max_partition_fetch_bytes.\n+            This size must be at least as large as the maximum message size\n+            the server allows or else it is possible for the producer to\n+            send messages larger than the consumer can fetch. If that\n+            happens, the consumer can get stuck trying to fetch a large\n+            message on a certain partition. Default: 1048576.\n+        request_timeout_ms (int): Client request timeout in milliseconds.\n+            Default: 40000.\n+        retry_backoff_ms (int): Milliseconds to backoff when retrying on\n+            errors. Default: 100.\n+        reconnect_backoff_ms (int): The amount of time in milliseconds to\n+            wait before attempting to reconnect to a given host.\n+            Default: 50.\n+        auto_offset_reset (str): A policy for resetting offsets on\n+            OffsetOutOfRange errors: 'earliest' will move to the oldest\n+            available message, 'latest' will move to the most recent. Any\n+            ofther value will raise the exception. Default: 'latest'.\n+        enable_auto_commit (bool): If true the consumer's offset will be\n+            periodically committed in the background. Default: True.\n+        auto_commit_interval_ms (int): milliseconds between automatic\n+            offset commits, if enable_auto_commit is True. Default: 5000.\n+        default_offset_commit_callback (callable): called as\n+            callback(offsets, response) response will be either an Exception\n+            or a OffsetCommitResponse struct. This callback can be used to\n+            trigger custom actions when a commit request completes.\n+        check_crcs (bool): Automatically check the CRC32 of the records\n+            consumed. This ensures no on-the-wire or on-disk corruption to\n+            the messages occurred. This check adds some overhead, so it may\n+            be disabled in cases seeking extreme performance. Default: True\n+        metadata_max_age_ms (int): The period of time in milliseconds after\n+            which we force a refresh of metadata even if we haven't seen any\n+            partition leadership changes to proactively discover any new\n+            brokers or partitions. Default: 300000\n+        partition_assignment_strategy (list): List of objects to use to","path":"aiokafka/consumer.py","position":102,"original_position":100,"commit_id":"9c278e3f25420f0533211cd1baf920361ba36dcf","original_commit_id":"4ea6bf64721c3c5ff32e284691379b16237ee27a","user":{"login":"fabregas","id":129505,"avatar_url":"https://avatars.githubusercontent.com/u/129505?v=3","gravatar_id":"","url":"https://api.github.com/users/fabregas","html_url":"https://github.com/fabregas","followers_url":"https://api.github.com/users/fabregas/followers","following_url":"https://api.github.com/users/fabregas/following{/other_user}","gists_url":"https://api.github.com/users/fabregas/gists{/gist_id}","starred_url":"https://api.github.com/users/fabregas/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/fabregas/subscriptions","organizations_url":"https://api.github.com/users/fabregas/orgs","repos_url":"https://api.github.com/users/fabregas/repos","events_url":"https://api.github.com/users/fabregas/events{/privacy}","received_events_url":"https://api.github.com/users/fabregas/received_events","type":"User","site_admin":false},"body":"done","created_at":"2016-03-16T13:44:26Z","updated_at":"2016-03-16T13:44:26Z","html_url":"https://github.com/aio-libs/aiokafka/pull/22#discussion_r56334378","pull_request_url":"https://api.github.com/repos/aio-libs/aiokafka/pulls/22","_links":{"self":{"href":"https://api.github.com/repos/aio-libs/aiokafka/pulls/comments/56334378"},"html":{"href":"https://github.com/aio-libs/aiokafka/pull/22#discussion_r56334378"},"pull_request":{"href":"https://api.github.com/repos/aio-libs/aiokafka/pulls/22"}}},"pull_request":{"url":"https://api.github.com/repos/aio-libs/aiokafka/pulls/22","id":62779288,"html_url":"https://github.com/aio-libs/aiokafka/pull/22","diff_url":"https://github.com/aio-libs/aiokafka/pull/22.diff","patch_url":"https://github.com/aio-libs/aiokafka/pull/22.patch","issue_url":"https://api.github.com/repos/aio-libs/aiokafka/issues/22","number":22,"state":"open","locked":false,"title":"refactored consumer ","user":{"login":"fabregas","id":129505,"avatar_url":"https://avatars.githubusercontent.com/u/129505?v=3","gravatar_id":"","url":"https://api.github.com/users/fabregas","html_url":"https://github.com/fabregas","followers_url":"https://api.github.com/users/fabregas/followers","following_url":"https://api.github.com/users/fabregas/following{/other_user}","gists_url":"https://api.github.com/users/fabregas/gists{/gist_id}","starred_url":"https://api.github.com/users/fabregas/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/fabregas/subscriptions","organizations_url":"https://api.github.com/users/fabregas/orgs","repos_url":"https://api.github.com/users/fabregas/repos","events_url":"https://api.github.com/users/fabregas/events{/privacy}","received_events_url":"https://api.github.com/users/fabregas/received_events","type":"User","site_admin":false},"body":"","created_at":"2016-03-14T16:46:38Z","updated_at":"2016-03-16T13:44:26Z","closed_at":null,"merged_at":null,"merge_commit_sha":"9204801461404bb1ba9b936151c369bda1747d12","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/aio-libs/aiokafka/pulls/22/commits","review_comments_url":"https://api.github.com/repos/aio-libs/aiokafka/pulls/22/comments","review_comment_url":"https://api.github.com/repos/aio-libs/aiokafka/pulls/comments{/number}","comments_url":"https://api.github.com/repos/aio-libs/aiokafka/issues/22/comments","statuses_url":"https://api.github.com/repos/aio-libs/aiokafka/statuses/9c278e3f25420f0533211cd1baf920361ba36dcf","head":{"label":"fabregas:refactor_consumer_v2","ref":"refactor_consumer_v2","sha":"9c278e3f25420f0533211cd1baf920361ba36dcf","user":{"login":"fabregas","id":129505,"avatar_url":"https://avatars.githubusercontent.com/u/129505?v=3","gravatar_id":"","url":"https://api.github.com/users/fabregas","html_url":"https://github.com/fabregas","followers_url":"https://api.github.com/users/fabregas/followers","following_url":"https://api.github.com/users/fabregas/following{/other_user}","gists_url":"https://api.github.com/users/fabregas/gists{/gist_id}","starred_url":"https://api.github.com/users/fabregas/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/fabregas/subscriptions","organizations_url":"https://api.github.com/users/fabregas/orgs","repos_url":"https://api.github.com/users/fabregas/repos","events_url":"https://api.github.com/users/fabregas/events{/privacy}","received_events_url":"https://api.github.com/users/fabregas/received_events","type":"User","site_admin":false},"repo":{"id":51517734,"name":"aiokafka","full_name":"fabregas/aiokafka","owner":{"login":"fabregas","id":129505,"avatar_url":"https://avatars.githubusercontent.com/u/129505?v=3","gravatar_id":"","url":"https://api.github.com/users/fabregas","html_url":"https://github.com/fabregas","followers_url":"https://api.github.com/users/fabregas/followers","following_url":"https://api.github.com/users/fabregas/following{/other_user}","gists_url":"https://api.github.com/users/fabregas/gists{/gist_id}","starred_url":"https://api.github.com/users/fabregas/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/fabregas/subscriptions","organizations_url":"https://api.github.com/users/fabregas/orgs","repos_url":"https://api.github.com/users/fabregas/repos","events_url":"https://api.github.com/users/fabregas/events{/privacy}","received_events_url":"https://api.github.com/users/fabregas/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/fabregas/aiokafka","description":"asyncio client for kafka","fork":true,"url":"https://api.github.com/repos/fabregas/aiokafka","forks_url":"https://api.github.com/repos/fabregas/aiokafka/forks","keys_url":"https://api.github.com/repos/fabregas/aiokafka/keys{/key_id}","collaborators_url":"https://api.github.com/repos/fabregas/aiokafka/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/fabregas/aiokafka/teams","hooks_url":"https://api.github.com/repos/fabregas/aiokafka/hooks","issue_events_url":"https://api.github.com/repos/fabregas/aiokafka/issues/events{/number}","events_url":"https://api.github.com/repos/fabregas/aiokafka/events","assignees_url":"https://api.github.com/repos/fabregas/aiokafka/assignees{/user}","branches_url":"https://api.github.com/repos/fabregas/aiokafka/branches{/branch}","tags_url":"https://api.github.com/repos/fabregas/aiokafka/tags","blobs_url":"https://api.github.com/repos/fabregas/aiokafka/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/fabregas/aiokafka/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/fabregas/aiokafka/git/refs{/sha}","trees_url":"https://api.github.com/repos/fabregas/aiokafka/git/trees{/sha}","statuses_url":"https://api.github.com/repos/fabregas/aiokafka/statuses/{sha}","languages_url":"https://api.github.com/repos/fabregas/aiokafka/languages","stargazers_url":"https://api.github.com/repos/fabregas/aiokafka/stargazers","contributors_url":"https://api.github.com/repos/fabregas/aiokafka/contributors","subscribers_url":"https://api.github.com/repos/fabregas/aiokafka/subscribers","subscription_url":"https://api.github.com/repos/fabregas/aiokafka/subscription","commits_url":"https://api.github.com/repos/fabregas/aiokafka/commits{/sha}","git_commits_url":"https://api.github.com/repos/fabregas/aiokafka/git/commits{/sha}","comments_url":"https://api.github.com/repos/fabregas/aiokafka/comments{/number}","issue_comment_url":"https://api.github.com/repos/fabregas/aiokafka/issues/comments{/number}","contents_url":"https://api.github.com/repos/fabregas/aiokafka/contents/{+path}","compare_url":"https://api.github.com/repos/fabregas/aiokafka/compare/{base}...{head}","merges_url":"https://api.github.com/repos/fabregas/aiokafka/merges","archive_url":"https://api.github.com/repos/fabregas/aiokafka/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/fabregas/aiokafka/downloads","issues_url":"https://api.github.com/repos/fabregas/aiokafka/issues{/number}","pulls_url":"https://api.github.com/repos/fabregas/aiokafka/pulls{/number}","milestones_url":"https://api.github.com/repos/fabregas/aiokafka/milestones{/number}","notifications_url":"https://api.github.com/repos/fabregas/aiokafka/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/fabregas/aiokafka/labels{/name}","releases_url":"https://api.github.com/repos/fabregas/aiokafka/releases{/id}","deployments_url":"https://api.github.com/repos/fabregas/aiokafka/deployments","created_at":"2016-02-11T13:52:20Z","updated_at":"2016-02-11T13:52:21Z","pushed_at":"2016-03-16T13:42:32Z","git_url":"git://github.com/fabregas/aiokafka.git","ssh_url":"git@github.com:fabregas/aiokafka.git","clone_url":"https://github.com/fabregas/aiokafka.git","svn_url":"https://github.com/fabregas/aiokafka","homepage":null,"size":265,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":1,"forks":0,"open_issues":1,"watchers":0,"default_branch":"master"}},"base":{"label":"aio-libs:v0.9","ref":"v0.9","sha":"cb641aac8953e4a7b77cc7f31aa368c8b4f61a9a","user":{"login":"aio-libs","id":7049303,"avatar_url":"https://avatars.githubusercontent.com/u/7049303?v=3","gravatar_id":"","url":"https://api.github.com/users/aio-libs","html_url":"https://github.com/aio-libs","followers_url":"https://api.github.com/users/aio-libs/followers","following_url":"https://api.github.com/users/aio-libs/following{/other_user}","gists_url":"https://api.github.com/users/aio-libs/gists{/gist_id}","starred_url":"https://api.github.com/users/aio-libs/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/aio-libs/subscriptions","organizations_url":"https://api.github.com/users/aio-libs/orgs","repos_url":"https://api.github.com/users/aio-libs/repos","events_url":"https://api.github.com/users/aio-libs/events{/privacy}","received_events_url":"https://api.github.com/users/aio-libs/received_events","type":"Organization","site_admin":false},"repo":{"id":26046938,"name":"aiokafka","full_name":"aio-libs/aiokafka","owner":{"login":"aio-libs","id":7049303,"avatar_url":"https://avatars.githubusercontent.com/u/7049303?v=3","gravatar_id":"","url":"https://api.github.com/users/aio-libs","html_url":"https://github.com/aio-libs","followers_url":"https://api.github.com/users/aio-libs/followers","following_url":"https://api.github.com/users/aio-libs/following{/other_user}","gists_url":"https://api.github.com/users/aio-libs/gists{/gist_id}","starred_url":"https://api.github.com/users/aio-libs/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/aio-libs/subscriptions","organizations_url":"https://api.github.com/users/aio-libs/orgs","repos_url":"https://api.github.com/users/aio-libs/repos","events_url":"https://api.github.com/users/aio-libs/events{/privacy}","received_events_url":"https://api.github.com/users/aio-libs/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/aio-libs/aiokafka","description":"asyncio client for kafka","fork":false,"url":"https://api.github.com/repos/aio-libs/aiokafka","forks_url":"https://api.github.com/repos/aio-libs/aiokafka/forks","keys_url":"https://api.github.com/repos/aio-libs/aiokafka/keys{/key_id}","collaborators_url":"https://api.github.com/repos/aio-libs/aiokafka/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/aio-libs/aiokafka/teams","hooks_url":"https://api.github.com/repos/aio-libs/aiokafka/hooks","issue_events_url":"https://api.github.com/repos/aio-libs/aiokafka/issues/events{/number}","events_url":"https://api.github.com/repos/aio-libs/aiokafka/events","assignees_url":"https://api.github.com/repos/aio-libs/aiokafka/assignees{/user}","branches_url":"https://api.github.com/repos/aio-libs/aiokafka/branches{/branch}","tags_url":"https://api.github.com/repos/aio-libs/aiokafka/tags","blobs_url":"https://api.github.com/repos/aio-libs/aiokafka/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/aio-libs/aiokafka/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/aio-libs/aiokafka/git/refs{/sha}","trees_url":"https://api.github.com/repos/aio-libs/aiokafka/git/trees{/sha}","statuses_url":"https://api.github.com/repos/aio-libs/aiokafka/statuses/{sha}","languages_url":"https://api.github.com/repos/aio-libs/aiokafka/languages","stargazers_url":"https://api.github.com/repos/aio-libs/aiokafka/stargazers","contributors_url":"https://api.github.com/repos/aio-libs/aiokafka/contributors","subscribers_url":"https://api.github.com/repos/aio-libs/aiokafka/subscribers","subscription_url":"https://api.github.com/repos/aio-libs/aiokafka/subscription","commits_url":"https://api.github.com/repos/aio-libs/aiokafka/commits{/sha}","git_commits_url":"https://api.github.com/repos/aio-libs/aiokafka/git/commits{/sha}","comments_url":"https://api.github.com/repos/aio-libs/aiokafka/comments{/number}","issue_comment_url":"https://api.github.com/repos/aio-libs/aiokafka/issues/comments{/number}","contents_url":"https://api.github.com/repos/aio-libs/aiokafka/contents/{+path}","compare_url":"https://api.github.com/repos/aio-libs/aiokafka/compare/{base}...{head}","merges_url":"https://api.github.com/repos/aio-libs/aiokafka/merges","archive_url":"https://api.github.com/repos/aio-libs/aiokafka/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/aio-libs/aiokafka/downloads","issues_url":"https://api.github.com/repos/aio-libs/aiokafka/issues{/number}","pulls_url":"https://api.github.com/repos/aio-libs/aiokafka/pulls{/number}","milestones_url":"https://api.github.com/repos/aio-libs/aiokafka/milestones{/number}","notifications_url":"https://api.github.com/repos/aio-libs/aiokafka/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/aio-libs/aiokafka/labels{/name}","releases_url":"https://api.github.com/repos/aio-libs/aiokafka/releases{/id}","deployments_url":"https://api.github.com/repos/aio-libs/aiokafka/deployments","created_at":"2014-11-01T09:42:10Z","updated_at":"2016-02-27T02:39:25Z","pushed_at":"2016-03-16T13:42:33Z","git_url":"git://github.com/aio-libs/aiokafka.git","ssh_url":"git@github.com:aio-libs/aiokafka.git","clone_url":"https://github.com/aio-libs/aiokafka.git","svn_url":"https://github.com/aio-libs/aiokafka","homepage":null,"size":193,"stargazers_count":12,"watchers_count":12,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":5,"mirror_url":null,"open_issues_count":7,"forks":5,"open_issues":7,"watchers":12,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/aio-libs/aiokafka/pulls/22"},"html":{"href":"https://github.com/aio-libs/aiokafka/pull/22"},"issue":{"href":"https://api.github.com/repos/aio-libs/aiokafka/issues/22"},"comments":{"href":"https://api.github.com/repos/aio-libs/aiokafka/issues/22/comments"},"review_comments":{"href":"https://api.github.com/repos/aio-libs/aiokafka/pulls/22/comments"},"review_comment":{"href":"https://api.github.com/repos/aio-libs/aiokafka/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/aio-libs/aiokafka/pulls/22/commits"},"statuses":{"href":"https://api.github.com/repos/aio-libs/aiokafka/statuses/9c278e3f25420f0533211cd1baf920361ba36dcf"}}}}