{"action":"created","comment":{"url":"https://api.github.com/repos/allegro/hermes/pulls/comments/227710116","pull_request_review_id":167804248,"id":227710116,"node_id":"MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDIyNzcxMDExNg==","diff_hunk":"@@ -1,64 +1,66 @@\n package pl.allegro.tech.hermes.management.infrastructure.kafka.service;\n \n-import kafka.api.FetchRequestBuilder;\n-import kafka.javaapi.FetchResponse;\n-import kafka.javaapi.consumer.SimpleConsumer;\n-import kafka.message.MessageAndOffset;\n+import java.time.Duration;\n+import java.util.Collections;\n+import org.apache.kafka.clients.consumer.ConsumerRecord;\n+import org.apache.kafka.clients.consumer.ConsumerRecords;\n+import org.apache.kafka.clients.consumer.KafkaConsumer;\n+import org.apache.kafka.common.TopicPartition;\n import org.slf4j.Logger;\n import org.slf4j.LoggerFactory;\n import pl.allegro.tech.hermes.common.kafka.KafkaTopic;\n-import pl.allegro.tech.hermes.common.kafka.SimpleConsumerPool;\n+import pl.allegro.tech.hermes.common.kafka.KafkaConsumerPool;\n import pl.allegro.tech.hermes.management.domain.topic.SingleMessageReaderException;\n \n-import java.nio.ByteBuffer;\n-\n import static java.lang.String.format;\n \n public class KafkaRawMessageReader {\n \n     private static final Logger logger = LoggerFactory.getLogger(KafkaRawMessageReader.class);\n \n-    private final SimpleConsumerPool simpleConsumerPool;\n+    private final KafkaConsumerPool consumerPool;\n+    private final int pollTimeoutMillis;\n \n-    public KafkaRawMessageReader(SimpleConsumerPool simpleConsumerPool) {\n-        this.simpleConsumerPool = simpleConsumerPool;\n+    public KafkaRawMessageReader(KafkaConsumerPool consumerPool,\n+                                 int pollTimeoutMillis) {\n+        this.consumerPool = consumerPool;\n+        this.pollTimeoutMillis = pollTimeoutMillis;\n     }\n \n-    public byte[] readMessage(KafkaTopic topic, int partition, long offset) {\n-        FetchResponse fetchResponse = fetch(topic, partition, offset);\n-        for (MessageAndOffset messageAndOffset : fetchResponse.messageSet(topic.name().asString(), partition)) {\n-            if (messageAndOffset.offset() == offset) {\n-                return readPayloadAsBytes(messageAndOffset);\n+    byte[] readMessage(KafkaTopic topic, int partition, long offset) {\n+        KafkaConsumer<byte[], byte[]> kafkaConsumer = consumerPool.get(topic, partition);\n+        TopicPartition topicPartition = new TopicPartition(topic.name().asString(), partition);\n+\n+        try {\n+            kafkaConsumer.assign(Collections.singleton(topicPartition));\n+            kafkaConsumer.seek(topicPartition, offset);\n+            ConsumerRecords<byte[], byte[]> records = kafkaConsumer.poll(Duration.ofMillis(pollTimeoutMillis));\n+            for (ConsumerRecord<byte[], byte[]> record : records.records(topicPartition)) {\n+                if (record.offset() == offset) {\n+                    return record.value();\n+                }\n+                logger.info(\"Found an old offset: {} Expecting: {}\", record.offset(), offset);\n             }\n-            logger.info(\"Found an old offset: {} Expecting: {}\", messageAndOffset.offset(), offset);\n+            throw messageNotFoundException(topic, partition, offset);\n+        } catch (Exception e) {\n+            throw pollingException(topic, partition, offset, e);\n         }\n-        throw messageReaderException(topic, partition, offset, fetchResponse, \"Cannot find message\");\n     }\n \n-    private byte[] readPayloadAsBytes(MessageAndOffset messageAndOffset) {\n-        ByteBuffer payload = messageAndOffset.message().payload();\n-        byte[] bytes = new byte[payload.limit()];\n-        payload.get(bytes);\n-        return bytes;\n+    private static SingleMessageReaderException messageNotFoundException(KafkaTopic topic, int partition, long offset) {\n+        String cause = buildErrorMessage(topic, partition, offset, \"Cannot find message\");\n+        logger.error(cause);\n+        return new SingleMessageReaderException(cause);\n     }\n \n-    private FetchResponse fetch(KafkaTopic topic, int partition, long offset) {\n-        SimpleConsumer simpleConsumer = simpleConsumerPool.get(topic, partition);\n-        FetchResponse fetchResponse = simpleConsumer.fetch(new FetchRequestBuilder()\n-                .clientId(simpleConsumer.clientId())\n-                .addFetch(topic.name().asString(), partition, offset, simpleConsumerPool.getBufferSize())\n-                .build()\n-        );\n-        if (fetchResponse.hasError()) {\n-            throw messageReaderException(topic, partition, offset, fetchResponse, \"Cannot read offset\");\n-        }\n-        return fetchResponse;\n+    private static SingleMessageReaderException pollingException(KafkaTopic topic, int partition, long offset, Throwable throwable) {\n+        String cause = buildErrorMessage(topic, partition, offset, \"Error during polling kafka message\");\n+        logger.error(cause, throwable);\n+        return new SingleMessageReaderException(cause, throwable);\n     }\n \n-    private SingleMessageReaderException messageReaderException(KafkaTopic topic, int partition, long offset, FetchResponse response, String message) {\n-        String cause = message + format(\"[offset %d, kafka_topic %s, partition %d, kafka_response_code: %d]\",\n-                offset, topic.name().asString(), partition, response.errorCode(topic.name().asString(), partition));\n-        logger.error(cause);\n-        return new SingleMessageReaderException(cause);\n+    private static String buildErrorMessage(KafkaTopic topic, int partition, long offset, String message) {\n+        return message + format(\"[offset %d, kafka_topic %s, partition %d]\",","path":"hermes-management/src/main/java/pl/allegro/tech/hermes/management/infrastructure/kafka/service/KafkaRawMessageReader.java","position":101,"original_position":101,"commit_id":"f9559e9bdaa55088330a2057cf37bd4e7bf5c437","original_commit_id":"f9559e9bdaa55088330a2057cf37bd4e7bf5c437","user":{"login":"cristaloleg","id":366002,"node_id":"MDQ6VXNlcjM2NjAwMg==","avatar_url":"https://avatars1.githubusercontent.com/u/366002?v=4","gravatar_id":"","url":"https://api.github.com/users/cristaloleg","html_url":"https://github.com/cristaloleg","followers_url":"https://api.github.com/users/cristaloleg/followers","following_url":"https://api.github.com/users/cristaloleg/following{/other_user}","gists_url":"https://api.github.com/users/cristaloleg/gists{/gist_id}","starred_url":"https://api.github.com/users/cristaloleg/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cristaloleg/subscriptions","organizations_url":"https://api.github.com/users/cristaloleg/orgs","repos_url":"https://api.github.com/users/cristaloleg/repos","events_url":"https://api.github.com/users/cristaloleg/events{/privacy}","received_events_url":"https://api.github.com/users/cristaloleg/received_events","type":"User","site_admin":false},"body":"`format(\"%s [offset %d, kafka_topic %s, partition %d]\", message, offset, topic.name().asString(), partition);`\r\nWDYT?","created_at":"2018-10-24T09:27:08Z","updated_at":"2018-10-24T09:38:28Z","html_url":"https://github.com/allegro/hermes/pull/914#discussion_r227710116","pull_request_url":"https://api.github.com/repos/allegro/hermes/pulls/914","author_association":"MEMBER","_links":{"self":{"href":"https://api.github.com/repos/allegro/hermes/pulls/comments/227710116"},"html":{"href":"https://github.com/allegro/hermes/pull/914#discussion_r227710116"},"pull_request":{"href":"https://api.github.com/repos/allegro/hermes/pulls/914"}}},"pull_request":{"url":"https://api.github.com/repos/allegro/hermes/pulls/914","id":218982605,"node_id":"MDExOlB1bGxSZXF1ZXN0MjE4OTgyNjA1","html_url":"https://github.com/allegro/hermes/pull/914","diff_url":"https://github.com/allegro/hermes/pull/914.diff","patch_url":"https://github.com/allegro/hermes/pull/914.patch","issue_url":"https://api.github.com/repos/allegro/hermes/issues/914","number":914,"state":"open","locked":false,"title":"Upgrading kafka to version 2.0","user":{"login":"faderskd","id":11329819,"node_id":"MDQ6VXNlcjExMzI5ODE5","avatar_url":"https://avatars2.githubusercontent.com/u/11329819?v=4","gravatar_id":"","url":"https://api.github.com/users/faderskd","html_url":"https://github.com/faderskd","followers_url":"https://api.github.com/users/faderskd/followers","following_url":"https://api.github.com/users/faderskd/following{/other_user}","gists_url":"https://api.github.com/users/faderskd/gists{/gist_id}","starred_url":"https://api.github.com/users/faderskd/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/faderskd/subscriptions","organizations_url":"https://api.github.com/users/faderskd/orgs","repos_url":"https://api.github.com/users/faderskd/repos","events_url":"https://api.github.com/users/faderskd/events{/privacy}","received_events_url":"https://api.github.com/users/faderskd/received_events","type":"User","site_admin":false},"body":"Very basic implementation for upgrading Kafka to version 2.0","created_at":"2018-09-28T14:14:43Z","updated_at":"2018-10-24T09:38:28Z","closed_at":null,"merged_at":null,"merge_commit_sha":"94b905dcd0211ff5c8f870b78d747a5eb8955e4c","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/allegro/hermes/pulls/914/commits","review_comments_url":"https://api.github.com/repos/allegro/hermes/pulls/914/comments","review_comment_url":"https://api.github.com/repos/allegro/hermes/pulls/comments{/number}","comments_url":"https://api.github.com/repos/allegro/hermes/issues/914/comments","statuses_url":"https://api.github.com/repos/allegro/hermes/statuses/f9559e9bdaa55088330a2057cf37bd4e7bf5c437","head":{"label":"allegro:feature/switching-to-kafka2","ref":"feature/switching-to-kafka2","sha":"f9559e9bdaa55088330a2057cf37bd4e7bf5c437","user":{"login":"allegro","id":562236,"node_id":"MDEyOk9yZ2FuaXphdGlvbjU2MjIzNg==","avatar_url":"https://avatars3.githubusercontent.com/u/562236?v=4","gravatar_id":"","url":"https://api.github.com/users/allegro","html_url":"https://github.com/allegro","followers_url":"https://api.github.com/users/allegro/followers","following_url":"https://api.github.com/users/allegro/following{/other_user}","gists_url":"https://api.github.com/users/allegro/gists{/gist_id}","starred_url":"https://api.github.com/users/allegro/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/allegro/subscriptions","organizations_url":"https://api.github.com/users/allegro/orgs","repos_url":"https://api.github.com/users/allegro/repos","events_url":"https://api.github.com/users/allegro/events{/privacy}","received_events_url":"https://api.github.com/users/allegro/received_events","type":"Organization","site_admin":false},"repo":{"id":35662043,"node_id":"MDEwOlJlcG9zaXRvcnkzNTY2MjA0Mw==","name":"hermes","full_name":"allegro/hermes","private":false,"owner":{"login":"allegro","id":562236,"node_id":"MDEyOk9yZ2FuaXphdGlvbjU2MjIzNg==","avatar_url":"https://avatars3.githubusercontent.com/u/562236?v=4","gravatar_id":"","url":"https://api.github.com/users/allegro","html_url":"https://github.com/allegro","followers_url":"https://api.github.com/users/allegro/followers","following_url":"https://api.github.com/users/allegro/following{/other_user}","gists_url":"https://api.github.com/users/allegro/gists{/gist_id}","starred_url":"https://api.github.com/users/allegro/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/allegro/subscriptions","organizations_url":"https://api.github.com/users/allegro/orgs","repos_url":"https://api.github.com/users/allegro/repos","events_url":"https://api.github.com/users/allegro/events{/privacy}","received_events_url":"https://api.github.com/users/allegro/received_events","type":"Organization","site_admin":false},"html_url":"https://github.com/allegro/hermes","description":"Fast and reliable message broker built on top of Kafka.","fork":false,"url":"https://api.github.com/repos/allegro/hermes","forks_url":"https://api.github.com/repos/allegro/hermes/forks","keys_url":"https://api.github.com/repos/allegro/hermes/keys{/key_id}","collaborators_url":"https://api.github.com/repos/allegro/hermes/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/allegro/hermes/teams","hooks_url":"https://api.github.com/repos/allegro/hermes/hooks","issue_events_url":"https://api.github.com/repos/allegro/hermes/issues/events{/number}","events_url":"https://api.github.com/repos/allegro/hermes/events","assignees_url":"https://api.github.com/repos/allegro/hermes/assignees{/user}","branches_url":"https://api.github.com/repos/allegro/hermes/branches{/branch}","tags_url":"https://api.github.com/repos/allegro/hermes/tags","blobs_url":"https://api.github.com/repos/allegro/hermes/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/allegro/hermes/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/allegro/hermes/git/refs{/sha}","trees_url":"https://api.github.com/repos/allegro/hermes/git/trees{/sha}","statuses_url":"https://api.github.com/repos/allegro/hermes/statuses/{sha}","languages_url":"https://api.github.com/repos/allegro/hermes/languages","stargazers_url":"https://api.github.com/repos/allegro/hermes/stargazers","contributors_url":"https://api.github.com/repos/allegro/hermes/contributors","subscribers_url":"https://api.github.com/repos/allegro/hermes/subscribers","subscription_url":"https://api.github.com/repos/allegro/hermes/subscription","commits_url":"https://api.github.com/repos/allegro/hermes/commits{/sha}","git_commits_url":"https://api.github.com/repos/allegro/hermes/git/commits{/sha}","comments_url":"https://api.github.com/repos/allegro/hermes/comments{/number}","issue_comment_url":"https://api.github.com/repos/allegro/hermes/issues/comments{/number}","contents_url":"https://api.github.com/repos/allegro/hermes/contents/{+path}","compare_url":"https://api.github.com/repos/allegro/hermes/compare/{base}...{head}","merges_url":"https://api.github.com/repos/allegro/hermes/merges","archive_url":"https://api.github.com/repos/allegro/hermes/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/allegro/hermes/downloads","issues_url":"https://api.github.com/repos/allegro/hermes/issues{/number}","pulls_url":"https://api.github.com/repos/allegro/hermes/pulls{/number}","milestones_url":"https://api.github.com/repos/allegro/hermes/milestones{/number}","notifications_url":"https://api.github.com/repos/allegro/hermes/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/allegro/hermes/labels{/name}","releases_url":"https://api.github.com/repos/allegro/hermes/releases{/id}","deployments_url":"https://api.github.com/repos/allegro/hermes/deployments","created_at":"2015-05-15T08:11:16Z","updated_at":"2018-10-24T04:10:37Z","pushed_at":"2018-10-23T08:00:02Z","git_url":"git://github.com/allegro/hermes.git","ssh_url":"git@github.com:allegro/hermes.git","clone_url":"https://github.com/allegro/hermes.git","svn_url":"https://github.com/allegro/hermes","homepage":"http://hermes.allegro.tech","size":6336,"stargazers_count":382,"watchers_count":382,"language":"Java","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":92,"mirror_url":null,"archived":false,"open_issues_count":72,"license":null,"forks":92,"open_issues":72,"watchers":382,"default_branch":"master"}},"base":{"label":"allegro:master","ref":"master","sha":"60c57bc51f6c1bed2ec063e6405b4391d341b05e","user":{"login":"allegro","id":562236,"node_id":"MDEyOk9yZ2FuaXphdGlvbjU2MjIzNg==","avatar_url":"https://avatars3.githubusercontent.com/u/562236?v=4","gravatar_id":"","url":"https://api.github.com/users/allegro","html_url":"https://github.com/allegro","followers_url":"https://api.github.com/users/allegro/followers","following_url":"https://api.github.com/users/allegro/following{/other_user}","gists_url":"https://api.github.com/users/allegro/gists{/gist_id}","starred_url":"https://api.github.com/users/allegro/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/allegro/subscriptions","organizations_url":"https://api.github.com/users/allegro/orgs","repos_url":"https://api.github.com/users/allegro/repos","events_url":"https://api.github.com/users/allegro/events{/privacy}","received_events_url":"https://api.github.com/users/allegro/received_events","type":"Organization","site_admin":false},"repo":{"id":35662043,"node_id":"MDEwOlJlcG9zaXRvcnkzNTY2MjA0Mw==","name":"hermes","full_name":"allegro/hermes","private":false,"owner":{"login":"allegro","id":562236,"node_id":"MDEyOk9yZ2FuaXphdGlvbjU2MjIzNg==","avatar_url":"https://avatars3.githubusercontent.com/u/562236?v=4","gravatar_id":"","url":"https://api.github.com/users/allegro","html_url":"https://github.com/allegro","followers_url":"https://api.github.com/users/allegro/followers","following_url":"https://api.github.com/users/allegro/following{/other_user}","gists_url":"https://api.github.com/users/allegro/gists{/gist_id}","starred_url":"https://api.github.com/users/allegro/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/allegro/subscriptions","organizations_url":"https://api.github.com/users/allegro/orgs","repos_url":"https://api.github.com/users/allegro/repos","events_url":"https://api.github.com/users/allegro/events{/privacy}","received_events_url":"https://api.github.com/users/allegro/received_events","type":"Organization","site_admin":false},"html_url":"https://github.com/allegro/hermes","description":"Fast and reliable message broker built on top of Kafka.","fork":false,"url":"https://api.github.com/repos/allegro/hermes","forks_url":"https://api.github.com/repos/allegro/hermes/forks","keys_url":"https://api.github.com/repos/allegro/hermes/keys{/key_id}","collaborators_url":"https://api.github.com/repos/allegro/hermes/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/allegro/hermes/teams","hooks_url":"https://api.github.com/repos/allegro/hermes/hooks","issue_events_url":"https://api.github.com/repos/allegro/hermes/issues/events{/number}","events_url":"https://api.github.com/repos/allegro/hermes/events","assignees_url":"https://api.github.com/repos/allegro/hermes/assignees{/user}","branches_url":"https://api.github.com/repos/allegro/hermes/branches{/branch}","tags_url":"https://api.github.com/repos/allegro/hermes/tags","blobs_url":"https://api.github.com/repos/allegro/hermes/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/allegro/hermes/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/allegro/hermes/git/refs{/sha}","trees_url":"https://api.github.com/repos/allegro/hermes/git/trees{/sha}","statuses_url":"https://api.github.com/repos/allegro/hermes/statuses/{sha}","languages_url":"https://api.github.com/repos/allegro/hermes/languages","stargazers_url":"https://api.github.com/repos/allegro/hermes/stargazers","contributors_url":"https://api.github.com/repos/allegro/hermes/contributors","subscribers_url":"https://api.github.com/repos/allegro/hermes/subscribers","subscription_url":"https://api.github.com/repos/allegro/hermes/subscription","commits_url":"https://api.github.com/repos/allegro/hermes/commits{/sha}","git_commits_url":"https://api.github.com/repos/allegro/hermes/git/commits{/sha}","comments_url":"https://api.github.com/repos/allegro/hermes/comments{/number}","issue_comment_url":"https://api.github.com/repos/allegro/hermes/issues/comments{/number}","contents_url":"https://api.github.com/repos/allegro/hermes/contents/{+path}","compare_url":"https://api.github.com/repos/allegro/hermes/compare/{base}...{head}","merges_url":"https://api.github.com/repos/allegro/hermes/merges","archive_url":"https://api.github.com/repos/allegro/hermes/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/allegro/hermes/downloads","issues_url":"https://api.github.com/repos/allegro/hermes/issues{/number}","pulls_url":"https://api.github.com/repos/allegro/hermes/pulls{/number}","milestones_url":"https://api.github.com/repos/allegro/hermes/milestones{/number}","notifications_url":"https://api.github.com/repos/allegro/hermes/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/allegro/hermes/labels{/name}","releases_url":"https://api.github.com/repos/allegro/hermes/releases{/id}","deployments_url":"https://api.github.com/repos/allegro/hermes/deployments","created_at":"2015-05-15T08:11:16Z","updated_at":"2018-10-24T04:10:37Z","pushed_at":"2018-10-23T08:00:02Z","git_url":"git://github.com/allegro/hermes.git","ssh_url":"git@github.com:allegro/hermes.git","clone_url":"https://github.com/allegro/hermes.git","svn_url":"https://github.com/allegro/hermes","homepage":"http://hermes.allegro.tech","size":6336,"stargazers_count":382,"watchers_count":382,"language":"Java","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":92,"mirror_url":null,"archived":false,"open_issues_count":72,"license":null,"forks":92,"open_issues":72,"watchers":382,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/allegro/hermes/pulls/914"},"html":{"href":"https://github.com/allegro/hermes/pull/914"},"issue":{"href":"https://api.github.com/repos/allegro/hermes/issues/914"},"comments":{"href":"https://api.github.com/repos/allegro/hermes/issues/914/comments"},"review_comments":{"href":"https://api.github.com/repos/allegro/hermes/pulls/914/comments"},"review_comment":{"href":"https://api.github.com/repos/allegro/hermes/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/allegro/hermes/pulls/914/commits"},"statuses":{"href":"https://api.github.com/repos/allegro/hermes/statuses/f9559e9bdaa55088330a2057cf37bd4e7bf5c437"}},"author_association":"CONTRIBUTOR"}}
{"actor":{"display_login":"cristaloleg"}}