{"action":"created","comment":{"url":"https://api.github.com/repos/apache/flink/pulls/comments/60731184","id":60731184,"diff_hunk":"@@ -0,0 +1,481 @@\n+/*\n+ * Licensed to the Apache Software Foundation (ASF) under one or more\n+ * contributor license agreements.  See the NOTICE file distributed with\n+ * this work for additional information regarding copyright ownership.\n+ * The ASF licenses this file to You under the Apache License, Version 2.0\n+ * (the \"License\"); you may not use this file except in compliance with\n+ * the License.  You may obtain a copy of the License at\n+ *\n+ *    http://www.apache.org/licenses/LICENSE-2.0\n+ *\n+ * Unless required by applicable law or agreed to in writing, software\n+ * distributed under the License is distributed on an \"AS IS\" BASIS,\n+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+ * See the License for the specific language governing permissions and\n+ * limitations under the License.\n+ */\n+\n+package org.apache.flink.streaming.connectors.kinesis;\n+\n+import com.amazonaws.regions.Regions;\n+import org.apache.flink.api.common.typeinfo.TypeInformation;\n+import org.apache.flink.api.java.typeutils.ResultTypeQueryable;\n+import org.apache.flink.configuration.Configuration;\n+import org.apache.flink.runtime.state.CheckpointListener;\n+import org.apache.flink.streaming.api.checkpoint.CheckpointedAsynchronously;\n+import org.apache.flink.streaming.api.functions.source.RichParallelSourceFunction;\n+import org.apache.flink.streaming.api.watermark.Watermark;\n+import org.apache.flink.streaming.connectors.kinesis.config.KinesisConfigConstants;\n+import org.apache.flink.streaming.connectors.kinesis.internals.KinesisDataFetcher;\n+import org.apache.flink.streaming.connectors.kinesis.config.CredentialProviderType;\n+import org.apache.flink.streaming.connectors.kinesis.config.InitialPosition;\n+import org.apache.flink.streaming.connectors.kinesis.model.KinesisStreamShard;\n+import org.apache.flink.streaming.connectors.kinesis.model.SentinelSequenceNumber;\n+import org.apache.flink.streaming.connectors.kinesis.proxy.KinesisProxy;\n+import org.apache.flink.streaming.connectors.kinesis.serialization.KinesisDeserializationSchemaWrapper;\n+import org.apache.flink.streaming.connectors.kinesis.serialization.KinesisDeserializationSchema;\n+import org.apache.flink.streaming.connectors.kinesis.util.AWSUtil;\n+import org.apache.flink.streaming.util.serialization.DeserializationSchema;\n+import org.slf4j.Logger;\n+import org.slf4j.LoggerFactory;\n+\n+import java.io.IOException;\n+import java.util.Collections;\n+import java.util.Map;\n+import java.util.HashMap;\n+import java.util.List;\n+import java.util.ArrayList;\n+import java.util.Properties;\n+\n+import static com.google.common.base.Preconditions.checkArgument;\n+import static com.google.common.base.Preconditions.checkNotNull;\n+\n+/**\n+ * The Flink Kinesis Consumer is a parallel streaming data source that pulls data from multiple AWS Kinesis streams\n+ * within the same AWS service region. Each instance of the consumer is responsible for fetching data records from\n+ * one or more Kinesis shards.\n+ *\n+ * <p>To leverage Flink's checkpointing mechanics for exactly-once streaming processing guarantees, the Flink Kinesis\n+ * consumer is implemented with the AWS Java SDK, instead of the officially recommended AWS Kinesis Client Library, for\n+ * low-level control on the management of stream state. The Flink Kinesis Connector also supports setting the initial\n+ * starting points of Kinesis streams, namely TRIM_HORIZON and LATEST:</p>\n+ *\n+ * <ul>\n+ * </ul>\n+ *\n+ * <p><b>NOTE:</b> The current implementation does not correctly handle resharding of AWS Kinesis streams.</p>\n+ * <p><b>NOTE:</b> Since Kinesis and Kafka share many common abstractions, the implementation is heavily based on\n+ * the Flink Kafka Consumer.</p>\n+ *\n+ * @param <T> the type of data emitted\n+ */\n+public class FlinkKinesisConsumer<T> extends RichParallelSourceFunction<T>\n+\timplements CheckpointListener, CheckpointedAsynchronously<HashMap<KinesisStreamShard, String>>, ResultTypeQueryable<T> {\n+\n+\tprivate static final long serialVersionUID = 1L;\n+\n+\tprivate static final Logger LOG = LoggerFactory.getLogger(FlinkKinesisConsumer.class);\n+\n+\t// ------------------------------------------------------------------------\n+\t//  Consumer properties\n+\t// ------------------------------------------------------------------------\n+\n+\t/** The complete list of shards */\n+\tprivate final List<KinesisStreamShard> shards;\n+\n+\t/** Properties to parametrize settings such as AWS service region, initial position in stream,\n+\t * shard list retrieval behaviours, etc */\n+\tprivate final Properties configProps;\n+\n+\t/** User supplied deseriliazation schema to convert Kinesis byte messages to Flink objects */\n+\tprivate final KinesisDeserializationSchema<T> deserializer;\n+\n+\t// ------------------------------------------------------------------------\n+\t//  Runtime state\n+\t// ------------------------------------------------------------------------\n+\n+\t/** Per-task fetcher for Kinesis data records, where each fetcher pulls data from one or more Kinesis shards */\n+\tprivate transient KinesisDataFetcher fetcher;\n+\n+\t/** The sequence numbers of the last fetched data records from Kinesis by this task */\n+\tprivate transient HashMap<KinesisStreamShard, String> lastSequenceNums;\n+\n+\t/** The sequence numbers to restore to upon restore from failure */\n+\tprivate transient HashMap<KinesisStreamShard, String> sequenceNumsToRestore;\n+\n+\tprivate volatile boolean hasAssignedShards;\n+\n+\tprivate volatile boolean running = true;\n+\n+\n+\t// ------------------------------------------------------------------------\n+\t//  Constructors\n+\t// ------------------------------------------------------------------------\n+\n+\t/**\n+\t * Creates a new Flink Kinesis Consumer.\n+\t *\n+\t * <p>The AWS credentials to be used, AWS region of the Kinesis streams, initial position to start streaming\n+\t * from are configured with a {@link Properties} instance.</p>\n+\t *\n+\t * @param stream\n+\t *           The single AWS Kinesis stream to read from.\n+\t * @param deserializer\n+\t *           The deserializer used to convert raw bytes of Kinesis records to Java objects (without key).\n+\t * @param configProps\n+\t *           The properties used to configure AWS credentials, AWS region, and initial starting position.\n+\t */\n+\tpublic FlinkKinesisConsumer(String stream, DeserializationSchema<T> deserializer, Properties configProps) {\n+\t\tthis(stream, new KinesisDeserializationSchemaWrapper<T>(deserializer), configProps);\n+\t}\n+\n+\t/**\n+\t * Creates a new Flink Kinesis Consumer.\n+\t *\n+\t * <p>The AWS credentials to be used, AWS region of the Kinesis streams, initial position to start streaming\n+\t * from are configured with a {@link Properties} instance.</p>\n+\t *\n+\t * @param stream\n+\t *           The single AWS Kinesis stream to read from.\n+\t * @param deserializer\n+\t *           The keyed deserializer used to convert raw bytes of Kinesis records to Java objects.\n+\t * @param configProps\n+\t *           The properties used to configure AWS credentials, AWS region, and initial starting position.\n+\t */\n+\tpublic FlinkKinesisConsumer(String stream, KinesisDeserializationSchema<T> deserializer, Properties configProps) {\n+\t\tthis(Collections.singletonList(stream), deserializer, configProps);\n+\t}\n+\n+\t/**\n+\t * Creates a new Flink Kinesis Consumer.\n+\t *\n+\t * <p>The AWS credentials to be used, AWS region of the Kinesis streams, initial position to start streaming\n+\t * from are configured with a {@link Properties} instance.</p>\n+\t *\n+\t * @param streams\n+\t *           The AWS Kinesis streams to read from.\n+\t * @param deserializer\n+\t *           The keyed deserializer used to convert raw bytes of Kinesis records to Java objects.\n+\t * @param configProps\n+\t *           The properties used to configure AWS credentials, AWS region, and initial starting position.\n+\t */\n+\tpublic FlinkKinesisConsumer(List<String> streams, KinesisDeserializationSchema<T> deserializer, Properties configProps) {\n+\t\tcheckNotNull(streams, \"streams can not be null\");\n+\n+\t\tthis.configProps = checkNotNull(configProps, \"configProps can not be null\");\n+\n+\t\t// check the configuration properties for any conflicting settings\n+\t\tvalidatePropertiesConfig(this.configProps);\n+\n+\t\tthis.deserializer = checkNotNull(deserializer, \"deserializer can not be null\");\n+\n+\t\tthis.shards = new KinesisProxy(configProps).getShardList(streams);\n+\t\tif (shards.size() == 0) {\n+\t\t\tthrow new RuntimeException(\"Unable to retrieve any shards for the requested streams \" + streams.toString() + \".\");\n+\t\t}\n+\n+\t\tif (LOG.isInfoEnabled()) {\n+\t\t\tMap<String, Integer> shardCountPerStream = new HashMap<>();\n+\t\t\tfor (KinesisStreamShard shard : shards) {\n+\t\t\t\tInteger shardCount = shardCountPerStream.get(shard.getStreamName());\n+\t\t\t\tif (shardCount == null) {\n+\t\t\t\t\tshardCount = 1;\n+\t\t\t\t} else {\n+\t\t\t\t\tshardCount++;\n+\t\t\t\t}\n+\t\t\t\tshardCountPerStream.put(shard.getStreamName(), shardCount);\n+\t\t\t}\n+\t\t\tStringBuilder sb = new StringBuilder();\n+\t\t\tfor (Map.Entry<String,Integer> streamAndShardCountPair : shardCountPerStream.entrySet()) {\n+\t\t\t\tsb.append(streamAndShardCountPair.getKey()).append(\" (\").append(streamAndShardCountPair.getValue()).append(\"), \");\n+\t\t\t}\n+\t\t\tLOG.info(\"Flink Kinesis Consumer is going to read the following streams (with number of shards): {}\", sb.toString());\n+\t\t}\n+\t}\n+\n+\t// ------------------------------------------------------------------------\n+\t//  Source life cycle\n+\t// ------------------------------------------------------------------------\n+\n+\t@Override\n+\tpublic void open(Configuration parameters) throws Exception {\n+\t\tsuper.open(parameters);\n+\n+\t\tfinal int numFlinkConsumerTasks = getRuntimeContext().getNumberOfParallelSubtasks();\n+\t\tfinal int thisConsumerTaskIndex = getRuntimeContext().getIndexOfThisSubtask();\n+\n+\t\t// pick which shards this consumer task works on, in a round-robin fashion\n+\t\tList<KinesisStreamShard> assignedShards = assignShards(this.shards, numFlinkConsumerTasks, thisConsumerTaskIndex);\n+\n+\t\t// if there are no shards assigned to this consumer task, return without doing anything.\n+\t\tif (assignedShards.isEmpty()) {\n+\t\t\tLOG.info(\"Consumer task {} has no shards assigned to it\", thisConsumerTaskIndex);\n+\t\t\thasAssignedShards = false;\n+\t\t\treturn;\n+\t\t} else {\n+\t\t\thasAssignedShards = true;\n+\t\t}\n+\n+\t\tif (LOG.isInfoEnabled()) {\n+\t\t\tStringBuilder sb = new StringBuilder();\n+\t\t\tfor (KinesisStreamShard shard : assignedShards) {\n+\t\t\t\tsb.append(shard.getStreamName()).append(\":\").append(shard.getShardId()).append(\", \");\n+\t\t\t}\n+\t\t\tLOG.info(\"Consumer task {} will read shards {} out of a total of {} shards\",\n+\t\t\t\tthisConsumerTaskIndex, sb.toString(), this.shards.size());\n+\t\t}\n+\n+\t\tfetcher = new KinesisDataFetcher(assignedShards, configProps, getRuntimeContext().getTaskName());\n+\n+\t\t// restore to the last known sequence numbers from the latest complete snapshot\n+\t\tif (sequenceNumsToRestore != null) {\n+\t\t\tif (LOG.isInfoEnabled()) {\n+\t\t\t\tLOG.info(\"Consumer task {} is restoring sequence numbers from previous checkpointed state\", thisConsumerTaskIndex);\n+\t\t\t}\n+\n+\t\t\tfor (Map.Entry<KinesisStreamShard, String> restoreSequenceNum : sequenceNumsToRestore.entrySet()) {\n+\t\t\t\t// advance the corresponding shard to the last known sequence number\n+\t\t\t\tfetcher.advanceSequenceNumberTo(restoreSequenceNum.getKey(), restoreSequenceNum.getValue());\n+\t\t\t}\n+\n+\t\t\tif (LOG.isInfoEnabled()) {\n+\t\t\t\tStringBuilder sb = new StringBuilder();\n+\t\t\t\tfor (Map.Entry<KinesisStreamShard, String> restoreSequenceNo : sequenceNumsToRestore.entrySet()) {\n+\t\t\t\t\tKinesisStreamShard shard = restoreSequenceNo.getKey();\n+\t\t\t\t\tsb.append(shard.getStreamName()).append(\":\").append(shard.getShardId())\n+\t\t\t\t\t\t.append(\" -> \").append(restoreSequenceNo.getValue()).append(\", \");\n+\t\t\t\t}\n+\t\t\t\tLOG.info(\"Advanced the starting sequence numbers of consumer task {}: {}\", thisConsumerTaskIndex, sb.toString());\n+\t\t\t}\n+\n+\t\t\t// initialize sequence numbers with restored state\n+\t\t\tlastSequenceNums = sequenceNumsToRestore;\n+\t\t\tsequenceNumsToRestore = null;\n+\t\t} else {\n+\t\t\t// start fresh with empty sequence numbers if there are no snapshots to restore from.\n+\t\t\tlastSequenceNums = new HashMap<>();\n+\n+\t\t\t// advance all assigned shards of this consumer task to either the earliest or latest sequence number,\n+\t\t\t// depending on the properties configuration (default is to set to latest sequence number).\n+\t\t\tInitialPosition initialPosition = InitialPosition.valueOf(configProps.getProperty(\n+\t\t\t\tKinesisConfigConstants.CONFIG_STREAM_INIT_POSITION_TYPE, InitialPosition.LATEST.toString()));\n+\n+\t\t\tSentinelSequenceNumber sentinelSequenceNum;\n+\t\t\tswitch (initialPosition) {\n+\t\t\t\tcase TRIM_HORIZON:\n+\t\t\t\t\tsentinelSequenceNum = SentinelSequenceNumber.SENTINEL_EARLIEST_SEQUENCE_NUM;\n+\t\t\t\t\tbreak;\n+\t\t\t\tcase LATEST:\n+\t\t\t\tdefault:\n+\t\t\t\t\tsentinelSequenceNum = SentinelSequenceNumber.SENTINEL_LATEST_SEQUENCE_NUM;\n+\t\t\t}\n+\n+\t\t\tfor (KinesisStreamShard assignedShard : assignedShards) {\n+\t\t\t\tfetcher.advanceSequenceNumberTo(assignedShard, sentinelSequenceNum.toString());\n+\t\t\t}\n+\n+\t\t\tif (LOG.isInfoEnabled()) {\n+\t\t\t\tStringBuilder sb = new StringBuilder();\n+\t\t\t\tfor (KinesisStreamShard assignedShard : assignedShards) {\n+\t\t\t\t\tsb.append(assignedShard.getStreamName()).append(\":\").append(assignedShard.getShardId())\n+\t\t\t\t\t\t.append(\" -> \").append(sentinelSequenceNum.toString()).append(\", \");\n+\t\t\t\t}\n+\t\t\t\tLOG.info(\"Advanced the starting sequence numbers of consumer task {}: {}\", thisConsumerTaskIndex, sb.toString());\n+\t\t\t}\n+\t\t}\n+\t}\n+\n+\t@Override\n+\tpublic void run(SourceContext<T> sourceContext) throws Exception {\n+\t\tif (hasAssignedShards) {\n+\t\t\tfetcher.run(sourceContext, deserializer, lastSequenceNums);\n+\t\t} else {\n+\t\t\t// this source never completes because there is no assigned shards,\n+\t\t\t// so emit a Long.MAX_VALUE watermark to no block watermark forwarding\n+\t\t\tsourceContext.emitWatermark(new Watermark(Long.MAX_VALUE));\n+\n+\t\t\tfinal Object waitLock = new Object();\n+\t\t\twhile (running) {\n+\t\t\t\ttry {\n+\t\t\t\t\tsynchronized (waitLock) {\n+\t\t\t\t\t\twaitLock.wait();\n+\t\t\t\t\t}\n+\t\t\t\t} catch (InterruptedException e) {\n+\t\t\t\t\t// do nothing\n+\t\t\t\t}\n+\t\t\t}\n+\t\t}\n+\n+\t\tsourceContext.close();\n+\t}\n+\n+\t@Override\n+\tpublic void cancel() {\n+\t\trunning = false;\n+\n+\t\t// interrupt the fetcher of any work\n+\t\tKinesisDataFetcher fetcher = this.fetcher;\n+\t\tthis.fetcher = null;\n+\t\tif (fetcher != null) {\n+\t\t\ttry {\n+\t\t\t\tfetcher.close();\n+\t\t\t} catch (IOException e) {\n+\t\t\t\tLOG.warn(\"Error while closing Kinesis data fetcher\", e);\n+\t\t\t}\n+\t\t}\n+\t}\n+\n+\t@Override\n+\tpublic void close() throws Exception {\n+\t\tcancel();\n+\t\tsuper.close();\n+\t}\n+\n+\t@Override\n+\tpublic TypeInformation<T> getProducedType() {\n+\t\treturn deserializer.getProducedType();\n+\t}\n+\n+\t// ------------------------------------------------------------------------\n+\t//  Checkpoint and restore\n+\t// ------------------------------------------------------------------------\n+\n+\t@Override\n+\tpublic HashMap<KinesisStreamShard, String> snapshotState(long checkpointId, long checkpointTimestamp) throws Exception {\n+\t\tif (lastSequenceNums == null) {\n+\t\t\tLOG.debug(\"snapshotState() requested on not yet opened source; returning null.\");\n+\t\t\treturn null;\n+\t\t}\n+\n+\t\tif (!running) {\n+\t\t\tLOG.debug(\"snapshotState() called on closed source; returning null.\");\n+\t\t\treturn null;\n+\t\t}\n+\n+\t\tif (LOG.isDebugEnabled()) {\n+\t\t\tLOG.debug(\"Snapshotting state. ...\");\n+\t\t}\n+\n+\t\t@SuppressWarnings(\"unchecked\")\n+\t\tHashMap<KinesisStreamShard, String> currentSequenceNums = (HashMap<KinesisStreamShard, String>) lastSequenceNums.clone();\n+\t\treturn currentSequenceNums;\n+\t}\n+\n+\t@Override\n+\tpublic void restoreState(HashMap<KinesisStreamShard, String> restoredState) throws Exception {\n+\t\tsequenceNumsToRestore = restoredState;\n+\t}\n+\n+\t@Override\n+\tpublic void notifyCheckpointComplete(long checkpointId) throws Exception {\n+\t\tif (fetcher == null) {\n+\t\t\tLOG.debug(\"notifyCheckpointComplete() called on uninitialized source\");\n+\t\t\treturn;\n+\t\t}\n+\n+\t\tif (!running) {\n+\t\t\tLOG.debug(\"notifyCheckpointComplete() called on closed source\");\n+\t\t\treturn;\n+\t\t}\n+\n+\t\tif (LOG.isDebugEnabled()) {\n+\t\t\tLOG.debug(\"Checkpointing completed for checkpoint id {}\", checkpointId);\n+\t\t}\n+\t}\n+\n+\t// ------------------------------------------------------------------------\n+\t//  Miscellaneous utilities\n+\t// ------------------------------------------------------------------------\n+\n+\t/**\n+\t * Utility function to assign shards to a specific consumer task in a round-robin fashion.\n+\t */\n+\tprotected static List<KinesisStreamShard> assignShards(List<KinesisStreamShard> shards, int numFlinkConsumerTasks, int thisConsumerTaskIndex) {\n+\t\tcheckArgument(numFlinkConsumerTasks > 0);\n+\t\tcheckArgument(thisConsumerTaskIndex < numFlinkConsumerTasks);\n+\n+\t\tList<KinesisStreamShard> closedShards = new ArrayList<>();\n+\t\tList<KinesisStreamShard> openShards = new ArrayList<>();\n+\n+\t\tfor (KinesisStreamShard shard : shards) {\n+\t\t\tif (shard.isClosed()) {\n+\t\t\t\tclosedShards.add(shard);\n+\t\t\t} else {\n+\t\t\t\topenShards.add(shard);\n+\t\t\t}\n+\t\t}\n+\n+\t\tList<KinesisStreamShard> subscribedShards = new ArrayList<>();\n+\n+\t\t// separately round-robin assign open and closed shards so that all tasks have a fair chance of being\n+\t\t// assigned open shards (set of data records in closed shards are bounded)\n+\n+\t\tfor (int i = 0; i < closedShards.size(); i++) {\n+\t\t\tif (i % numFlinkConsumerTasks == thisConsumerTaskIndex) {\n+\t\t\t\tsubscribedShards.add(closedShards.get(i));\n+\t\t\t}\n+\t\t}\n+\n+\t\tfor (int i = 0; i < openShards.size(); i++) {\n+\t\t\tif (i % numFlinkConsumerTasks == thisConsumerTaskIndex) {\n+\t\t\t\tsubscribedShards.add(openShards.get(i));\n+\t\t\t}\n+\t\t}\n+\t\treturn subscribedShards;\n+\t}\n+\n+\t/**\n+\t * Checks that the values specified for config keys in the properties config is recognizable.\n+\t */\n+\tprotected static void validatePropertiesConfig(Properties config) {","path":"flink-streaming-connectors/flink-connector-kinesis/src/main/java/org/apache/flink/streaming/connectors/kinesis/FlinkKinesisConsumer.java","position":430,"original_position":430,"commit_id":"e507555524ac59c91b4b1d010b1c219d4f424aa9","original_commit_id":"e507555524ac59c91b4b1d010b1c219d4f424aa9","user":{"login":"rmetzger","id":89049,"avatar_url":"https://avatars.githubusercontent.com/u/89049?v=3","gravatar_id":"","url":"https://api.github.com/users/rmetzger","html_url":"https://github.com/rmetzger","followers_url":"https://api.github.com/users/rmetzger/followers","following_url":"https://api.github.com/users/rmetzger/following{/other_user}","gists_url":"https://api.github.com/users/rmetzger/gists{/gist_id}","starred_url":"https://api.github.com/users/rmetzger/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/rmetzger/subscriptions","organizations_url":"https://api.github.com/users/rmetzger/orgs","repos_url":"https://api.github.com/users/rmetzger/repos","events_url":"https://api.github.com/users/rmetzger/events{/privacy}","received_events_url":"https://api.github.com/users/rmetzger/received_events","type":"User","site_admin":false},"body":"I think this function should ensure that the access key, secret key and region are set.\r\nOtherwise, the aws sdk will throw hard to understand exceptions","created_at":"2016-04-22T12:34:19Z","updated_at":"2016-04-22T12:34:19Z","html_url":"https://github.com/apache/flink/pull/1911#discussion_r60731184","pull_request_url":"https://api.github.com/repos/apache/flink/pulls/1911","_links":{"self":{"href":"https://api.github.com/repos/apache/flink/pulls/comments/60731184"},"html":{"href":"https://github.com/apache/flink/pull/1911#discussion_r60731184"},"pull_request":{"href":"https://api.github.com/repos/apache/flink/pulls/1911"}}},"pull_request":{"url":"https://api.github.com/repos/apache/flink/pulls/1911","id":66993188,"html_url":"https://github.com/apache/flink/pull/1911","diff_url":"https://github.com/apache/flink/pull/1911.diff","patch_url":"https://github.com/apache/flink/pull/1911.patch","issue_url":"https://api.github.com/repos/apache/flink/issues/1911","number":1911,"state":"open","locked":false,"title":"[FLINK-3229] Flink streaming consumer for AWS Kinesis","user":{"login":"tzulitai","id":5284370,"avatar_url":"https://avatars.githubusercontent.com/u/5284370?v=3","gravatar_id":"","url":"https://api.github.com/users/tzulitai","html_url":"https://github.com/tzulitai","followers_url":"https://api.github.com/users/tzulitai/followers","following_url":"https://api.github.com/users/tzulitai/following{/other_user}","gists_url":"https://api.github.com/users/tzulitai/gists{/gist_id}","starred_url":"https://api.github.com/users/tzulitai/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tzulitai/subscriptions","organizations_url":"https://api.github.com/users/tzulitai/orgs","repos_url":"https://api.github.com/users/tzulitai/repos","events_url":"https://api.github.com/users/tzulitai/events{/privacy}","received_events_url":"https://api.github.com/users/tzulitai/received_events","type":"User","site_admin":false},"body":"I've been using this consumer for a while in off-production environments.\r\nI understand we should have good test coverage for each PR, but since Kinesis is a hosted service, reliable integration tests are hard to pull off. To speed up merging Kinesis connector for the next release, I'm submitting the consumer now for some early reviews.\r\nOn the other hand, since @rmetzger is submitting a separate PR for Kinesis producer, I'd like to postpone writing more tests for the consumer, as well as corresponding modification to the document until both the consumer and producer are in place.","created_at":"2016-04-19T10:17:02Z","updated_at":"2016-04-22T12:34:19Z","closed_at":null,"merged_at":null,"merge_commit_sha":"68d14aa06f3d94ad4956afe6529b7d122928926b","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/apache/flink/pulls/1911/commits","review_comments_url":"https://api.github.com/repos/apache/flink/pulls/1911/comments","review_comment_url":"https://api.github.com/repos/apache/flink/pulls/comments{/number}","comments_url":"https://api.github.com/repos/apache/flink/issues/1911/comments","statuses_url":"https://api.github.com/repos/apache/flink/statuses/e507555524ac59c91b4b1d010b1c219d4f424aa9","head":{"label":"tzulitai:FLINK-3229","ref":"FLINK-3229","sha":"e507555524ac59c91b4b1d010b1c219d4f424aa9","user":{"login":"tzulitai","id":5284370,"avatar_url":"https://avatars.githubusercontent.com/u/5284370?v=3","gravatar_id":"","url":"https://api.github.com/users/tzulitai","html_url":"https://github.com/tzulitai","followers_url":"https://api.github.com/users/tzulitai/followers","following_url":"https://api.github.com/users/tzulitai/following{/other_user}","gists_url":"https://api.github.com/users/tzulitai/gists{/gist_id}","starred_url":"https://api.github.com/users/tzulitai/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tzulitai/subscriptions","organizations_url":"https://api.github.com/users/tzulitai/orgs","repos_url":"https://api.github.com/users/tzulitai/repos","events_url":"https://api.github.com/users/tzulitai/events{/privacy}","received_events_url":"https://api.github.com/users/tzulitai/received_events","type":"User","site_admin":false},"repo":{"id":55886834,"name":"flink","full_name":"tzulitai/flink","owner":{"login":"tzulitai","id":5284370,"avatar_url":"https://avatars.githubusercontent.com/u/5284370?v=3","gravatar_id":"","url":"https://api.github.com/users/tzulitai","html_url":"https://github.com/tzulitai","followers_url":"https://api.github.com/users/tzulitai/followers","following_url":"https://api.github.com/users/tzulitai/following{/other_user}","gists_url":"https://api.github.com/users/tzulitai/gists{/gist_id}","starred_url":"https://api.github.com/users/tzulitai/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tzulitai/subscriptions","organizations_url":"https://api.github.com/users/tzulitai/orgs","repos_url":"https://api.github.com/users/tzulitai/repos","events_url":"https://api.github.com/users/tzulitai/events{/privacy}","received_events_url":"https://api.github.com/users/tzulitai/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/tzulitai/flink","description":"Mirror of Apache Flink","fork":true,"url":"https://api.github.com/repos/tzulitai/flink","forks_url":"https://api.github.com/repos/tzulitai/flink/forks","keys_url":"https://api.github.com/repos/tzulitai/flink/keys{/key_id}","collaborators_url":"https://api.github.com/repos/tzulitai/flink/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/tzulitai/flink/teams","hooks_url":"https://api.github.com/repos/tzulitai/flink/hooks","issue_events_url":"https://api.github.com/repos/tzulitai/flink/issues/events{/number}","events_url":"https://api.github.com/repos/tzulitai/flink/events","assignees_url":"https://api.github.com/repos/tzulitai/flink/assignees{/user}","branches_url":"https://api.github.com/repos/tzulitai/flink/branches{/branch}","tags_url":"https://api.github.com/repos/tzulitai/flink/tags","blobs_url":"https://api.github.com/repos/tzulitai/flink/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/tzulitai/flink/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/tzulitai/flink/git/refs{/sha}","trees_url":"https://api.github.com/repos/tzulitai/flink/git/trees{/sha}","statuses_url":"https://api.github.com/repos/tzulitai/flink/statuses/{sha}","languages_url":"https://api.github.com/repos/tzulitai/flink/languages","stargazers_url":"https://api.github.com/repos/tzulitai/flink/stargazers","contributors_url":"https://api.github.com/repos/tzulitai/flink/contributors","subscribers_url":"https://api.github.com/repos/tzulitai/flink/subscribers","subscription_url":"https://api.github.com/repos/tzulitai/flink/subscription","commits_url":"https://api.github.com/repos/tzulitai/flink/commits{/sha}","git_commits_url":"https://api.github.com/repos/tzulitai/flink/git/commits{/sha}","comments_url":"https://api.github.com/repos/tzulitai/flink/comments{/number}","issue_comment_url":"https://api.github.com/repos/tzulitai/flink/issues/comments{/number}","contents_url":"https://api.github.com/repos/tzulitai/flink/contents/{+path}","compare_url":"https://api.github.com/repos/tzulitai/flink/compare/{base}...{head}","merges_url":"https://api.github.com/repos/tzulitai/flink/merges","archive_url":"https://api.github.com/repos/tzulitai/flink/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/tzulitai/flink/downloads","issues_url":"https://api.github.com/repos/tzulitai/flink/issues{/number}","pulls_url":"https://api.github.com/repos/tzulitai/flink/pulls{/number}","milestones_url":"https://api.github.com/repos/tzulitai/flink/milestones{/number}","notifications_url":"https://api.github.com/repos/tzulitai/flink/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/tzulitai/flink/labels{/name}","releases_url":"https://api.github.com/repos/tzulitai/flink/releases{/id}","deployments_url":"https://api.github.com/repos/tzulitai/flink/deployments","created_at":"2016-04-10T07:41:34Z","updated_at":"2016-04-10T07:41:51Z","pushed_at":"2016-04-21T06:20:03Z","git_url":"git://github.com/tzulitai/flink.git","ssh_url":"git@github.com:tzulitai/flink.git","clone_url":"https://github.com/tzulitai/flink.git","svn_url":"https://github.com/tzulitai/flink","homepage":null,"size":88429,"stargazers_count":0,"watchers_count":0,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":1,"mirror_url":null,"open_issues_count":0,"forks":1,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"apache:master","ref":"master","sha":"409c346814ac8d8fc3498e94b25b6e340be75dc3","user":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"repo":{"id":20587599,"name":"flink","full_name":"apache/flink","owner":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/apache/flink","description":"Mirror of Apache Flink","fork":false,"url":"https://api.github.com/repos/apache/flink","forks_url":"https://api.github.com/repos/apache/flink/forks","keys_url":"https://api.github.com/repos/apache/flink/keys{/key_id}","collaborators_url":"https://api.github.com/repos/apache/flink/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/apache/flink/teams","hooks_url":"https://api.github.com/repos/apache/flink/hooks","issue_events_url":"https://api.github.com/repos/apache/flink/issues/events{/number}","events_url":"https://api.github.com/repos/apache/flink/events","assignees_url":"https://api.github.com/repos/apache/flink/assignees{/user}","branches_url":"https://api.github.com/repos/apache/flink/branches{/branch}","tags_url":"https://api.github.com/repos/apache/flink/tags","blobs_url":"https://api.github.com/repos/apache/flink/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/apache/flink/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/apache/flink/git/refs{/sha}","trees_url":"https://api.github.com/repos/apache/flink/git/trees{/sha}","statuses_url":"https://api.github.com/repos/apache/flink/statuses/{sha}","languages_url":"https://api.github.com/repos/apache/flink/languages","stargazers_url":"https://api.github.com/repos/apache/flink/stargazers","contributors_url":"https://api.github.com/repos/apache/flink/contributors","subscribers_url":"https://api.github.com/repos/apache/flink/subscribers","subscription_url":"https://api.github.com/repos/apache/flink/subscription","commits_url":"https://api.github.com/repos/apache/flink/commits{/sha}","git_commits_url":"https://api.github.com/repos/apache/flink/git/commits{/sha}","comments_url":"https://api.github.com/repos/apache/flink/comments{/number}","issue_comment_url":"https://api.github.com/repos/apache/flink/issues/comments{/number}","contents_url":"https://api.github.com/repos/apache/flink/contents/{+path}","compare_url":"https://api.github.com/repos/apache/flink/compare/{base}...{head}","merges_url":"https://api.github.com/repos/apache/flink/merges","archive_url":"https://api.github.com/repos/apache/flink/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/apache/flink/downloads","issues_url":"https://api.github.com/repos/apache/flink/issues{/number}","pulls_url":"https://api.github.com/repos/apache/flink/pulls{/number}","milestones_url":"https://api.github.com/repos/apache/flink/milestones{/number}","notifications_url":"https://api.github.com/repos/apache/flink/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/apache/flink/labels{/name}","releases_url":"https://api.github.com/repos/apache/flink/releases{/id}","deployments_url":"https://api.github.com/repos/apache/flink/deployments","created_at":"2014-06-07T07:00:10Z","updated_at":"2016-04-22T12:18:29Z","pushed_at":"2016-04-22T12:24:43Z","git_url":"git://github.com/apache/flink.git","ssh_url":"git@github.com:apache/flink.git","clone_url":"https://github.com/apache/flink.git","svn_url":"https://github.com/apache/flink","homepage":null,"size":114894,"stargazers_count":1242,"watchers_count":1242,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":773,"mirror_url":"git://git.apache.org/flink.git","open_issues_count":85,"forks":773,"open_issues":85,"watchers":1242,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/apache/flink/pulls/1911"},"html":{"href":"https://github.com/apache/flink/pull/1911"},"issue":{"href":"https://api.github.com/repos/apache/flink/issues/1911"},"comments":{"href":"https://api.github.com/repos/apache/flink/issues/1911/comments"},"review_comments":{"href":"https://api.github.com/repos/apache/flink/pulls/1911/comments"},"review_comment":{"href":"https://api.github.com/repos/apache/flink/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/apache/flink/pulls/1911/commits"},"statuses":{"href":"https://api.github.com/repos/apache/flink/statuses/e507555524ac59c91b4b1d010b1c219d4f424aa9"}}}}