{"action":"created","comment":{"url":"https://api.github.com/repos/apache/flink/pulls/comments/84251732","pull_request_review_id":5039572,"id":84251732,"diff_hunk":"@@ -0,0 +1,317 @@\n+/*\n+ * Licensed to the Apache Software Foundation (ASF) under one\n+ * or more contributor license agreements.  See the NOTICE file\n+ * distributed with this work for additional information\n+ * regarding copyright ownership.  The ASF licenses this file\n+ * to you under the Apache License, Version 2.0 (the\n+ * \"License\"); you may not use this file except in compliance\n+ * with the License.  You may obtain a copy of the License at\n+ *\n+ *     http://www.apache.org/licenses/LICENSE-2.0\n+ *\n+ * Unless required by applicable law or agreed to in writing, software\n+ * distributed under the License is distributed on an \"AS IS\" BASIS,\n+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+ * See the License for the specific language governing permissions and\n+ * limitations under the License.\n+ */\n+package org.apache.flink.streaming.api.operators;\n+\n+import org.apache.flink.api.common.typeutils.TypeSerializer;\n+import org.apache.flink.core.memory.DataInputViewStreamWrapper;\n+import org.apache.flink.core.memory.DataOutputViewStreamWrapper;\n+import org.apache.flink.streaming.runtime.tasks.ProcessingTimeCallback;\n+import org.apache.flink.streaming.runtime.tasks.ProcessingTimeService;\n+import org.apache.flink.util.InstantiationUtil;\n+\n+import java.io.IOException;\n+import java.io.InputStream;\n+import java.io.OutputStream;\n+import java.util.HashSet;\n+import java.util.PriorityQueue;\n+import java.util.Set;\n+import java.util.concurrent.ScheduledFuture;\n+\n+import static org.apache.flink.util.Preconditions.checkNotNull;\n+\n+/**\n+ * {@link InternalTimerService} that stores timers on the Java heap.\n+ */\n+public class HeapInternalTimerService<K, N> implements InternalTimerService<N>, ProcessingTimeCallback {\n+\n+\tprivate final TypeSerializer<K> keySerializer;\n+\n+\tprivate final TypeSerializer<N> namespaceSerializer;\n+\n+\tprivate final ProcessingTimeService processingTimeService;\n+\n+\tprivate long currentWatermark = Long.MIN_VALUE;\n+\n+\tprivate final org.apache.flink.streaming.api.operators.Triggerable<K, N> triggerTarget;\n+\n+\tprivate final KeyContext keyContext;\n+\n+\t/**\n+\t * Processing time timers that are currently in-flight.\n+\t */\n+\tprivate final PriorityQueue<InternalTimer<K, N>> processingTimeTimersQueue;\n+\tprivate final Set<InternalTimer<K, N>> processingTimeTimers;\n+\n+\tprotected ScheduledFuture<?> nextTimer = null;\n+\n+\t/**\n+\t * Currently waiting watermark callbacks.\n+\t */\n+\tprivate final Set<InternalTimer<K, N>> watermarkTimers;\n+\tprivate final PriorityQueue<InternalTimer<K, N>> watermarkTimersQueue;\n+\n+\tpublic HeapInternalTimerService(\n+\t\t\tTypeSerializer<K> keySerializer,\n+\t\t\tTypeSerializer<N> namespaceSerializer,\n+\t\t\torg.apache.flink.streaming.api.operators.Triggerable<K, N> triggerTarget,\n+\t\t\tKeyContext keyContext,\n+\t\t\tProcessingTimeService processingTimeService) {\n+\t\tthis.keySerializer = checkNotNull(keySerializer);\n+\t\tthis.namespaceSerializer = checkNotNull(namespaceSerializer);\n+\t\tthis.triggerTarget = checkNotNull(triggerTarget);\n+\t\tthis.keyContext = keyContext;\n+\t\tthis.processingTimeService = checkNotNull(processingTimeService);\n+\n+\t\twatermarkTimers = new HashSet<>();","path":"flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/HeapInternalTimerService.java","position":80,"original_position":80,"commit_id":"e6eb7353102f187787ed146a9d39eb9c55b86e78","original_commit_id":"e6eb7353102f187787ed146a9d39eb9c55b86e78","user":{"login":"aljoscha","id":68551,"avatar_url":"https://avatars.githubusercontent.com/u/68551?v=3","gravatar_id":"","url":"https://api.github.com/users/aljoscha","html_url":"https://github.com/aljoscha","followers_url":"https://api.github.com/users/aljoscha/followers","following_url":"https://api.github.com/users/aljoscha/following{/other_user}","gists_url":"https://api.github.com/users/aljoscha/gists{/gist_id}","starred_url":"https://api.github.com/users/aljoscha/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/aljoscha/subscriptions","organizations_url":"https://api.github.com/users/aljoscha/orgs","repos_url":"https://api.github.com/users/aljoscha/repos","events_url":"https://api.github.com/users/aljoscha/events{/privacy}","received_events_url":"https://api.github.com/users/aljoscha/received_events","type":"User","site_admin":false},"body":"I'm renaming","created_at":"2016-10-20T10:11:37Z","updated_at":"2016-10-20T10:11:37Z","html_url":"https://github.com/apache/flink/pull/2570#discussion_r84251732","pull_request_url":"https://api.github.com/repos/apache/flink/pulls/2570","_links":{"self":{"href":"https://api.github.com/repos/apache/flink/pulls/comments/84251732"},"html":{"href":"https://github.com/apache/flink/pull/2570#discussion_r84251732"},"pull_request":{"href":"https://api.github.com/repos/apache/flink/pulls/2570"}}},"pull_request":{"url":"https://api.github.com/repos/apache/flink/pulls/2570","id":87335200,"html_url":"https://github.com/apache/flink/pull/2570","diff_url":"https://github.com/apache/flink/pull/2570.diff","patch_url":"https://github.com/apache/flink/pull/2570.patch","issue_url":"https://api.github.com/repos/apache/flink/issues/2570","number":2570,"state":"open","locked":false,"title":"[FLINK-3674] Add an interface for Time aware User Functions","user":{"login":"aljoscha","id":68551,"avatar_url":"https://avatars.githubusercontent.com/u/68551?v=3","gravatar_id":"","url":"https://api.github.com/users/aljoscha","html_url":"https://github.com/aljoscha","followers_url":"https://api.github.com/users/aljoscha/followers","following_url":"https://api.github.com/users/aljoscha/following{/other_user}","gists_url":"https://api.github.com/users/aljoscha/gists{/gist_id}","starred_url":"https://api.github.com/users/aljoscha/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/aljoscha/subscriptions","organizations_url":"https://api.github.com/users/aljoscha/orgs","repos_url":"https://api.github.com/users/aljoscha/repos","events_url":"https://api.github.com/users/aljoscha/events{/privacy}","received_events_url":"https://api.github.com/users/aljoscha/received_events","type":"User","site_admin":false},"body":"This moves the event-time/processing-time trigger code from\r\n`WindowOperator` behind a well defined interface that can be used by\r\noperators (and user functions).\r\n\r\n`InternalTimerService` is the new interface that has the same\r\nfunctionality that `WindowOperator` used to have. `TimerService` is the user\r\nfacing interface that does not allow dealing with namespaces/payloads\r\nand also does not allow deleting timers. There is a default\r\nimplementation in `HeapInternalTimerService` that can checkpoint timers to\r\na stream and also restore from a stream. Right now, this is managed in\r\n`AbstractStreamOperator` and operators can ask for an\r\n`InternalTimerService`.\r\n\r\nThis also adds tests for `HeapInternalTimerService`.\r\n\r\nThis adds two new user functions:\r\n - `TimelyFlatMapFunction`: an extension of `FlatMapFunction` that also\r\n   allows querying time and setting timers\r\n - `TimelyCoFlatMapFunction`: the same, but for `CoFlatMapFunction`\r\n\r\nThere are two new `StreamOperator` implementations for these that use the\r\n`InternalTimerService` interface.\r\n\r\nThis also adds tests for the two new operators.\r\n\r\nThis also adds the new interface `KeyContext` that is used for\r\nsetting/querying the current key context for state and timers. Timers\r\nare always scoped to a key, for now.\r\n\r\nAlso, this moves the handling of watermarks for both one-input and\r\ntwo-input operators to `AbstractStreamOperators` so that we have a central\r\nground-truth.\r\n\r\nThere was also a bunch of small changes that I had to do to make the proper change more clean. I would like to keep these as separate commits because they clearly document what was going on.\r\n\r\n## Note for Reviewers\r\nYou should probably start from the tests, i.e. `HeapInternalTimerServiceTest`, `TimelyFlatMapTest` and `TimelyCoFlatMapTest`. Then, the other interesting bits are `AbstractStreamOperator` that now deals with watermarks and checkpointing the timers and the `HeapInternalTimerService` as well. Keep in mind that this is just moving the code from `WindowOperator` to `HeapInternalTimerService` with some generalizations. I didn't try to optimize any of the data structures that are used.\r\n\r\nR: @StephanEwen @StefanRRichter @kl0u for review, please 😃 ","created_at":"2016-09-29T15:21:24Z","updated_at":"2016-10-20T10:11:37Z","closed_at":null,"merged_at":null,"merge_commit_sha":"7ab42ec0c91bb94a1034ae3e68d865d82b4a5365","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/apache/flink/pulls/2570/commits","review_comments_url":"https://api.github.com/repos/apache/flink/pulls/2570/comments","review_comment_url":"https://api.github.com/repos/apache/flink/pulls/comments{/number}","comments_url":"https://api.github.com/repos/apache/flink/issues/2570/comments","statuses_url":"https://api.github.com/repos/apache/flink/statuses/e6eb7353102f187787ed146a9d39eb9c55b86e78","head":{"label":"aljoscha:timely-function","ref":"timely-function","sha":"e6eb7353102f187787ed146a9d39eb9c55b86e78","user":{"login":"aljoscha","id":68551,"avatar_url":"https://avatars.githubusercontent.com/u/68551?v=3","gravatar_id":"","url":"https://api.github.com/users/aljoscha","html_url":"https://github.com/aljoscha","followers_url":"https://api.github.com/users/aljoscha/followers","following_url":"https://api.github.com/users/aljoscha/following{/other_user}","gists_url":"https://api.github.com/users/aljoscha/gists{/gist_id}","starred_url":"https://api.github.com/users/aljoscha/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/aljoscha/subscriptions","organizations_url":"https://api.github.com/users/aljoscha/orgs","repos_url":"https://api.github.com/users/aljoscha/repos","events_url":"https://api.github.com/users/aljoscha/events{/privacy}","received_events_url":"https://api.github.com/users/aljoscha/received_events","type":"User","site_admin":false},"repo":{"id":22066189,"name":"flink","full_name":"aljoscha/flink","owner":{"login":"aljoscha","id":68551,"avatar_url":"https://avatars.githubusercontent.com/u/68551?v=3","gravatar_id":"","url":"https://api.github.com/users/aljoscha","html_url":"https://github.com/aljoscha","followers_url":"https://api.github.com/users/aljoscha/followers","following_url":"https://api.github.com/users/aljoscha/following{/other_user}","gists_url":"https://api.github.com/users/aljoscha/gists{/gist_id}","starred_url":"https://api.github.com/users/aljoscha/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/aljoscha/subscriptions","organizations_url":"https://api.github.com/users/aljoscha/orgs","repos_url":"https://api.github.com/users/aljoscha/repos","events_url":"https://api.github.com/users/aljoscha/events{/privacy}","received_events_url":"https://api.github.com/users/aljoscha/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/aljoscha/flink","description":"Mirror of Apache Flink","fork":true,"url":"https://api.github.com/repos/aljoscha/flink","forks_url":"https://api.github.com/repos/aljoscha/flink/forks","keys_url":"https://api.github.com/repos/aljoscha/flink/keys{/key_id}","collaborators_url":"https://api.github.com/repos/aljoscha/flink/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/aljoscha/flink/teams","hooks_url":"https://api.github.com/repos/aljoscha/flink/hooks","issue_events_url":"https://api.github.com/repos/aljoscha/flink/issues/events{/number}","events_url":"https://api.github.com/repos/aljoscha/flink/events","assignees_url":"https://api.github.com/repos/aljoscha/flink/assignees{/user}","branches_url":"https://api.github.com/repos/aljoscha/flink/branches{/branch}","tags_url":"https://api.github.com/repos/aljoscha/flink/tags","blobs_url":"https://api.github.com/repos/aljoscha/flink/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/aljoscha/flink/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/aljoscha/flink/git/refs{/sha}","trees_url":"https://api.github.com/repos/aljoscha/flink/git/trees{/sha}","statuses_url":"https://api.github.com/repos/aljoscha/flink/statuses/{sha}","languages_url":"https://api.github.com/repos/aljoscha/flink/languages","stargazers_url":"https://api.github.com/repos/aljoscha/flink/stargazers","contributors_url":"https://api.github.com/repos/aljoscha/flink/contributors","subscribers_url":"https://api.github.com/repos/aljoscha/flink/subscribers","subscription_url":"https://api.github.com/repos/aljoscha/flink/subscription","commits_url":"https://api.github.com/repos/aljoscha/flink/commits{/sha}","git_commits_url":"https://api.github.com/repos/aljoscha/flink/git/commits{/sha}","comments_url":"https://api.github.com/repos/aljoscha/flink/comments{/number}","issue_comment_url":"https://api.github.com/repos/aljoscha/flink/issues/comments{/number}","contents_url":"https://api.github.com/repos/aljoscha/flink/contents/{+path}","compare_url":"https://api.github.com/repos/aljoscha/flink/compare/{base}...{head}","merges_url":"https://api.github.com/repos/aljoscha/flink/merges","archive_url":"https://api.github.com/repos/aljoscha/flink/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/aljoscha/flink/downloads","issues_url":"https://api.github.com/repos/aljoscha/flink/issues{/number}","pulls_url":"https://api.github.com/repos/aljoscha/flink/pulls{/number}","milestones_url":"https://api.github.com/repos/aljoscha/flink/milestones{/number}","notifications_url":"https://api.github.com/repos/aljoscha/flink/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/aljoscha/flink/labels{/name}","releases_url":"https://api.github.com/repos/aljoscha/flink/releases{/id}","deployments_url":"https://api.github.com/repos/aljoscha/flink/deployments","created_at":"2014-07-21T13:51:54Z","updated_at":"2016-06-25T06:57:26Z","pushed_at":"2016-10-20T10:09:27Z","git_url":"git://github.com/aljoscha/flink.git","ssh_url":"git@github.com:aljoscha/flink.git","clone_url":"https://github.com/aljoscha/flink.git","svn_url":"https://github.com/aljoscha/flink","homepage":null,"size":96078,"stargazers_count":0,"watchers_count":0,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"apache:master","ref":"master","sha":"398bd9b3198e70f199a1277a9f5074e81cbee3c8","user":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"repo":{"id":20587599,"name":"flink","full_name":"apache/flink","owner":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/apache/flink","description":"Mirror of Apache Flink","fork":false,"url":"https://api.github.com/repos/apache/flink","forks_url":"https://api.github.com/repos/apache/flink/forks","keys_url":"https://api.github.com/repos/apache/flink/keys{/key_id}","collaborators_url":"https://api.github.com/repos/apache/flink/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/apache/flink/teams","hooks_url":"https://api.github.com/repos/apache/flink/hooks","issue_events_url":"https://api.github.com/repos/apache/flink/issues/events{/number}","events_url":"https://api.github.com/repos/apache/flink/events","assignees_url":"https://api.github.com/repos/apache/flink/assignees{/user}","branches_url":"https://api.github.com/repos/apache/flink/branches{/branch}","tags_url":"https://api.github.com/repos/apache/flink/tags","blobs_url":"https://api.github.com/repos/apache/flink/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/apache/flink/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/apache/flink/git/refs{/sha}","trees_url":"https://api.github.com/repos/apache/flink/git/trees{/sha}","statuses_url":"https://api.github.com/repos/apache/flink/statuses/{sha}","languages_url":"https://api.github.com/repos/apache/flink/languages","stargazers_url":"https://api.github.com/repos/apache/flink/stargazers","contributors_url":"https://api.github.com/repos/apache/flink/contributors","subscribers_url":"https://api.github.com/repos/apache/flink/subscribers","subscription_url":"https://api.github.com/repos/apache/flink/subscription","commits_url":"https://api.github.com/repos/apache/flink/commits{/sha}","git_commits_url":"https://api.github.com/repos/apache/flink/git/commits{/sha}","comments_url":"https://api.github.com/repos/apache/flink/comments{/number}","issue_comment_url":"https://api.github.com/repos/apache/flink/issues/comments{/number}","contents_url":"https://api.github.com/repos/apache/flink/contents/{+path}","compare_url":"https://api.github.com/repos/apache/flink/compare/{base}...{head}","merges_url":"https://api.github.com/repos/apache/flink/merges","archive_url":"https://api.github.com/repos/apache/flink/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/apache/flink/downloads","issues_url":"https://api.github.com/repos/apache/flink/issues{/number}","pulls_url":"https://api.github.com/repos/apache/flink/pulls{/number}","milestones_url":"https://api.github.com/repos/apache/flink/milestones{/number}","notifications_url":"https://api.github.com/repos/apache/flink/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/apache/flink/labels{/name}","releases_url":"https://api.github.com/repos/apache/flink/releases{/id}","deployments_url":"https://api.github.com/repos/apache/flink/deployments","created_at":"2014-06-07T07:00:10Z","updated_at":"2016-10-20T08:51:43Z","pushed_at":"2016-10-20T10:09:28Z","git_url":"git://github.com/apache/flink.git","ssh_url":"git@github.com:apache/flink.git","clone_url":"https://github.com/apache/flink.git","svn_url":"https://github.com/apache/flink","homepage":null,"size":133921,"stargazers_count":1660,"watchers_count":1660,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":1136,"mirror_url":"git://git.apache.org/flink.git","open_issues_count":152,"forks":1136,"open_issues":152,"watchers":1660,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/apache/flink/pulls/2570"},"html":{"href":"https://github.com/apache/flink/pull/2570"},"issue":{"href":"https://api.github.com/repos/apache/flink/issues/2570"},"comments":{"href":"https://api.github.com/repos/apache/flink/issues/2570/comments"},"review_comments":{"href":"https://api.github.com/repos/apache/flink/pulls/2570/comments"},"review_comment":{"href":"https://api.github.com/repos/apache/flink/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/apache/flink/pulls/2570/commits"},"statuses":{"href":"https://api.github.com/repos/apache/flink/statuses/e6eb7353102f187787ed146a9d39eb9c55b86e78"}}}}
{"actor":{"display_login":"aljoscha"}}