{"action":"created","comment":{"url":"https://api.github.com/repos/apache/flink/pulls/comments/81944268","pull_request_review_id":2885454,"id":81944268,"diff_hunk":"@@ -106,6 +107,155 @@ public static void destroyHDFS() {\n \t//\t\t\t\t\t\tTESTS\n \n \t@Test\n+\tpublic void testFileReadingOperatorWithIngestionTime() throws Exception {\n+\t\tSet<org.apache.hadoop.fs.Path> filesCreated = new HashSet<>();\n+\t\tMap<Integer, String> expectedFileContents = new HashMap<>();\n+\n+\t\tfor(int i = 0; i < NO_OF_FILES; i++) {\n+\t\t\tTuple2<org.apache.hadoop.fs.Path, String> file = fillWithData(hdfsURI, \"file\", i, \"This is test line.\");\n+\t\t\tfilesCreated.add(file.f0);\n+\t\t\texpectedFileContents.put(i, file.f1);\n+\t\t}\n+\n+\t\tTextInputFormat format = new TextInputFormat(new Path(hdfsURI));\n+\t\tTypeInformation<String> typeInfo = TypeExtractor.getInputFormatTypes(format);\n+\n+\t\tfinal long watermarkInterval = 10;\n+\t\tExecutionConfig executionConfig = new ExecutionConfig();\n+\t\texecutionConfig.setAutoWatermarkInterval(watermarkInterval);\n+\n+\t\tContinuousFileReaderOperator<String, ?> reader = new ContinuousFileReaderOperator<>(format);\n+\t\treader.setOutputType(typeInfo, executionConfig);\n+\n+\t\tfinal TestTimeServiceProvider timeServiceProvider = new TestTimeServiceProvider();\n+\t\tfinal OneInputStreamOperatorTestHarness<FileInputSplit, String> tester =\n+\t\t\tnew OneInputStreamOperatorTestHarness<>(reader, executionConfig, timeServiceProvider);\n+\t\ttester.setTimeCharacteristic(TimeCharacteristic.IngestionTime);\n+\t\ttester.open();\n+\n+\t\tAssert.assertEquals(TimeCharacteristic.IngestionTime, tester.getTimeCharacteristic());\n+\n+\t\t// test that watermarks are correctly emitted\n+\n+\t\ttimeServiceProvider.setCurrentTime(201);\n+\t\ttimeServiceProvider.setCurrentTime(301);\n+\t\ttimeServiceProvider.setCurrentTime(401);\n+\t\ttimeServiceProvider.setCurrentTime(501);\n+\n+\t\tint i = 0;\n+\t\tfor(Object line: tester.getOutput()) {\n+\t\t\tif (!(line instanceof Watermark)) {\n+\t\t\t\tAssert.fail(\"Only watermarks are expected here \");\n+\t\t\t}\n+\t\t\tWatermark w = (Watermark) line;\n+\t\t\tAssert.assertEquals(200 + (i * 100), w.getTimestamp());\n+\t\t\ti++;\n+\t\t}\n+\n+\t\t// clear the output to get the elements only and the final watermark\n+\t\ttester.getOutput().clear();\n+\t\tAssert.assertEquals(0, tester.getOutput().size());\n+\n+\t\t// create the necessary splits for the test\n+\t\tFileInputSplit[] splits = format.createInputSplits(\n+\t\t\treader.getRuntimeContext().getNumberOfParallelSubtasks());\n+\n+\t\t// and feed them to the operator\n+\t\tMap<Integer, List<String>> actualFileContents = new HashMap<>();\n+\n+\t\tlong lastSeenWatermark = Long.MIN_VALUE;\n+\t\tint lineCounter = 0;\t// counter for the lines read from the splits\n+\t\tint watermarkCounter = 0;\n+\n+\t\tfor(FileInputSplit split: splits) {\n+\n+\t\t\t// set the next \"current processing time\".\n+\t\t\tlong nextTimestamp = timeServiceProvider.getCurrentProcessingTime() + watermarkInterval;\n+\t\t\ttimeServiceProvider.setCurrentTime(nextTimestamp);\n+\n+\t\t\t// send the next split to be read and wait until it is fully read.\n+\t\t\ttester.processElement(new StreamRecord<>(split));\n+\t\t\tsynchronized (tester.getCheckpointLock()) {\n+\t\t\t\twhile (tester.getOutput().isEmpty() || tester.getOutput().size() != (LINES_PER_FILE + 1)) {\n+\t\t\t\t\ttester.getCheckpointLock().wait(10);\n+\t\t\t\t}\n+\t\t\t}\n+\n+\t\t\t// verify that the results are the expected\n+\t\t\tfor(Object line: tester.getOutput()) {\n+\t\t\t\tif (line instanceof StreamRecord) {\n+\t\t\t\t\tStreamRecord<String> element = (StreamRecord<String>) line;\n+\t\t\t\t\tlineCounter++;\n+\n+\t\t\t\t\tAssert.assertEquals(nextTimestamp, element.getTimestamp());\n+\n+\t\t\t\t\tint fileIdx = Character.getNumericValue(element.getValue().charAt(0));\n+\t\t\t\t\tList<String> content = actualFileContents.get(fileIdx);\n+\t\t\t\t\tif (content == null) {\n+\t\t\t\t\t\tcontent = new ArrayList<>();\n+\t\t\t\t\t\tactualFileContents.put(fileIdx, content);\n+\t\t\t\t\t}\n+\t\t\t\t\tcontent.add(element.getValue() + \"\\n\");\n+\t\t\t\t} else if (line instanceof Watermark) {\n+\t\t\t\t\tlong watermark = ((Watermark) line).getTimestamp();\n+\n+\t\t\t\t\tAssert.assertEquals(nextTimestamp - (nextTimestamp % watermarkInterval), watermark);\n+\t\t\t\t\tAssert.assertTrue(watermark > lastSeenWatermark);\n+\t\t\t\t\twatermarkCounter++;\n+\n+\t\t\t\t\tlastSeenWatermark = watermark;\n+\t\t\t\t} else {\n+\t\t\t\t\tAssert.fail(\"Unknown element in the list.\");\n+\t\t\t\t}\n+\t\t\t}\n+\n+\t\t\t// clean the output to be ready for the next split\n+\t\t\ttester.getOutput().clear();\n+\t\t}\n+\n+\t\t// now we are processing one split after the other,\n+\t\t// so all the elements must be here by now.\n+\t\tAssert.assertEquals(NO_OF_FILES * LINES_PER_FILE, lineCounter);\n+\n+\t\t// because we expect one watermark per split.\n+\t\tAssert.assertEquals(NO_OF_FILES, watermarkCounter);","path":"flink-fs-tests/src/test/java/org/apache/flink/hdfstests/ContinuousFileMonitoringTest.java","position":138,"original_position":138,"commit_id":"79d48e77b4a9c8a8eaf4e1e3199e2787deebab63","original_commit_id":"79d48e77b4a9c8a8eaf4e1e3199e2787deebab63","user":{"login":"mxm","id":837221,"avatar_url":"https://avatars.githubusercontent.com/u/837221?v=3","gravatar_id":"","url":"https://api.github.com/users/mxm","html_url":"https://github.com/mxm","followers_url":"https://api.github.com/users/mxm/followers","following_url":"https://api.github.com/users/mxm/following{/other_user}","gists_url":"https://api.github.com/users/mxm/gists{/gist_id}","starred_url":"https://api.github.com/users/mxm/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mxm/subscriptions","organizations_url":"https://api.github.com/users/mxm/orgs","repos_url":"https://api.github.com/users/mxm/repos","events_url":"https://api.github.com/users/mxm/events{/privacy}","received_events_url":"https://api.github.com/users/mxm/received_events","type":"User","site_admin":false},"body":"This should be `Assert.assertEquals(splits.length, watermarkCounter);`.","created_at":"2016-10-05T10:33:00Z","updated_at":"2016-10-05T10:33:00Z","html_url":"https://github.com/apache/flink/pull/2593#discussion_r81944268","pull_request_url":"https://api.github.com/repos/apache/flink/pulls/2593","_links":{"self":{"href":"https://api.github.com/repos/apache/flink/pulls/comments/81944268"},"html":{"href":"https://github.com/apache/flink/pull/2593#discussion_r81944268"},"pull_request":{"href":"https://api.github.com/repos/apache/flink/pulls/2593"}}},"pull_request":{"url":"https://api.github.com/repos/apache/flink/pulls/2593","id":88034196,"html_url":"https://github.com/apache/flink/pull/2593","diff_url":"https://github.com/apache/flink/pull/2593.diff","patch_url":"https://github.com/apache/flink/pull/2593.patch","issue_url":"https://api.github.com/repos/apache/flink/issues/2593","number":2593,"state":"open","locked":false,"title":"[FLINK-4329] Fix Streaming File Source Timestamps/Watermarks Handling","user":{"login":"kl0u","id":2405800,"avatar_url":"https://avatars.githubusercontent.com/u/2405800?v=3","gravatar_id":"","url":"https://api.github.com/users/kl0u","html_url":"https://github.com/kl0u","followers_url":"https://api.github.com/users/kl0u/followers","following_url":"https://api.github.com/users/kl0u/following{/other_user}","gists_url":"https://api.github.com/users/kl0u/gists{/gist_id}","starred_url":"https://api.github.com/users/kl0u/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/kl0u/subscriptions","organizations_url":"https://api.github.com/users/kl0u/orgs","repos_url":"https://api.github.com/users/kl0u/repos","events_url":"https://api.github.com/users/kl0u/events{/privacy}","received_events_url":"https://api.github.com/users/kl0u/received_events","type":"User","site_admin":false},"body":"This is a quick fix for the [FLINK-4329] issue. The fix on the master is different but it contains more changes that are not easy to back-port to 1.1.","created_at":"2016-10-05T08:21:23Z","updated_at":"2016-10-05T10:53:14Z","closed_at":null,"merged_at":null,"merge_commit_sha":"3ff9bbb1dc84816420fe57f90d5baf59824a8f06","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/apache/flink/pulls/2593/commits","review_comments_url":"https://api.github.com/repos/apache/flink/pulls/2593/comments","review_comment_url":"https://api.github.com/repos/apache/flink/pulls/comments{/number}","comments_url":"https://api.github.com/repos/apache/flink/issues/2593/comments","statuses_url":"https://api.github.com/repos/apache/flink/statuses/79d48e77b4a9c8a8eaf4e1e3199e2787deebab63","head":{"label":"kl0u:injestion_fix_1.1","ref":"injestion_fix_1.1","sha":"79d48e77b4a9c8a8eaf4e1e3199e2787deebab63","user":{"login":"kl0u","id":2405800,"avatar_url":"https://avatars.githubusercontent.com/u/2405800?v=3","gravatar_id":"","url":"https://api.github.com/users/kl0u","html_url":"https://github.com/kl0u","followers_url":"https://api.github.com/users/kl0u/followers","following_url":"https://api.github.com/users/kl0u/following{/other_user}","gists_url":"https://api.github.com/users/kl0u/gists{/gist_id}","starred_url":"https://api.github.com/users/kl0u/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/kl0u/subscriptions","organizations_url":"https://api.github.com/users/kl0u/orgs","repos_url":"https://api.github.com/users/kl0u/repos","events_url":"https://api.github.com/users/kl0u/events{/privacy}","received_events_url":"https://api.github.com/users/kl0u/received_events","type":"User","site_admin":false},"repo":{"id":38500934,"name":"flink","full_name":"kl0u/flink","owner":{"login":"kl0u","id":2405800,"avatar_url":"https://avatars.githubusercontent.com/u/2405800?v=3","gravatar_id":"","url":"https://api.github.com/users/kl0u","html_url":"https://github.com/kl0u","followers_url":"https://api.github.com/users/kl0u/followers","following_url":"https://api.github.com/users/kl0u/following{/other_user}","gists_url":"https://api.github.com/users/kl0u/gists{/gist_id}","starred_url":"https://api.github.com/users/kl0u/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/kl0u/subscriptions","organizations_url":"https://api.github.com/users/kl0u/orgs","repos_url":"https://api.github.com/users/kl0u/repos","events_url":"https://api.github.com/users/kl0u/events{/privacy}","received_events_url":"https://api.github.com/users/kl0u/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/kl0u/flink","description":"Mirror of Apache Flink","fork":true,"url":"https://api.github.com/repos/kl0u/flink","forks_url":"https://api.github.com/repos/kl0u/flink/forks","keys_url":"https://api.github.com/repos/kl0u/flink/keys{/key_id}","collaborators_url":"https://api.github.com/repos/kl0u/flink/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/kl0u/flink/teams","hooks_url":"https://api.github.com/repos/kl0u/flink/hooks","issue_events_url":"https://api.github.com/repos/kl0u/flink/issues/events{/number}","events_url":"https://api.github.com/repos/kl0u/flink/events","assignees_url":"https://api.github.com/repos/kl0u/flink/assignees{/user}","branches_url":"https://api.github.com/repos/kl0u/flink/branches{/branch}","tags_url":"https://api.github.com/repos/kl0u/flink/tags","blobs_url":"https://api.github.com/repos/kl0u/flink/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/kl0u/flink/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/kl0u/flink/git/refs{/sha}","trees_url":"https://api.github.com/repos/kl0u/flink/git/trees{/sha}","statuses_url":"https://api.github.com/repos/kl0u/flink/statuses/{sha}","languages_url":"https://api.github.com/repos/kl0u/flink/languages","stargazers_url":"https://api.github.com/repos/kl0u/flink/stargazers","contributors_url":"https://api.github.com/repos/kl0u/flink/contributors","subscribers_url":"https://api.github.com/repos/kl0u/flink/subscribers","subscription_url":"https://api.github.com/repos/kl0u/flink/subscription","commits_url":"https://api.github.com/repos/kl0u/flink/commits{/sha}","git_commits_url":"https://api.github.com/repos/kl0u/flink/git/commits{/sha}","comments_url":"https://api.github.com/repos/kl0u/flink/comments{/number}","issue_comment_url":"https://api.github.com/repos/kl0u/flink/issues/comments{/number}","contents_url":"https://api.github.com/repos/kl0u/flink/contents/{+path}","compare_url":"https://api.github.com/repos/kl0u/flink/compare/{base}...{head}","merges_url":"https://api.github.com/repos/kl0u/flink/merges","archive_url":"https://api.github.com/repos/kl0u/flink/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/kl0u/flink/downloads","issues_url":"https://api.github.com/repos/kl0u/flink/issues{/number}","pulls_url":"https://api.github.com/repos/kl0u/flink/pulls{/number}","milestones_url":"https://api.github.com/repos/kl0u/flink/milestones{/number}","notifications_url":"https://api.github.com/repos/kl0u/flink/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/kl0u/flink/labels{/name}","releases_url":"https://api.github.com/repos/kl0u/flink/releases{/id}","deployments_url":"https://api.github.com/repos/kl0u/flink/deployments","created_at":"2015-07-03T16:09:57Z","updated_at":"2016-01-13T16:29:47Z","pushed_at":"2016-10-04T15:11:20Z","git_url":"git://github.com/kl0u/flink.git","ssh_url":"git@github.com:kl0u/flink.git","clone_url":"https://github.com/kl0u/flink.git","svn_url":"https://github.com/kl0u/flink","homepage":null,"size":93941,"stargazers_count":0,"watchers_count":0,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"apache:release-1.1","ref":"release-1.1","sha":"a31a22ec7fc08cce8da4e5e97010d1b2cf8124e7","user":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"repo":{"id":20587599,"name":"flink","full_name":"apache/flink","owner":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/apache/flink","description":"Mirror of Apache Flink","fork":false,"url":"https://api.github.com/repos/apache/flink","forks_url":"https://api.github.com/repos/apache/flink/forks","keys_url":"https://api.github.com/repos/apache/flink/keys{/key_id}","collaborators_url":"https://api.github.com/repos/apache/flink/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/apache/flink/teams","hooks_url":"https://api.github.com/repos/apache/flink/hooks","issue_events_url":"https://api.github.com/repos/apache/flink/issues/events{/number}","events_url":"https://api.github.com/repos/apache/flink/events","assignees_url":"https://api.github.com/repos/apache/flink/assignees{/user}","branches_url":"https://api.github.com/repos/apache/flink/branches{/branch}","tags_url":"https://api.github.com/repos/apache/flink/tags","blobs_url":"https://api.github.com/repos/apache/flink/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/apache/flink/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/apache/flink/git/refs{/sha}","trees_url":"https://api.github.com/repos/apache/flink/git/trees{/sha}","statuses_url":"https://api.github.com/repos/apache/flink/statuses/{sha}","languages_url":"https://api.github.com/repos/apache/flink/languages","stargazers_url":"https://api.github.com/repos/apache/flink/stargazers","contributors_url":"https://api.github.com/repos/apache/flink/contributors","subscribers_url":"https://api.github.com/repos/apache/flink/subscribers","subscription_url":"https://api.github.com/repos/apache/flink/subscription","commits_url":"https://api.github.com/repos/apache/flink/commits{/sha}","git_commits_url":"https://api.github.com/repos/apache/flink/git/commits{/sha}","comments_url":"https://api.github.com/repos/apache/flink/comments{/number}","issue_comment_url":"https://api.github.com/repos/apache/flink/issues/comments{/number}","contents_url":"https://api.github.com/repos/apache/flink/contents/{+path}","compare_url":"https://api.github.com/repos/apache/flink/compare/{base}...{head}","merges_url":"https://api.github.com/repos/apache/flink/merges","archive_url":"https://api.github.com/repos/apache/flink/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/apache/flink/downloads","issues_url":"https://api.github.com/repos/apache/flink/issues{/number}","pulls_url":"https://api.github.com/repos/apache/flink/pulls{/number}","milestones_url":"https://api.github.com/repos/apache/flink/milestones{/number}","notifications_url":"https://api.github.com/repos/apache/flink/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/apache/flink/labels{/name}","releases_url":"https://api.github.com/repos/apache/flink/releases{/id}","deployments_url":"https://api.github.com/repos/apache/flink/deployments","created_at":"2014-06-07T07:00:10Z","updated_at":"2016-10-05T04:21:03Z","pushed_at":"2016-10-05T10:51:27Z","git_url":"git://github.com/apache/flink.git","ssh_url":"git@github.com:apache/flink.git","clone_url":"https://github.com/apache/flink.git","svn_url":"https://github.com/apache/flink","homepage":null,"size":129592,"stargazers_count":1616,"watchers_count":1616,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":1101,"mirror_url":"git://git.apache.org/flink.git","open_issues_count":152,"forks":1101,"open_issues":152,"watchers":1616,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/apache/flink/pulls/2593"},"html":{"href":"https://github.com/apache/flink/pull/2593"},"issue":{"href":"https://api.github.com/repos/apache/flink/issues/2593"},"comments":{"href":"https://api.github.com/repos/apache/flink/issues/2593/comments"},"review_comments":{"href":"https://api.github.com/repos/apache/flink/pulls/2593/comments"},"review_comment":{"href":"https://api.github.com/repos/apache/flink/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/apache/flink/pulls/2593/commits"},"statuses":{"href":"https://api.github.com/repos/apache/flink/statuses/79d48e77b4a9c8a8eaf4e1e3199e2787deebab63"}}}}
{"actor":{"display_login":"mxm"}}