{"action":"created","comment":{"url":"https://api.github.com/repos/apache/metron/pulls/comments/215621040","pull_request_review_id":152924489,"id":215621040,"node_id":"MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDIxNTYyMTA0MA==","diff_hunk":"@@ -150,27 +159,98 @@ public void testProcessingTime() throws Exception {\n     kafkaComponent.writeMessages(inputTopic, message2);\n     kafkaComponent.writeMessages(inputTopic, message3);\n \n+    // retrieve the profile measurement using PROFILE_GET\n+    String profileGetExpression = \"PROFILE_GET('processing-time-test', '10.0.0.1', PROFILE_FIXED('5', 'MINUTES'))\";\n+    List<Integer> actuals = execute(profileGetExpression, List.class);\n+\n     // storm needs at least one message to close its event window\n     int attempt = 0;\n-    while(profilerTable.getPutLog().size() == 0 && attempt++ < 10) {\n+    while(actuals.size() == 0 && attempt++ < 10) {\n \n-      // sleep, at least beyond the current window\n-      Thread.sleep(windowDurationMillis + windowLagMillis);\n+      // wait for the profiler to flush\n+      long sleep = windowDurationMillis;\n+      LOG.debug(\"Waiting {} millis for profiler to flush\", sleep);\n+      Thread.sleep(sleep);\n \n-      // send another message to help close the current event window\n+      // write another message to advance time.  this ensures that we are testing the 'normal' flush mechanism.\n+      // if we do not send additional messages to advance time, then it is the profile TTL mechanism which\n+      // will ultimately flush the profile\n       kafkaComponent.writeMessages(inputTopic, message2);\n+\n+      // retrieve the profile measurement using PROFILE_GET\n+      actuals = execute(profileGetExpression, List.class);\n     }\n \n-    // validate what was flushed\n-    List<Integer> actuals = read(\n-            profilerTable.getPutLog(),\n-            columnFamily,\n-            columnBuilder.getColumnQualifier(\"value\"),\n-            Integer.class);\n-    assertEquals(1, actuals.size());\n+    // the profile should count at least 3 messages\n+    assertTrue(actuals.size() > 0);\n     assertTrue(actuals.get(0) >= 3);\n   }\n \n+  /**\n+   * The Profiler can generate profiles based on processing time.  With processing time,\n+   * the Profiler builds profiles based on when the telemetry is processed.\n+   *\n+   * <p>Not defining a 'timestampField' within the Profiler configuration tells the Profiler\n+   * to use processing time.\n+   *\n+   * <p>There are two mechanisms that will cause a profile to flush.\n+   *\n+   * (1) As new messages arrive, time is advanced. The splitter bolt attaches a timestamp to each\n+   * message (which can be either event or system time.)  This advances time and leads to profile\n+   * measurements being flushed.\n+   *\n+   * (2) If no messages arrive to advance time, then the \"time to live\" mechanism will flush a profile\n+   * after a period of time.\n+   *\n+   * <p>This test specifically tests the *second* mechanism when a profile is flushed by the\n+   * \"time to live\" mechanism.\n+   */\n+  @Test\n+  public void testProcessingTimeWithTimeToLiveFlush() throws Exception {","path":"metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/integration/ProfilerIntegrationTest.java","position":210,"original_position":210,"commit_id":"e70cf943faf0c92abe7f27996a72974e87e80117","original_commit_id":"e70cf943faf0c92abe7f27996a72974e87e80117","user":{"login":"nickwallen","id":2475409,"node_id":"MDQ6VXNlcjI0NzU0MDk=","avatar_url":"https://avatars3.githubusercontent.com/u/2475409?v=4","gravatar_id":"","url":"https://api.github.com/users/nickwallen","html_url":"https://github.com/nickwallen","followers_url":"https://api.github.com/users/nickwallen/followers","following_url":"https://api.github.com/users/nickwallen/following{/other_user}","gists_url":"https://api.github.com/users/nickwallen/gists{/gist_id}","starred_url":"https://api.github.com/users/nickwallen/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/nickwallen/subscriptions","organizations_url":"https://api.github.com/users/nickwallen/orgs","repos_url":"https://api.github.com/users/nickwallen/repos","events_url":"https://api.github.com/users/nickwallen/events{/privacy}","received_events_url":"https://api.github.com/users/nickwallen/received_events","type":"User","site_admin":false},"body":"Ok, fair point. Let me see if I can't move this to the README and clean it up.","created_at":"2018-09-06T13:18:41Z","updated_at":"2018-09-06T13:18:41Z","html_url":"https://github.com/apache/metron/pull/1174#discussion_r215621040","pull_request_url":"https://api.github.com/repos/apache/metron/pulls/1174","author_association":"CONTRIBUTOR","_links":{"self":{"href":"https://api.github.com/repos/apache/metron/pulls/comments/215621040"},"html":{"href":"https://github.com/apache/metron/pull/1174#discussion_r215621040"},"pull_request":{"href":"https://api.github.com/repos/apache/metron/pulls/1174"}},"in_reply_to_id":215452413},"pull_request":{"url":"https://api.github.com/repos/apache/metron/pulls/1174","id":210538412,"node_id":"MDExOlB1bGxSZXF1ZXN0MjEwNTM4NDEy","html_url":"https://github.com/apache/metron/pull/1174","diff_url":"https://github.com/apache/metron/pull/1174.diff","patch_url":"https://github.com/apache/metron/pull/1174.patch","issue_url":"https://api.github.com/repos/apache/metron/issues/1174","number":1174,"state":"open","locked":false,"title":"METRON-1748 Improve Storm Profiler Integration Test","user":{"login":"nickwallen","id":2475409,"node_id":"MDQ6VXNlcjI0NzU0MDk=","avatar_url":"https://avatars3.githubusercontent.com/u/2475409?v=4","gravatar_id":"","url":"https://api.github.com/users/nickwallen","html_url":"https://github.com/nickwallen","followers_url":"https://api.github.com/users/nickwallen/followers","following_url":"https://api.github.com/users/nickwallen/following{/other_user}","gists_url":"https://api.github.com/users/nickwallen/gists{/gist_id}","starred_url":"https://api.github.com/users/nickwallen/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/nickwallen/subscriptions","organizations_url":"https://api.github.com/users/nickwallen/orgs","repos_url":"https://api.github.com/users/nickwallen/repos","events_url":"https://api.github.com/users/nickwallen/events{/privacy}","received_events_url":"https://api.github.com/users/nickwallen/received_events","type":"User","site_admin":false},"body":"Improved the Storm Profiler integration tests based on improvements I made for the Spark Profiler feature branch.\r\n\r\n* The output of the Profiler is now validated using the client library; `PROFILE_GET`.  This is better validation that things are working end-to-end.\r\n\r\n* Using more telemetry messages to validate event time processing.\r\n\r\n* Added logging around cache maintenence.\r\n\r\n* Added logging of the tuple windows that are passed from Storm's WindowManager.  This can help debug issues where the time lag and window size are incorrectly set.\r\n\r\n## Testing\r\n\r\nThis only changes the integration tests and adds logging.  Running the integration tests is sufficient to test the changes there.  Running up the development environment is sufficient for the extra caution.\r\n\r\n## Pull Request Checklist\r\n\r\n- [x] Is there a JIRA ticket associated with this PR? If not one needs to be created at [Metron Jira](https://issues.apache.org/jira/browse/METRON/?selectedTab=com.atlassian.jira.jira-projects-plugin:summary-panel).\r\n- [x] Does your PR title start with METRON-XXXX where XXXX is the JIRA number you are trying to resolve? Pay particular attention to the hyphen \"-\" character.\r\n- [x] Has your PR been rebased against the latest commit within the target branch (typically master)?\r\n- [x] Have you included steps to reproduce the behavior or problem that is being changed or addressed?\r\n- [x] Have you included steps or a guide to how the change may be verified and tested manually?\r\n- [x] Have you ensured that the full suite of tests and checks have been executed in the root metron folder via:\r\n- [x] Have you written or updated unit tests and or integration tests to verify your changes?\r\n- [x] If adding new dependencies to the code, are these dependencies licensed in a way that is compatible for inclusion under [ASF 2.0](http://www.apache.org/legal/resolved.html#category-a)?\r\n- [x] Have you verified the basic functionality of the build by building and running locally with Vagrant full-dev environment or the equivalent?","created_at":"2018-08-23T18:36:05Z","updated_at":"2018-09-06T13:18:41Z","closed_at":null,"merged_at":null,"merge_commit_sha":"6b59e7bb12b49aa128228361ded01b920d3eba36","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/apache/metron/pulls/1174/commits","review_comments_url":"https://api.github.com/repos/apache/metron/pulls/1174/comments","review_comment_url":"https://api.github.com/repos/apache/metron/pulls/comments{/number}","comments_url":"https://api.github.com/repos/apache/metron/issues/1174/comments","statuses_url":"https://api.github.com/repos/apache/metron/statuses/e70cf943faf0c92abe7f27996a72974e87e80117","head":{"label":"nickwallen:METRON-1748","ref":"METRON-1748","sha":"e70cf943faf0c92abe7f27996a72974e87e80117","user":{"login":"nickwallen","id":2475409,"node_id":"MDQ6VXNlcjI0NzU0MDk=","avatar_url":"https://avatars3.githubusercontent.com/u/2475409?v=4","gravatar_id":"","url":"https://api.github.com/users/nickwallen","html_url":"https://github.com/nickwallen","followers_url":"https://api.github.com/users/nickwallen/followers","following_url":"https://api.github.com/users/nickwallen/following{/other_user}","gists_url":"https://api.github.com/users/nickwallen/gists{/gist_id}","starred_url":"https://api.github.com/users/nickwallen/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/nickwallen/subscriptions","organizations_url":"https://api.github.com/users/nickwallen/orgs","repos_url":"https://api.github.com/users/nickwallen/repos","events_url":"https://api.github.com/users/nickwallen/events{/privacy}","received_events_url":"https://api.github.com/users/nickwallen/received_events","type":"User","site_admin":false},"repo":{"id":51026795,"node_id":"MDEwOlJlcG9zaXRvcnk1MTAyNjc5NQ==","name":"metron","full_name":"nickwallen/metron","owner":{"login":"nickwallen","id":2475409,"node_id":"MDQ6VXNlcjI0NzU0MDk=","avatar_url":"https://avatars3.githubusercontent.com/u/2475409?v=4","gravatar_id":"","url":"https://api.github.com/users/nickwallen","html_url":"https://github.com/nickwallen","followers_url":"https://api.github.com/users/nickwallen/followers","following_url":"https://api.github.com/users/nickwallen/following{/other_user}","gists_url":"https://api.github.com/users/nickwallen/gists{/gist_id}","starred_url":"https://api.github.com/users/nickwallen/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/nickwallen/subscriptions","organizations_url":"https://api.github.com/users/nickwallen/orgs","repos_url":"https://api.github.com/users/nickwallen/repos","events_url":"https://api.github.com/users/nickwallen/events{/privacy}","received_events_url":"https://api.github.com/users/nickwallen/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/nickwallen/metron","description":"Mirror of Apache Metron (Incubating)","fork":true,"url":"https://api.github.com/repos/nickwallen/metron","forks_url":"https://api.github.com/repos/nickwallen/metron/forks","keys_url":"https://api.github.com/repos/nickwallen/metron/keys{/key_id}","collaborators_url":"https://api.github.com/repos/nickwallen/metron/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/nickwallen/metron/teams","hooks_url":"https://api.github.com/repos/nickwallen/metron/hooks","issue_events_url":"https://api.github.com/repos/nickwallen/metron/issues/events{/number}","events_url":"https://api.github.com/repos/nickwallen/metron/events","assignees_url":"https://api.github.com/repos/nickwallen/metron/assignees{/user}","branches_url":"https://api.github.com/repos/nickwallen/metron/branches{/branch}","tags_url":"https://api.github.com/repos/nickwallen/metron/tags","blobs_url":"https://api.github.com/repos/nickwallen/metron/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/nickwallen/metron/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/nickwallen/metron/git/refs{/sha}","trees_url":"https://api.github.com/repos/nickwallen/metron/git/trees{/sha}","statuses_url":"https://api.github.com/repos/nickwallen/metron/statuses/{sha}","languages_url":"https://api.github.com/repos/nickwallen/metron/languages","stargazers_url":"https://api.github.com/repos/nickwallen/metron/stargazers","contributors_url":"https://api.github.com/repos/nickwallen/metron/contributors","subscribers_url":"https://api.github.com/repos/nickwallen/metron/subscribers","subscription_url":"https://api.github.com/repos/nickwallen/metron/subscription","commits_url":"https://api.github.com/repos/nickwallen/metron/commits{/sha}","git_commits_url":"https://api.github.com/repos/nickwallen/metron/git/commits{/sha}","comments_url":"https://api.github.com/repos/nickwallen/metron/comments{/number}","issue_comment_url":"https://api.github.com/repos/nickwallen/metron/issues/comments{/number}","contents_url":"https://api.github.com/repos/nickwallen/metron/contents/{+path}","compare_url":"https://api.github.com/repos/nickwallen/metron/compare/{base}...{head}","merges_url":"https://api.github.com/repos/nickwallen/metron/merges","archive_url":"https://api.github.com/repos/nickwallen/metron/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/nickwallen/metron/downloads","issues_url":"https://api.github.com/repos/nickwallen/metron/issues{/number}","pulls_url":"https://api.github.com/repos/nickwallen/metron/pulls{/number}","milestones_url":"https://api.github.com/repos/nickwallen/metron/milestones{/number}","notifications_url":"https://api.github.com/repos/nickwallen/metron/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/nickwallen/metron/labels{/name}","releases_url":"https://api.github.com/repos/nickwallen/metron/releases{/id}","deployments_url":"https://api.github.com/repos/nickwallen/metron/deployments","created_at":"2016-02-03T20:03:41Z","updated_at":"2018-09-04T17:01:42Z","pushed_at":"2018-09-05T20:36:37Z","git_url":"git://github.com/nickwallen/metron.git","ssh_url":"git@github.com:nickwallen/metron.git","clone_url":"https://github.com/nickwallen/metron.git","svn_url":"https://github.com/nickwallen/metron","homepage":null,"size":65341,"stargazers_count":2,"watchers_count":2,"language":"Java","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":0,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0","node_id":"MDc6TGljZW5zZTI="},"forks":0,"open_issues":0,"watchers":2,"default_branch":"master"}},"base":{"label":"apache:master","ref":"master","sha":"d32bd50d43aae87af9ec12d2daea83b2f4eca342","user":{"login":"apache","id":47359,"node_id":"MDEyOk9yZ2FuaXphdGlvbjQ3MzU5","avatar_url":"https://avatars0.githubusercontent.com/u/47359?v=4","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"repo":{"id":47607561,"node_id":"MDEwOlJlcG9zaXRvcnk0NzYwNzU2MQ==","name":"metron","full_name":"apache/metron","owner":{"login":"apache","id":47359,"node_id":"MDEyOk9yZ2FuaXphdGlvbjQ3MzU5","avatar_url":"https://avatars0.githubusercontent.com/u/47359?v=4","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/apache/metron","description":"Mirror of Apache Metron","fork":false,"url":"https://api.github.com/repos/apache/metron","forks_url":"https://api.github.com/repos/apache/metron/forks","keys_url":"https://api.github.com/repos/apache/metron/keys{/key_id}","collaborators_url":"https://api.github.com/repos/apache/metron/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/apache/metron/teams","hooks_url":"https://api.github.com/repos/apache/metron/hooks","issue_events_url":"https://api.github.com/repos/apache/metron/issues/events{/number}","events_url":"https://api.github.com/repos/apache/metron/events","assignees_url":"https://api.github.com/repos/apache/metron/assignees{/user}","branches_url":"https://api.github.com/repos/apache/metron/branches{/branch}","tags_url":"https://api.github.com/repos/apache/metron/tags","blobs_url":"https://api.github.com/repos/apache/metron/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/apache/metron/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/apache/metron/git/refs{/sha}","trees_url":"https://api.github.com/repos/apache/metron/git/trees{/sha}","statuses_url":"https://api.github.com/repos/apache/metron/statuses/{sha}","languages_url":"https://api.github.com/repos/apache/metron/languages","stargazers_url":"https://api.github.com/repos/apache/metron/stargazers","contributors_url":"https://api.github.com/repos/apache/metron/contributors","subscribers_url":"https://api.github.com/repos/apache/metron/subscribers","subscription_url":"https://api.github.com/repos/apache/metron/subscription","commits_url":"https://api.github.com/repos/apache/metron/commits{/sha}","git_commits_url":"https://api.github.com/repos/apache/metron/git/commits{/sha}","comments_url":"https://api.github.com/repos/apache/metron/comments{/number}","issue_comment_url":"https://api.github.com/repos/apache/metron/issues/comments{/number}","contents_url":"https://api.github.com/repos/apache/metron/contents/{+path}","compare_url":"https://api.github.com/repos/apache/metron/compare/{base}...{head}","merges_url":"https://api.github.com/repos/apache/metron/merges","archive_url":"https://api.github.com/repos/apache/metron/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/apache/metron/downloads","issues_url":"https://api.github.com/repos/apache/metron/issues{/number}","pulls_url":"https://api.github.com/repos/apache/metron/pulls{/number}","milestones_url":"https://api.github.com/repos/apache/metron/milestones{/number}","notifications_url":"https://api.github.com/repos/apache/metron/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/apache/metron/labels{/name}","releases_url":"https://api.github.com/repos/apache/metron/releases{/id}","deployments_url":"https://api.github.com/repos/apache/metron/deployments","created_at":"2015-12-08T08:00:05Z","updated_at":"2018-09-04T23:56:22Z","pushed_at":"2018-09-05T20:48:39Z","git_url":"git://github.com/apache/metron.git","ssh_url":"git@github.com:apache/metron.git","clone_url":"https://github.com/apache/metron.git","svn_url":"https://github.com/apache/metron","homepage":"","size":61750,"stargazers_count":506,"watchers_count":506,"language":"Java","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":374,"mirror_url":null,"archived":false,"open_issues_count":28,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0","node_id":"MDc6TGljZW5zZTI="},"forks":374,"open_issues":28,"watchers":506,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/apache/metron/pulls/1174"},"html":{"href":"https://github.com/apache/metron/pull/1174"},"issue":{"href":"https://api.github.com/repos/apache/metron/issues/1174"},"comments":{"href":"https://api.github.com/repos/apache/metron/issues/1174/comments"},"review_comments":{"href":"https://api.github.com/repos/apache/metron/pulls/1174/comments"},"review_comment":{"href":"https://api.github.com/repos/apache/metron/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/apache/metron/pulls/1174/commits"},"statuses":{"href":"https://api.github.com/repos/apache/metron/statuses/e70cf943faf0c92abe7f27996a72974e87e80117"}},"author_association":"CONTRIBUTOR"}}
{"actor":{"display_login":"nickwallen"}}