{"action":"created","comment":{"url":"https://api.github.com/repos/apache/incubator-metron/pulls/comments/78369111","id":78369111,"diff_hunk":"@@ -0,0 +1,211 @@\n+/*\n+ *\n+ *  Licensed to the Apache Software Foundation (ASF) under one\n+ *  or more contributor license agreements.  See the NOTICE file\n+ *  distributed with this work for additional information\n+ *  regarding copyright ownership.  The ASF licenses this file\n+ *  to you under the Apache License, Version 2.0 (the\n+ *  \"License\"); you may not use this file except in compliance\n+ *  with the License.  You may obtain a copy of the License at\n+ *\n+ *      http://www.apache.org/licenses/LICENSE-2.0\n+ *\n+ *  Unless required by applicable law or agreed to in writing, software\n+ *  distributed under the License is distributed on an \"AS IS\" BASIS,\n+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+ *  See the License for the specific language governing permissions and\n+ *  limitations under the License.\n+ *\n+ */\n+\n+package org.apache.metron.profiler.hbase;\n+\n+import org.apache.hadoop.hbase.util.Bytes;\n+import org.apache.metron.profiler.ProfileMeasurement;\n+import org.apache.metron.profiler.ProfilePeriod;\n+\n+import java.nio.ByteBuffer;\n+import java.security.MessageDigest;\n+import java.security.NoSuchAlgorithmException;\n+import java.util.ArrayList;\n+import java.util.List;\n+import java.util.concurrent.TimeUnit;\n+\n+/**\n+ * A RowKeyBuilder that uses a salt to prevent hot-spotting.\n+ *\n+ * Responsible for building the row keys used to store profile data in HBase.  The row key is composed of the following\n+ * fields in the given order.\n+ * <ul>\n+ * <li>salt - A salt that helps prevent hot-spotting.\n+ * <li>profile - The name of the profile.\n+ * <li>entity - The name of the entity being profiled.\n+ * <li>group(s) - The group(s) used to sort the data in HBase. For example, a group may distinguish between weekends and weekdays.\n+ * <li>year - The year based on UTC.\n+ * <li>day of year - The current day within the year based on UTC; [1, 366]\n+ * <li>hour - The hour within the day based on UTC; [0, 23]\n+ * </ul>period - The period within the hour.  The number of periods per hour can be defined by the user; defaults to 4.\n+ */\n+public class SaltyRowKeyBuilder implements RowKeyBuilder {\n+\n+  /**\n+   * A salt can be prepended to the row key to help prevent hot-spotting.  The salt\n+   * divisor is used to generate the salt.  The salt divisor should be roughly equal\n+   * to the number of nodes in the Hbase cluster.\n+   */\n+  private int saltDivisor;\n+\n+  /**\n+   * An hour is divided into multiple periods.  This defines how many periods\n+   * will exist within each given hour.\n+   */\n+  private int periodsPerHour;\n+\n+  public SaltyRowKeyBuilder() {\n+    this.saltDivisor = 1000;\n+    this.periodsPerHour = 4;\n+  }\n+\n+  public SaltyRowKeyBuilder(int saltDivisor, int periodsPerHour) {\n+    this.saltDivisor = saltDivisor;\n+    this.periodsPerHour = periodsPerHour;\n+  }\n+\n+  /**\n+   * Builds a list of row keys necessary to retrieve profile measurements over\n+   * a time horizon.\n+   *\n+   * @param profile The name of the profile.\n+   * @param entity The name of the entity.\n+   * @param groups The group(s) used to sort the profile data.\n+   * @param durationAgo How long ago?\n+   * @param unit The time units of how long ago.\n+   * @return All of the row keys necessary to retrieve the profile measurements.\n+   */\n+  @Override\n+  public List<byte[]> rowKeys(String profile, String entity, List<Object> groups, long durationAgo, TimeUnit unit) {","path":"metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/hbase/SaltyRowKeyBuilder.java","position":86,"original_position":86,"commit_id":"a5eab363b39f7deb85f65414b0ed7090bcbcd5e4","original_commit_id":"a5eab363b39f7deb85f65414b0ed7090bcbcd5e4","user":{"login":"cestella","id":540359,"avatar_url":"https://avatars.githubusercontent.com/u/540359?v=3","gravatar_id":"","url":"https://api.github.com/users/cestella","html_url":"https://github.com/cestella","followers_url":"https://api.github.com/users/cestella/followers","following_url":"https://api.github.com/users/cestella/following{/other_user}","gists_url":"https://api.github.com/users/cestella/gists{/gist_id}","starred_url":"https://api.github.com/users/cestella/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cestella/subscriptions","organizations_url":"https://api.github.com/users/cestella/orgs","repos_url":"https://api.github.com/users/cestella/repos","events_url":"https://api.github.com/users/cestella/events{/privacy}","received_events_url":"https://api.github.com/users/cestella/received_events","type":"User","site_admin":false},"body":"Is there a reason why we do not have a method on RowKeyBuilder which takes an arbitrary start and end time?","created_at":"2016-09-12T13:13:27Z","updated_at":"2016-09-12T13:13:27Z","html_url":"https://github.com/apache/incubator-metron/pull/236#discussion_r78369111","pull_request_url":"https://api.github.com/repos/apache/incubator-metron/pulls/236","_links":{"self":{"href":"https://api.github.com/repos/apache/incubator-metron/pulls/comments/78369111"},"html":{"href":"https://github.com/apache/incubator-metron/pull/236#discussion_r78369111"},"pull_request":{"href":"https://api.github.com/repos/apache/incubator-metron/pulls/236"}}},"pull_request":{"url":"https://api.github.com/repos/apache/incubator-metron/pulls/236","id":83192737,"html_url":"https://github.com/apache/incubator-metron/pull/236","diff_url":"https://github.com/apache/incubator-metron/pull/236.diff","patch_url":"https://github.com/apache/incubator-metron/pull/236.patch","issue_url":"https://api.github.com/repos/apache/incubator-metron/issues/236","number":236,"state":"open","locked":false,"title":"METRON-389 Create Java API to Read Profile Data During Model Scoring","user":{"login":"nickwallen","id":2475409,"avatar_url":"https://avatars.githubusercontent.com/u/2475409?v=3","gravatar_id":"","url":"https://api.github.com/users/nickwallen","html_url":"https://github.com/nickwallen","followers_url":"https://api.github.com/users/nickwallen/followers","following_url":"https://api.github.com/users/nickwallen/following{/other_user}","gists_url":"https://api.github.com/users/nickwallen/gists{/gist_id}","starred_url":"https://api.github.com/users/nickwallen/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/nickwallen/subscriptions","organizations_url":"https://api.github.com/users/nickwallen/orgs","repos_url":"https://api.github.com/users/nickwallen/repos","events_url":"https://api.github.com/users/nickwallen/events{/privacy}","received_events_url":"https://api.github.com/users/nickwallen/received_events","type":"User","site_admin":false},"body":"[METRON-389](https://issues.apache.org/jira/browse/METRON-389)\r\n\r\n### Changes\r\n* Created a Java API for reading profile data written by the Profiler.  This is contained within a new project `metron-analytics/metron-profiler-client`.\r\n* Altered the Profiler to use deterministic row keys so that multiple Gets can fetch the data.  This should scale much better at high volumes.\r\n\r\n### Points of Interest\r\n* `ProfileMeasurement` - Represents a single data point within a Profile.  A Profile is effectively a time series.  To this end a Profile is composed of many ProfileMeasurement values which in aggregate form a time series.  \r\n* `ProfilePeriod` - The Profiler captures one `ProfileMeasurement` each `ProfilePeriod`.  A `ProfilePeriod` will occur at fixed, deterministic points in time.  This allows for efficient retrieval of profile data.\r\n* `RowKeyBuilder` - Builds row keys that can be used to read or write profile data to HBase.\r\n* `ColumnBuilder` - Defines the columns of data stored with a profile measurement.\r\n* `ProfileHBaseMapper` - Defines for the `HBaseBolt` how profile measurements are stored in HBase.  This class leverages a `RowKeyBuilder` and `ColumnBuilder`.\r\n\r\n### Testing\r\nSpin up a Metron install and write some Profile data to HBase.  Read the profile data using `metron-profiler-client`.","created_at":"2016-08-30T00:59:02Z","updated_at":"2016-09-12T13:13:27Z","closed_at":null,"merged_at":null,"merge_commit_sha":"40cad03bb5d739c6b8d922ab991c1b8ab558dd4a","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/apache/incubator-metron/pulls/236/commits","review_comments_url":"https://api.github.com/repos/apache/incubator-metron/pulls/236/comments","review_comment_url":"https://api.github.com/repos/apache/incubator-metron/pulls/comments{/number}","comments_url":"https://api.github.com/repos/apache/incubator-metron/issues/236/comments","statuses_url":"https://api.github.com/repos/apache/incubator-metron/statuses/a5eab363b39f7deb85f65414b0ed7090bcbcd5e4","head":{"label":"nickwallen:METRON-389-FINAL","ref":"METRON-389-FINAL","sha":"a5eab363b39f7deb85f65414b0ed7090bcbcd5e4","user":{"login":"nickwallen","id":2475409,"avatar_url":"https://avatars.githubusercontent.com/u/2475409?v=3","gravatar_id":"","url":"https://api.github.com/users/nickwallen","html_url":"https://github.com/nickwallen","followers_url":"https://api.github.com/users/nickwallen/followers","following_url":"https://api.github.com/users/nickwallen/following{/other_user}","gists_url":"https://api.github.com/users/nickwallen/gists{/gist_id}","starred_url":"https://api.github.com/users/nickwallen/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/nickwallen/subscriptions","organizations_url":"https://api.github.com/users/nickwallen/orgs","repos_url":"https://api.github.com/users/nickwallen/repos","events_url":"https://api.github.com/users/nickwallen/events{/privacy}","received_events_url":"https://api.github.com/users/nickwallen/received_events","type":"User","site_admin":false},"repo":{"id":51026795,"name":"incubator-metron","full_name":"nickwallen/incubator-metron","owner":{"login":"nickwallen","id":2475409,"avatar_url":"https://avatars.githubusercontent.com/u/2475409?v=3","gravatar_id":"","url":"https://api.github.com/users/nickwallen","html_url":"https://github.com/nickwallen","followers_url":"https://api.github.com/users/nickwallen/followers","following_url":"https://api.github.com/users/nickwallen/following{/other_user}","gists_url":"https://api.github.com/users/nickwallen/gists{/gist_id}","starred_url":"https://api.github.com/users/nickwallen/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/nickwallen/subscriptions","organizations_url":"https://api.github.com/users/nickwallen/orgs","repos_url":"https://api.github.com/users/nickwallen/repos","events_url":"https://api.github.com/users/nickwallen/events{/privacy}","received_events_url":"https://api.github.com/users/nickwallen/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/nickwallen/incubator-metron","description":"Mirror of Apache Metron (Incubating)","fork":true,"url":"https://api.github.com/repos/nickwallen/incubator-metron","forks_url":"https://api.github.com/repos/nickwallen/incubator-metron/forks","keys_url":"https://api.github.com/repos/nickwallen/incubator-metron/keys{/key_id}","collaborators_url":"https://api.github.com/repos/nickwallen/incubator-metron/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/nickwallen/incubator-metron/teams","hooks_url":"https://api.github.com/repos/nickwallen/incubator-metron/hooks","issue_events_url":"https://api.github.com/repos/nickwallen/incubator-metron/issues/events{/number}","events_url":"https://api.github.com/repos/nickwallen/incubator-metron/events","assignees_url":"https://api.github.com/repos/nickwallen/incubator-metron/assignees{/user}","branches_url":"https://api.github.com/repos/nickwallen/incubator-metron/branches{/branch}","tags_url":"https://api.github.com/repos/nickwallen/incubator-metron/tags","blobs_url":"https://api.github.com/repos/nickwallen/incubator-metron/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/nickwallen/incubator-metron/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/nickwallen/incubator-metron/git/refs{/sha}","trees_url":"https://api.github.com/repos/nickwallen/incubator-metron/git/trees{/sha}","statuses_url":"https://api.github.com/repos/nickwallen/incubator-metron/statuses/{sha}","languages_url":"https://api.github.com/repos/nickwallen/incubator-metron/languages","stargazers_url":"https://api.github.com/repos/nickwallen/incubator-metron/stargazers","contributors_url":"https://api.github.com/repos/nickwallen/incubator-metron/contributors","subscribers_url":"https://api.github.com/repos/nickwallen/incubator-metron/subscribers","subscription_url":"https://api.github.com/repos/nickwallen/incubator-metron/subscription","commits_url":"https://api.github.com/repos/nickwallen/incubator-metron/commits{/sha}","git_commits_url":"https://api.github.com/repos/nickwallen/incubator-metron/git/commits{/sha}","comments_url":"https://api.github.com/repos/nickwallen/incubator-metron/comments{/number}","issue_comment_url":"https://api.github.com/repos/nickwallen/incubator-metron/issues/comments{/number}","contents_url":"https://api.github.com/repos/nickwallen/incubator-metron/contents/{+path}","compare_url":"https://api.github.com/repos/nickwallen/incubator-metron/compare/{base}...{head}","merges_url":"https://api.github.com/repos/nickwallen/incubator-metron/merges","archive_url":"https://api.github.com/repos/nickwallen/incubator-metron/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/nickwallen/incubator-metron/downloads","issues_url":"https://api.github.com/repos/nickwallen/incubator-metron/issues{/number}","pulls_url":"https://api.github.com/repos/nickwallen/incubator-metron/pulls{/number}","milestones_url":"https://api.github.com/repos/nickwallen/incubator-metron/milestones{/number}","notifications_url":"https://api.github.com/repos/nickwallen/incubator-metron/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/nickwallen/incubator-metron/labels{/name}","releases_url":"https://api.github.com/repos/nickwallen/incubator-metron/releases{/id}","deployments_url":"https://api.github.com/repos/nickwallen/incubator-metron/deployments","created_at":"2016-02-03T20:03:41Z","updated_at":"2016-04-11T16:30:33Z","pushed_at":"2016-09-10T12:11:13Z","git_url":"git://github.com/nickwallen/incubator-metron.git","ssh_url":"git@github.com:nickwallen/incubator-metron.git","clone_url":"https://github.com/nickwallen/incubator-metron.git","svn_url":"https://github.com/nickwallen/incubator-metron","homepage":null,"size":46686,"stargazers_count":2,"watchers_count":2,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":2,"default_branch":"master"}},"base":{"label":"apache:master","ref":"master","sha":"8b623b87f5754edfa0f1ef27b9b740af7b4f6f2e","user":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"repo":{"id":47607561,"name":"incubator-metron","full_name":"apache/incubator-metron","owner":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/apache/incubator-metron","description":"Mirror of Apache Metron (Incubating)","fork":false,"url":"https://api.github.com/repos/apache/incubator-metron","forks_url":"https://api.github.com/repos/apache/incubator-metron/forks","keys_url":"https://api.github.com/repos/apache/incubator-metron/keys{/key_id}","collaborators_url":"https://api.github.com/repos/apache/incubator-metron/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/apache/incubator-metron/teams","hooks_url":"https://api.github.com/repos/apache/incubator-metron/hooks","issue_events_url":"https://api.github.com/repos/apache/incubator-metron/issues/events{/number}","events_url":"https://api.github.com/repos/apache/incubator-metron/events","assignees_url":"https://api.github.com/repos/apache/incubator-metron/assignees{/user}","branches_url":"https://api.github.com/repos/apache/incubator-metron/branches{/branch}","tags_url":"https://api.github.com/repos/apache/incubator-metron/tags","blobs_url":"https://api.github.com/repos/apache/incubator-metron/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/apache/incubator-metron/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/apache/incubator-metron/git/refs{/sha}","trees_url":"https://api.github.com/repos/apache/incubator-metron/git/trees{/sha}","statuses_url":"https://api.github.com/repos/apache/incubator-metron/statuses/{sha}","languages_url":"https://api.github.com/repos/apache/incubator-metron/languages","stargazers_url":"https://api.github.com/repos/apache/incubator-metron/stargazers","contributors_url":"https://api.github.com/repos/apache/incubator-metron/contributors","subscribers_url":"https://api.github.com/repos/apache/incubator-metron/subscribers","subscription_url":"https://api.github.com/repos/apache/incubator-metron/subscription","commits_url":"https://api.github.com/repos/apache/incubator-metron/commits{/sha}","git_commits_url":"https://api.github.com/repos/apache/incubator-metron/git/commits{/sha}","comments_url":"https://api.github.com/repos/apache/incubator-metron/comments{/number}","issue_comment_url":"https://api.github.com/repos/apache/incubator-metron/issues/comments{/number}","contents_url":"https://api.github.com/repos/apache/incubator-metron/contents/{+path}","compare_url":"https://api.github.com/repos/apache/incubator-metron/compare/{base}...{head}","merges_url":"https://api.github.com/repos/apache/incubator-metron/merges","archive_url":"https://api.github.com/repos/apache/incubator-metron/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/apache/incubator-metron/downloads","issues_url":"https://api.github.com/repos/apache/incubator-metron/issues{/number}","pulls_url":"https://api.github.com/repos/apache/incubator-metron/pulls{/number}","milestones_url":"https://api.github.com/repos/apache/incubator-metron/milestones{/number}","notifications_url":"https://api.github.com/repos/apache/incubator-metron/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/apache/incubator-metron/labels{/name}","releases_url":"https://api.github.com/repos/apache/incubator-metron/releases{/id}","deployments_url":"https://api.github.com/repos/apache/incubator-metron/deployments","created_at":"2015-12-08T08:00:05Z","updated_at":"2016-09-10T01:58:46Z","pushed_at":"2016-09-12T12:57:39Z","git_url":"git://github.com/apache/incubator-metron.git","ssh_url":"git@github.com:apache/incubator-metron.git","clone_url":"https://github.com/apache/incubator-metron.git","svn_url":"https://github.com/apache/incubator-metron","homepage":null,"size":48021,"stargazers_count":102,"watchers_count":102,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":109,"mirror_url":"git://git.apache.org/incubator-metron.git","open_issues_count":15,"forks":109,"open_issues":15,"watchers":102,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/apache/incubator-metron/pulls/236"},"html":{"href":"https://github.com/apache/incubator-metron/pull/236"},"issue":{"href":"https://api.github.com/repos/apache/incubator-metron/issues/236"},"comments":{"href":"https://api.github.com/repos/apache/incubator-metron/issues/236/comments"},"review_comments":{"href":"https://api.github.com/repos/apache/incubator-metron/pulls/236/comments"},"review_comment":{"href":"https://api.github.com/repos/apache/incubator-metron/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/apache/incubator-metron/pulls/236/commits"},"statuses":{"href":"https://api.github.com/repos/apache/incubator-metron/statuses/a5eab363b39f7deb85f65414b0ed7090bcbcd5e4"}}}}
{"actor":{"display_login":"cestella"}}