{"action":"created","comment":{"url":"https://api.github.com/repos/linkedin/pinot/pulls/comments/77418095","id":77418095,"diff_hunk":"@@ -0,0 +1,140 @@\n+/**\n+ * Copyright (C) 2014-2016 LinkedIn Corp. (pinot-core@linkedin.com)\n+ *\n+ * Licensed under the Apache License, Version 2.0 (the \"License\");\n+ * you may not use this file except in compliance with the License.\n+ * You may obtain a copy of the License at\n+ *\n+ *         http://www.apache.org/licenses/LICENSE-2.0\n+ *\n+ * Unless required by applicable law or agreed to in writing, software\n+ * distributed under the License is distributed on an \"AS IS\" BASIS,\n+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+ * See the License for the specific language governing permissions and\n+ * limitations under the License.\n+ */\n+package com.linkedin.pinot.core.query.aggregation.function;\n+\n+import com.clearspring.analytics.stream.cardinality.CardinalityMergeException;\n+import com.clearspring.analytics.stream.cardinality.HyperLogLog;\n+import com.linkedin.pinot.common.Utils;\n+import com.linkedin.pinot.common.data.FieldSpec.DataType;\n+import com.linkedin.pinot.common.request.AggregationInfo;\n+import com.linkedin.pinot.core.common.Block;\n+import com.linkedin.pinot.core.query.aggregation.AggregationFunction;\n+import com.linkedin.pinot.core.query.aggregation.CombineLevel;\n+import java.io.Serializable;\n+import java.util.List;\n+import org.json.JSONException;\n+import org.json.JSONObject;\n+import org.slf4j.Logger;\n+import org.slf4j.LoggerFactory;\n+\n+/**\n+ * Class to implement the old 'fasthll' aggregation function,\n+ * only combine and reduce related methods are used,\n+ * calling other methods will throw UnsupportedOperationException.\n+ *\n+ * fasthll takes advantage of pre-aggregated results for fast distinct count estimation\n+ *\n+ * TODO: fully deprecate and remove this class when new aggregation routine is finished.\n+ */\n+public class FastHllAggregationFunction implements AggregationFunction<HyperLogLog, Long> {\n+  private static final Logger LOGGER = LoggerFactory.getLogger(FastHllAggregationFunction.class);\n+\n+  private String _distinctCountHLLColumnName;\n+\n+  @Override\n+  public void init(AggregationInfo aggregationInfo) {\n+    _distinctCountHLLColumnName = aggregationInfo.getAggregationParams().get(\"column\");\n+  }\n+\n+  @Override\n+  public HyperLogLog aggregate(Block docIdSetBlock, Block[] block) {\n+    throw new UnsupportedOperationException(\"Calling old aggregate method of FastHllAggregationFunction is deprecated.\");\n+  }\n+\n+  @Override\n+  public HyperLogLog aggregate(HyperLogLog mergedResult, int docId, Block[] block) {\n+    throw new UnsupportedOperationException(\"Calling old aggregate method of FastHllAggregationFunction is deprecated.\");\n+  }\n+\n+  @Override\n+  public List<HyperLogLog> combine(List<HyperLogLog> aggregationResultList, CombineLevel combineLevel) {\n+    if ((aggregationResultList == null) || aggregationResultList.isEmpty()) {\n+      return null;\n+    }\n+    HyperLogLog hllResult = aggregationResultList.get(0);\n+    for (int i = 1; i < aggregationResultList.size(); ++i) {\n+      try {\n+        hllResult.addAll(aggregationResultList.get(i));","path":"pinot-core/src/main/java/com/linkedin/pinot/core/query/aggregation/function/FastHllAggregationFunction.java","position":70,"original_position":70,"commit_id":"24e3e9402676e14d81d6e13e75b935214367c2c0","original_commit_id":"5de7046f58a7018fbd7c9cabe2251d90c05d8e30","user":{"login":"victor2100","id":1632918,"avatar_url":"https://avatars.githubusercontent.com/u/1632918?v=3","gravatar_id":"","url":"https://api.github.com/users/victor2100","html_url":"https://github.com/victor2100","followers_url":"https://api.github.com/users/victor2100/followers","following_url":"https://api.github.com/users/victor2100/following{/other_user}","gists_url":"https://api.github.com/users/victor2100/gists{/gist_id}","starred_url":"https://api.github.com/users/victor2100/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/victor2100/subscriptions","organizations_url":"https://api.github.com/users/victor2100/orgs","repos_url":"https://api.github.com/users/victor2100/repos","events_url":"https://api.github.com/users/victor2100/events{/privacy}","received_events_url":"https://api.github.com/users/victor2100/received_events","type":"User","site_admin":false},"body":"This is the old version aggregation function, it just mimic the structure of combine of old distinctcounthll and other functions to have a pre-check of the null value. I'm not sure it's absolutely necessary, but it's always safe to do that.","created_at":"2016-09-02T22:05:02Z","updated_at":"2016-09-02T22:05:02Z","html_url":"https://github.com/linkedin/pinot/pull/440#discussion_r77418095","pull_request_url":"https://api.github.com/repos/linkedin/pinot/pulls/440","_links":{"self":{"href":"https://api.github.com/repos/linkedin/pinot/pulls/comments/77418095"},"html":{"href":"https://github.com/linkedin/pinot/pull/440#discussion_r77418095"},"pull_request":{"href":"https://api.github.com/repos/linkedin/pinot/pulls/440"}}},"pull_request":{"url":"https://api.github.com/repos/linkedin/pinot/pulls/440","id":82803106,"html_url":"https://github.com/linkedin/pinot/pull/440","diff_url":"https://github.com/linkedin/pinot/pull/440.diff","patch_url":"https://github.com/linkedin/pinot/pull/440.patch","issue_url":"https://api.github.com/repos/linkedin/pinot/issues/440","number":440,"state":"open","locked":false,"title":"startree hll config and aggregation function in query processing","user":{"login":"victor2100","id":1632918,"avatar_url":"https://avatars.githubusercontent.com/u/1632918?v=3","gravatar_id":"","url":"https://api.github.com/users/victor2100","html_url":"https://github.com/victor2100","followers_url":"https://api.github.com/users/victor2100/followers","following_url":"https://api.github.com/users/victor2100/following{/other_user}","gists_url":"https://api.github.com/users/victor2100/gists{/gist_id}","starred_url":"https://api.github.com/users/victor2100/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/victor2100/subscriptions","organizations_url":"https://api.github.com/users/victor2100/orgs","repos_url":"https://api.github.com/users/victor2100/repos","events_url":"https://api.github.com/users/victor2100/events{/privacy}","received_events_url":"https://api.github.com/users/victor2100/received_events","type":"User","site_admin":false},"body":"1. Add hll config in startree metadata.\r\n2. Implement fasthll aggregation function under old and new aggregation function schemes.\r\n3. Allow fasthll in user query to utilize the startree index.\r\n4. Pass SegmentMetadata into AggregationFunctionContext to give hll function enough info.\r\n5. Implement fetch string value inside SingleValueBlockCache.","created_at":"2016-08-25T22:18:34Z","updated_at":"2016-09-02T22:05:02Z","closed_at":null,"merged_at":null,"merge_commit_sha":"86a241b1a47b3c436b0418acd98b30df9c949193","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/linkedin/pinot/pulls/440/commits","review_comments_url":"https://api.github.com/repos/linkedin/pinot/pulls/440/comments","review_comment_url":"https://api.github.com/repos/linkedin/pinot/pulls/comments{/number}","comments_url":"https://api.github.com/repos/linkedin/pinot/issues/440/comments","statuses_url":"https://api.github.com/repos/linkedin/pinot/statuses/24e3e9402676e14d81d6e13e75b935214367c2c0","head":{"label":"victor2100:hll_aggr","ref":"hll_aggr","sha":"24e3e9402676e14d81d6e13e75b935214367c2c0","user":{"login":"victor2100","id":1632918,"avatar_url":"https://avatars.githubusercontent.com/u/1632918?v=3","gravatar_id":"","url":"https://api.github.com/users/victor2100","html_url":"https://github.com/victor2100","followers_url":"https://api.github.com/users/victor2100/followers","following_url":"https://api.github.com/users/victor2100/following{/other_user}","gists_url":"https://api.github.com/users/victor2100/gists{/gist_id}","starred_url":"https://api.github.com/users/victor2100/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/victor2100/subscriptions","organizations_url":"https://api.github.com/users/victor2100/orgs","repos_url":"https://api.github.com/users/victor2100/repos","events_url":"https://api.github.com/users/victor2100/events{/privacy}","received_events_url":"https://api.github.com/users/victor2100/received_events","type":"User","site_admin":false},"repo":{"id":64504291,"name":"pinot","full_name":"victor2100/pinot","owner":{"login":"victor2100","id":1632918,"avatar_url":"https://avatars.githubusercontent.com/u/1632918?v=3","gravatar_id":"","url":"https://api.github.com/users/victor2100","html_url":"https://github.com/victor2100","followers_url":"https://api.github.com/users/victor2100/followers","following_url":"https://api.github.com/users/victor2100/following{/other_user}","gists_url":"https://api.github.com/users/victor2100/gists{/gist_id}","starred_url":"https://api.github.com/users/victor2100/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/victor2100/subscriptions","organizations_url":"https://api.github.com/users/victor2100/orgs","repos_url":"https://api.github.com/users/victor2100/repos","events_url":"https://api.github.com/users/victor2100/events{/privacy}","received_events_url":"https://api.github.com/users/victor2100/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/victor2100/pinot","description":"A realtime distributed OLAP datastore","fork":true,"url":"https://api.github.com/repos/victor2100/pinot","forks_url":"https://api.github.com/repos/victor2100/pinot/forks","keys_url":"https://api.github.com/repos/victor2100/pinot/keys{/key_id}","collaborators_url":"https://api.github.com/repos/victor2100/pinot/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/victor2100/pinot/teams","hooks_url":"https://api.github.com/repos/victor2100/pinot/hooks","issue_events_url":"https://api.github.com/repos/victor2100/pinot/issues/events{/number}","events_url":"https://api.github.com/repos/victor2100/pinot/events","assignees_url":"https://api.github.com/repos/victor2100/pinot/assignees{/user}","branches_url":"https://api.github.com/repos/victor2100/pinot/branches{/branch}","tags_url":"https://api.github.com/repos/victor2100/pinot/tags","blobs_url":"https://api.github.com/repos/victor2100/pinot/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/victor2100/pinot/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/victor2100/pinot/git/refs{/sha}","trees_url":"https://api.github.com/repos/victor2100/pinot/git/trees{/sha}","statuses_url":"https://api.github.com/repos/victor2100/pinot/statuses/{sha}","languages_url":"https://api.github.com/repos/victor2100/pinot/languages","stargazers_url":"https://api.github.com/repos/victor2100/pinot/stargazers","contributors_url":"https://api.github.com/repos/victor2100/pinot/contributors","subscribers_url":"https://api.github.com/repos/victor2100/pinot/subscribers","subscription_url":"https://api.github.com/repos/victor2100/pinot/subscription","commits_url":"https://api.github.com/repos/victor2100/pinot/commits{/sha}","git_commits_url":"https://api.github.com/repos/victor2100/pinot/git/commits{/sha}","comments_url":"https://api.github.com/repos/victor2100/pinot/comments{/number}","issue_comment_url":"https://api.github.com/repos/victor2100/pinot/issues/comments{/number}","contents_url":"https://api.github.com/repos/victor2100/pinot/contents/{+path}","compare_url":"https://api.github.com/repos/victor2100/pinot/compare/{base}...{head}","merges_url":"https://api.github.com/repos/victor2100/pinot/merges","archive_url":"https://api.github.com/repos/victor2100/pinot/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/victor2100/pinot/downloads","issues_url":"https://api.github.com/repos/victor2100/pinot/issues{/number}","pulls_url":"https://api.github.com/repos/victor2100/pinot/pulls{/number}","milestones_url":"https://api.github.com/repos/victor2100/pinot/milestones{/number}","notifications_url":"https://api.github.com/repos/victor2100/pinot/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/victor2100/pinot/labels{/name}","releases_url":"https://api.github.com/repos/victor2100/pinot/releases{/id}","deployments_url":"https://api.github.com/repos/victor2100/pinot/deployments","created_at":"2016-07-29T19:06:22Z","updated_at":"2016-07-29T19:06:30Z","pushed_at":"2016-09-02T21:59:27Z","git_url":"git://github.com/victor2100/pinot.git","ssh_url":"git@github.com:victor2100/pinot.git","clone_url":"https://github.com/victor2100/pinot.git","svn_url":"https://github.com/victor2100/pinot","homepage":"https://github.com/linkedin/pinot/wiki","size":140549,"stargazers_count":0,"watchers_count":0,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"linkedin:master","ref":"master","sha":"8920e3ee299c4e098b782d670a899169bc956184","user":{"login":"linkedin","id":357098,"avatar_url":"https://avatars.githubusercontent.com/u/357098?v=3","gravatar_id":"","url":"https://api.github.com/users/linkedin","html_url":"https://github.com/linkedin","followers_url":"https://api.github.com/users/linkedin/followers","following_url":"https://api.github.com/users/linkedin/following{/other_user}","gists_url":"https://api.github.com/users/linkedin/gists{/gist_id}","starred_url":"https://api.github.com/users/linkedin/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/linkedin/subscriptions","organizations_url":"https://api.github.com/users/linkedin/orgs","repos_url":"https://api.github.com/users/linkedin/repos","events_url":"https://api.github.com/users/linkedin/events{/privacy}","received_events_url":"https://api.github.com/users/linkedin/received_events","type":"Organization","site_admin":false},"repo":{"id":19961085,"name":"pinot","full_name":"linkedin/pinot","owner":{"login":"linkedin","id":357098,"avatar_url":"https://avatars.githubusercontent.com/u/357098?v=3","gravatar_id":"","url":"https://api.github.com/users/linkedin","html_url":"https://github.com/linkedin","followers_url":"https://api.github.com/users/linkedin/followers","following_url":"https://api.github.com/users/linkedin/following{/other_user}","gists_url":"https://api.github.com/users/linkedin/gists{/gist_id}","starred_url":"https://api.github.com/users/linkedin/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/linkedin/subscriptions","organizations_url":"https://api.github.com/users/linkedin/orgs","repos_url":"https://api.github.com/users/linkedin/repos","events_url":"https://api.github.com/users/linkedin/events{/privacy}","received_events_url":"https://api.github.com/users/linkedin/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/linkedin/pinot","description":"A realtime distributed OLAP datastore","fork":false,"url":"https://api.github.com/repos/linkedin/pinot","forks_url":"https://api.github.com/repos/linkedin/pinot/forks","keys_url":"https://api.github.com/repos/linkedin/pinot/keys{/key_id}","collaborators_url":"https://api.github.com/repos/linkedin/pinot/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/linkedin/pinot/teams","hooks_url":"https://api.github.com/repos/linkedin/pinot/hooks","issue_events_url":"https://api.github.com/repos/linkedin/pinot/issues/events{/number}","events_url":"https://api.github.com/repos/linkedin/pinot/events","assignees_url":"https://api.github.com/repos/linkedin/pinot/assignees{/user}","branches_url":"https://api.github.com/repos/linkedin/pinot/branches{/branch}","tags_url":"https://api.github.com/repos/linkedin/pinot/tags","blobs_url":"https://api.github.com/repos/linkedin/pinot/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/linkedin/pinot/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/linkedin/pinot/git/refs{/sha}","trees_url":"https://api.github.com/repos/linkedin/pinot/git/trees{/sha}","statuses_url":"https://api.github.com/repos/linkedin/pinot/statuses/{sha}","languages_url":"https://api.github.com/repos/linkedin/pinot/languages","stargazers_url":"https://api.github.com/repos/linkedin/pinot/stargazers","contributors_url":"https://api.github.com/repos/linkedin/pinot/contributors","subscribers_url":"https://api.github.com/repos/linkedin/pinot/subscribers","subscription_url":"https://api.github.com/repos/linkedin/pinot/subscription","commits_url":"https://api.github.com/repos/linkedin/pinot/commits{/sha}","git_commits_url":"https://api.github.com/repos/linkedin/pinot/git/commits{/sha}","comments_url":"https://api.github.com/repos/linkedin/pinot/comments{/number}","issue_comment_url":"https://api.github.com/repos/linkedin/pinot/issues/comments{/number}","contents_url":"https://api.github.com/repos/linkedin/pinot/contents/{+path}","compare_url":"https://api.github.com/repos/linkedin/pinot/compare/{base}...{head}","merges_url":"https://api.github.com/repos/linkedin/pinot/merges","archive_url":"https://api.github.com/repos/linkedin/pinot/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/linkedin/pinot/downloads","issues_url":"https://api.github.com/repos/linkedin/pinot/issues{/number}","pulls_url":"https://api.github.com/repos/linkedin/pinot/pulls{/number}","milestones_url":"https://api.github.com/repos/linkedin/pinot/milestones{/number}","notifications_url":"https://api.github.com/repos/linkedin/pinot/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/linkedin/pinot/labels{/name}","releases_url":"https://api.github.com/repos/linkedin/pinot/releases{/id}","deployments_url":"https://api.github.com/repos/linkedin/pinot/deployments","created_at":"2014-05-19T23:27:48Z","updated_at":"2016-09-02T05:59:10Z","pushed_at":"2016-09-02T21:59:29Z","git_url":"git://github.com/linkedin/pinot.git","ssh_url":"git@github.com:linkedin/pinot.git","clone_url":"https://github.com/linkedin/pinot.git","svn_url":"https://github.com/linkedin/pinot","homepage":"https://github.com/linkedin/pinot/wiki","size":143390,"stargazers_count":1337,"watchers_count":1337,"language":"Java","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":217,"mirror_url":null,"open_issues_count":78,"forks":217,"open_issues":78,"watchers":1337,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/linkedin/pinot/pulls/440"},"html":{"href":"https://github.com/linkedin/pinot/pull/440"},"issue":{"href":"https://api.github.com/repos/linkedin/pinot/issues/440"},"comments":{"href":"https://api.github.com/repos/linkedin/pinot/issues/440/comments"},"review_comments":{"href":"https://api.github.com/repos/linkedin/pinot/pulls/440/comments"},"review_comment":{"href":"https://api.github.com/repos/linkedin/pinot/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/linkedin/pinot/pulls/440/commits"},"statuses":{"href":"https://api.github.com/repos/linkedin/pinot/statuses/24e3e9402676e14d81d6e13e75b935214367c2c0"}}}}
{"actor":{"display_login":"victor2100"}}