{"action":"created","comment":{"url":"https://api.github.com/repos/linkedin/pinot/pulls/comments/77693993","id":77693993,"diff_hunk":"@@ -0,0 +1,175 @@\n+/**\n+ * Copyright (C) 2014-2016 LinkedIn Corp. (pinot-core@linkedin.com)\n+ *\n+ * Licensed under the Apache License, Version 2.0 (the \"License\");\n+ * you may not use this file except in compliance with the License.\n+ * You may obtain a copy of the License at\n+ *\n+ *         http://www.apache.org/licenses/LICENSE-2.0\n+ *\n+ * Unless required by applicable law or agreed to in writing, software\n+ * distributed under the License is distributed on an \"AS IS\" BASIS,\n+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+ * See the License for the specific language governing permissions and\n+ * limitations under the License.\n+ */\n+package com.linkedin.pinot.core.operator.aggregation.function;\n+\n+import com.clearspring.analytics.stream.cardinality.HyperLogLog;\n+import com.google.common.base.Preconditions;\n+import com.linkedin.pinot.common.Utils;\n+import com.linkedin.pinot.core.operator.aggregation.AggregationResultHolder;\n+import com.linkedin.pinot.core.operator.aggregation.groupby.GroupByResultHolder;\n+import com.linkedin.pinot.core.startree.hll.HllUtil;\n+import java.util.List;\n+\n+\n+/**\n+ * Class to implement the new 'fasthll' aggregation function,\n+ * fasthll takes advantage of pre-aggregated results for fast distinct count estimation.\n+ */\n+public class FastHllAggregationFunction implements AggregationFunction {\n+  private static final String FUNCTION_NAME = AggregationFunctionFactory.FASTHLL_AGGREGATION_FUNCTION;\n+  // TODO: change or not?\n+  private static final ResultDataType RESULT_DATA_TYPE = ResultDataType.HLL_PREAGGREGATED;\n+  private final int hllLog2m;\n+\n+  public FastHllAggregationFunction(int hllLog2m) {\n+    this.hllLog2m = hllLog2m;\n+  }\n+\n+  /**\n+   * Performs 'fasthll' aggregation on the input array.\n+   *\n+   * {@inheritDoc}\n+   *\n+   * @param length\n+   * @param resultHolder\n+   * @param valueArray\n+   */\n+  @Override\n+  public void aggregate(int length, AggregationResultHolder resultHolder, Object... valueArray) {\n+    Preconditions.checkArgument(valueArray.length == 1);\n+    Preconditions.checkArgument(valueArray[0] instanceof String[]);\n+    final String[] values = (String[]) valueArray[0];\n+    Preconditions.checkState(length <= values.length);\n+\n+    HyperLogLog hll = resultHolder.getResult();\n+    if (hll == null) {\n+      hll = new HyperLogLog(hllLog2m);\n+      resultHolder.setValue(hll);\n+    }\n+\n+    for (int i = 0; i < length; i++) {\n+      try {\n+        HyperLogLog value = HllUtil.convertStringToHll(values[i]);\n+        hll.addAll(value);\n+      } catch (Exception e) {\n+        Utils.rethrowException(e);\n+      }\n+    }\n+  }\n+\n+  /**\n+   * {@inheritDoc}\n+   *\n+   * While the interface allows for variable number of valueArrays, we do not support\n+   * multiple columns within one aggregation function right now.\n+   *\n+   * @param length\n+   * @param groupKeys\n+   * @param resultHolder\n+   * @param valueArray\n+   */\n+  @Override\n+  public void aggregateGroupBySV(int length, int[] groupKeys, GroupByResultHolder resultHolder, Object... valueArray) {\n+    Preconditions.checkArgument(valueArray.length == 1);\n+    Preconditions.checkArgument(valueArray[0] instanceof String[]);\n+    final String[] values = (String[]) valueArray[0];\n+    Preconditions.checkState(length <= values.length);\n+\n+    for (int i = 0; i < length; i++) {\n+      int groupKey = groupKeys[i];\n+      HyperLogLog hll = resultHolder.getResult(groupKey);\n+      if (hll == null) {\n+        hll = new HyperLogLog(hllLog2m);\n+        resultHolder.setValueForKey(groupKey, hll);\n+      }\n+      try {\n+        HyperLogLog value = HllUtil.convertStringToHll(values[i]);\n+        hll.addAll(value);\n+      } catch (Exception e) {\n+        Utils.rethrowException(e);\n+      }\n+    }\n+  }\n+\n+  /**\n+   * {@inheritDoc}\n+   *\n+   * @param length\n+   * @param docIdToGroupKeys\n+   * @param resultHolder\n+   * @param valueArray\n+   */\n+  @Override\n+  public void aggregateGroupByMV(int length, int[][] docIdToGroupKeys, GroupByResultHolder resultHolder,\n+      Object... valueArray) {\n+    Preconditions.checkArgument(valueArray.length == 1);\n+    Preconditions.checkArgument(valueArray[0] instanceof String[]);\n+    final String[] values = (String[]) valueArray[0];\n+    Preconditions.checkState(length <= values.length);\n+\n+    for (int i = 0; i < length; i++) {\n+      for (int groupKey : docIdToGroupKeys[i]) {\n+        HyperLogLog hll = resultHolder.getResult(groupKey);\n+        if (hll == null) {\n+          hll = new HyperLogLog(hllLog2m);\n+          resultHolder.setValueForKey(groupKey, hll);\n+        }\n+        try {\n+          HyperLogLog value = HllUtil.convertStringToHll(values[i]);\n+          hll.addAll(value);\n+        } catch (Exception e) {\n+          Utils.rethrowException(e);\n+        }\n+      }\n+    }\n+  }\n+\n+  /**\n+   * {@inheritDoc}\n+   *\n+   * @return\n+   */\n+  @Override\n+  public double getDefaultValue() {\n+    throw new RuntimeException(\"Unsupported method getDefaultValue() for class \" + getClass().getName());\n+  }\n+\n+  /**\n+   * {@inheritDoc}\n+   * @return\n+   */\n+  @Override\n+  public ResultDataType getResultDataType() {\n+    return RESULT_DATA_TYPE;\n+  }\n+\n+  @Override\n+  public String getName() {\n+    return FUNCTION_NAME;\n+  }\n+\n+  /**\n+   * {@inheritDoc}\n+   *\n+   * @param combinedResult\n+   * @return\n+   */\n+  @Override\n+  public Double reduce(List<Object> combinedResult) {","path":"pinot-core/src/main/java/com/linkedin/pinot/core/operator/aggregation/function/FastHllAggregationFunction.java","position":171,"original_position":171,"commit_id":"4225281df9ecbf52de9f4a99152ef5c503efe69f","original_commit_id":"fa5e03b5a8421f41c31e1d017188320e9c51373e","user":{"login":"victor2100","id":1632918,"avatar_url":"https://avatars.githubusercontent.com/u/1632918?v=3","gravatar_id":"","url":"https://api.github.com/users/victor2100","html_url":"https://github.com/victor2100","followers_url":"https://api.github.com/users/victor2100/followers","following_url":"https://api.github.com/users/victor2100/following{/other_user}","gists_url":"https://api.github.com/users/victor2100/gists{/gist_id}","starred_url":"https://api.github.com/users/victor2100/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/victor2100/subscriptions","organizations_url":"https://api.github.com/users/victor2100/orgs","repos_url":"https://api.github.com/users/victor2100/repos","events_url":"https://api.github.com/users/victor2100/events{/privacy}","received_events_url":"https://api.github.com/users/victor2100/received_events","type":"User","site_admin":false},"body":"The reduce part of this new aggregation function is never got called. I think the transition of old aggregation API to new API is not fully completed. Currently the reduce is supported by old API and I think it should work fine for multiple segments. (That's part of the reason we need to keep the old API in system)","created_at":"2016-09-06T18:40:13Z","updated_at":"2016-09-06T18:40:13Z","html_url":"https://github.com/linkedin/pinot/pull/440#discussion_r77693993","pull_request_url":"https://api.github.com/repos/linkedin/pinot/pulls/440","_links":{"self":{"href":"https://api.github.com/repos/linkedin/pinot/pulls/comments/77693993"},"html":{"href":"https://github.com/linkedin/pinot/pull/440#discussion_r77693993"},"pull_request":{"href":"https://api.github.com/repos/linkedin/pinot/pulls/440"}}},"pull_request":{"url":"https://api.github.com/repos/linkedin/pinot/pulls/440","id":82803106,"html_url":"https://github.com/linkedin/pinot/pull/440","diff_url":"https://github.com/linkedin/pinot/pull/440.diff","patch_url":"https://github.com/linkedin/pinot/pull/440.patch","issue_url":"https://api.github.com/repos/linkedin/pinot/issues/440","number":440,"state":"open","locked":false,"title":"startree hll config and aggregation function in query processing","user":{"login":"victor2100","id":1632918,"avatar_url":"https://avatars.githubusercontent.com/u/1632918?v=3","gravatar_id":"","url":"https://api.github.com/users/victor2100","html_url":"https://github.com/victor2100","followers_url":"https://api.github.com/users/victor2100/followers","following_url":"https://api.github.com/users/victor2100/following{/other_user}","gists_url":"https://api.github.com/users/victor2100/gists{/gist_id}","starred_url":"https://api.github.com/users/victor2100/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/victor2100/subscriptions","organizations_url":"https://api.github.com/users/victor2100/orgs","repos_url":"https://api.github.com/users/victor2100/repos","events_url":"https://api.github.com/users/victor2100/events{/privacy}","received_events_url":"https://api.github.com/users/victor2100/received_events","type":"User","site_admin":false},"body":"1. Add hll config in startree metadata.\r\n2. Implement fasthll aggregation function under old and new aggregation function schemes.\r\n3. Allow fasthll in user query to utilize the startree index.\r\n4. Pass SegmentMetadata into AggregationFunctionContext to give hll function enough info.\r\n5. Implement fetch string value inside SingleValueBlockCache.","created_at":"2016-08-25T22:18:34Z","updated_at":"2016-09-06T18:40:13Z","closed_at":null,"merged_at":null,"merge_commit_sha":"200cd57623bc78fbe3c962409daafd0ed2094c17","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/linkedin/pinot/pulls/440/commits","review_comments_url":"https://api.github.com/repos/linkedin/pinot/pulls/440/comments","review_comment_url":"https://api.github.com/repos/linkedin/pinot/pulls/comments{/number}","comments_url":"https://api.github.com/repos/linkedin/pinot/issues/440/comments","statuses_url":"https://api.github.com/repos/linkedin/pinot/statuses/4225281df9ecbf52de9f4a99152ef5c503efe69f","head":{"label":"victor2100:hll_aggr","ref":"hll_aggr","sha":"4225281df9ecbf52de9f4a99152ef5c503efe69f","user":{"login":"victor2100","id":1632918,"avatar_url":"https://avatars.githubusercontent.com/u/1632918?v=3","gravatar_id":"","url":"https://api.github.com/users/victor2100","html_url":"https://github.com/victor2100","followers_url":"https://api.github.com/users/victor2100/followers","following_url":"https://api.github.com/users/victor2100/following{/other_user}","gists_url":"https://api.github.com/users/victor2100/gists{/gist_id}","starred_url":"https://api.github.com/users/victor2100/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/victor2100/subscriptions","organizations_url":"https://api.github.com/users/victor2100/orgs","repos_url":"https://api.github.com/users/victor2100/repos","events_url":"https://api.github.com/users/victor2100/events{/privacy}","received_events_url":"https://api.github.com/users/victor2100/received_events","type":"User","site_admin":false},"repo":{"id":64504291,"name":"pinot","full_name":"victor2100/pinot","owner":{"login":"victor2100","id":1632918,"avatar_url":"https://avatars.githubusercontent.com/u/1632918?v=3","gravatar_id":"","url":"https://api.github.com/users/victor2100","html_url":"https://github.com/victor2100","followers_url":"https://api.github.com/users/victor2100/followers","following_url":"https://api.github.com/users/victor2100/following{/other_user}","gists_url":"https://api.github.com/users/victor2100/gists{/gist_id}","starred_url":"https://api.github.com/users/victor2100/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/victor2100/subscriptions","organizations_url":"https://api.github.com/users/victor2100/orgs","repos_url":"https://api.github.com/users/victor2100/repos","events_url":"https://api.github.com/users/victor2100/events{/privacy}","received_events_url":"https://api.github.com/users/victor2100/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/victor2100/pinot","description":"A realtime distributed OLAP datastore","fork":true,"url":"https://api.github.com/repos/victor2100/pinot","forks_url":"https://api.github.com/repos/victor2100/pinot/forks","keys_url":"https://api.github.com/repos/victor2100/pinot/keys{/key_id}","collaborators_url":"https://api.github.com/repos/victor2100/pinot/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/victor2100/pinot/teams","hooks_url":"https://api.github.com/repos/victor2100/pinot/hooks","issue_events_url":"https://api.github.com/repos/victor2100/pinot/issues/events{/number}","events_url":"https://api.github.com/repos/victor2100/pinot/events","assignees_url":"https://api.github.com/repos/victor2100/pinot/assignees{/user}","branches_url":"https://api.github.com/repos/victor2100/pinot/branches{/branch}","tags_url":"https://api.github.com/repos/victor2100/pinot/tags","blobs_url":"https://api.github.com/repos/victor2100/pinot/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/victor2100/pinot/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/victor2100/pinot/git/refs{/sha}","trees_url":"https://api.github.com/repos/victor2100/pinot/git/trees{/sha}","statuses_url":"https://api.github.com/repos/victor2100/pinot/statuses/{sha}","languages_url":"https://api.github.com/repos/victor2100/pinot/languages","stargazers_url":"https://api.github.com/repos/victor2100/pinot/stargazers","contributors_url":"https://api.github.com/repos/victor2100/pinot/contributors","subscribers_url":"https://api.github.com/repos/victor2100/pinot/subscribers","subscription_url":"https://api.github.com/repos/victor2100/pinot/subscription","commits_url":"https://api.github.com/repos/victor2100/pinot/commits{/sha}","git_commits_url":"https://api.github.com/repos/victor2100/pinot/git/commits{/sha}","comments_url":"https://api.github.com/repos/victor2100/pinot/comments{/number}","issue_comment_url":"https://api.github.com/repos/victor2100/pinot/issues/comments{/number}","contents_url":"https://api.github.com/repos/victor2100/pinot/contents/{+path}","compare_url":"https://api.github.com/repos/victor2100/pinot/compare/{base}...{head}","merges_url":"https://api.github.com/repos/victor2100/pinot/merges","archive_url":"https://api.github.com/repos/victor2100/pinot/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/victor2100/pinot/downloads","issues_url":"https://api.github.com/repos/victor2100/pinot/issues{/number}","pulls_url":"https://api.github.com/repos/victor2100/pinot/pulls{/number}","milestones_url":"https://api.github.com/repos/victor2100/pinot/milestones{/number}","notifications_url":"https://api.github.com/repos/victor2100/pinot/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/victor2100/pinot/labels{/name}","releases_url":"https://api.github.com/repos/victor2100/pinot/releases{/id}","deployments_url":"https://api.github.com/repos/victor2100/pinot/deployments","created_at":"2016-07-29T19:06:22Z","updated_at":"2016-07-29T19:06:30Z","pushed_at":"2016-09-06T18:04:32Z","git_url":"git://github.com/victor2100/pinot.git","ssh_url":"git@github.com:victor2100/pinot.git","clone_url":"https://github.com/victor2100/pinot.git","svn_url":"https://github.com/victor2100/pinot","homepage":"https://github.com/linkedin/pinot/wiki","size":140637,"stargazers_count":0,"watchers_count":0,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"linkedin:master","ref":"master","sha":"aaaffd399dfe1d3c72fc16116e3aa72bed0a9881","user":{"login":"linkedin","id":357098,"avatar_url":"https://avatars.githubusercontent.com/u/357098?v=3","gravatar_id":"","url":"https://api.github.com/users/linkedin","html_url":"https://github.com/linkedin","followers_url":"https://api.github.com/users/linkedin/followers","following_url":"https://api.github.com/users/linkedin/following{/other_user}","gists_url":"https://api.github.com/users/linkedin/gists{/gist_id}","starred_url":"https://api.github.com/users/linkedin/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/linkedin/subscriptions","organizations_url":"https://api.github.com/users/linkedin/orgs","repos_url":"https://api.github.com/users/linkedin/repos","events_url":"https://api.github.com/users/linkedin/events{/privacy}","received_events_url":"https://api.github.com/users/linkedin/received_events","type":"Organization","site_admin":false},"repo":{"id":19961085,"name":"pinot","full_name":"linkedin/pinot","owner":{"login":"linkedin","id":357098,"avatar_url":"https://avatars.githubusercontent.com/u/357098?v=3","gravatar_id":"","url":"https://api.github.com/users/linkedin","html_url":"https://github.com/linkedin","followers_url":"https://api.github.com/users/linkedin/followers","following_url":"https://api.github.com/users/linkedin/following{/other_user}","gists_url":"https://api.github.com/users/linkedin/gists{/gist_id}","starred_url":"https://api.github.com/users/linkedin/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/linkedin/subscriptions","organizations_url":"https://api.github.com/users/linkedin/orgs","repos_url":"https://api.github.com/users/linkedin/repos","events_url":"https://api.github.com/users/linkedin/events{/privacy}","received_events_url":"https://api.github.com/users/linkedin/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/linkedin/pinot","description":"A realtime distributed OLAP datastore","fork":false,"url":"https://api.github.com/repos/linkedin/pinot","forks_url":"https://api.github.com/repos/linkedin/pinot/forks","keys_url":"https://api.github.com/repos/linkedin/pinot/keys{/key_id}","collaborators_url":"https://api.github.com/repos/linkedin/pinot/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/linkedin/pinot/teams","hooks_url":"https://api.github.com/repos/linkedin/pinot/hooks","issue_events_url":"https://api.github.com/repos/linkedin/pinot/issues/events{/number}","events_url":"https://api.github.com/repos/linkedin/pinot/events","assignees_url":"https://api.github.com/repos/linkedin/pinot/assignees{/user}","branches_url":"https://api.github.com/repos/linkedin/pinot/branches{/branch}","tags_url":"https://api.github.com/repos/linkedin/pinot/tags","blobs_url":"https://api.github.com/repos/linkedin/pinot/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/linkedin/pinot/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/linkedin/pinot/git/refs{/sha}","trees_url":"https://api.github.com/repos/linkedin/pinot/git/trees{/sha}","statuses_url":"https://api.github.com/repos/linkedin/pinot/statuses/{sha}","languages_url":"https://api.github.com/repos/linkedin/pinot/languages","stargazers_url":"https://api.github.com/repos/linkedin/pinot/stargazers","contributors_url":"https://api.github.com/repos/linkedin/pinot/contributors","subscribers_url":"https://api.github.com/repos/linkedin/pinot/subscribers","subscription_url":"https://api.github.com/repos/linkedin/pinot/subscription","commits_url":"https://api.github.com/repos/linkedin/pinot/commits{/sha}","git_commits_url":"https://api.github.com/repos/linkedin/pinot/git/commits{/sha}","comments_url":"https://api.github.com/repos/linkedin/pinot/comments{/number}","issue_comment_url":"https://api.github.com/repos/linkedin/pinot/issues/comments{/number}","contents_url":"https://api.github.com/repos/linkedin/pinot/contents/{+path}","compare_url":"https://api.github.com/repos/linkedin/pinot/compare/{base}...{head}","merges_url":"https://api.github.com/repos/linkedin/pinot/merges","archive_url":"https://api.github.com/repos/linkedin/pinot/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/linkedin/pinot/downloads","issues_url":"https://api.github.com/repos/linkedin/pinot/issues{/number}","pulls_url":"https://api.github.com/repos/linkedin/pinot/pulls{/number}","milestones_url":"https://api.github.com/repos/linkedin/pinot/milestones{/number}","notifications_url":"https://api.github.com/repos/linkedin/pinot/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/linkedin/pinot/labels{/name}","releases_url":"https://api.github.com/repos/linkedin/pinot/releases{/id}","deployments_url":"https://api.github.com/repos/linkedin/pinot/deployments","created_at":"2014-05-19T23:27:48Z","updated_at":"2016-09-06T12:47:56Z","pushed_at":"2016-09-06T18:32:53Z","git_url":"git://github.com/linkedin/pinot.git","ssh_url":"git@github.com:linkedin/pinot.git","clone_url":"https://github.com/linkedin/pinot.git","svn_url":"https://github.com/linkedin/pinot","homepage":"https://github.com/linkedin/pinot/wiki","size":143532,"stargazers_count":1340,"watchers_count":1340,"language":"Java","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":219,"mirror_url":null,"open_issues_count":75,"forks":219,"open_issues":75,"watchers":1340,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/linkedin/pinot/pulls/440"},"html":{"href":"https://github.com/linkedin/pinot/pull/440"},"issue":{"href":"https://api.github.com/repos/linkedin/pinot/issues/440"},"comments":{"href":"https://api.github.com/repos/linkedin/pinot/issues/440/comments"},"review_comments":{"href":"https://api.github.com/repos/linkedin/pinot/pulls/440/comments"},"review_comment":{"href":"https://api.github.com/repos/linkedin/pinot/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/linkedin/pinot/pulls/440/commits"},"statuses":{"href":"https://api.github.com/repos/linkedin/pinot/statuses/4225281df9ecbf52de9f4a99152ef5c503efe69f"}}}}
{"actor":{"display_login":"victor2100"}}