{"action":"created","comment":{"url":"https://api.github.com/repos/linkedin/pinot/pulls/comments/83598087","pull_request_review_id":4426124,"id":83598087,"diff_hunk":"@@ -0,0 +1,161 @@\n+/**\n+ * Copyright (C) 2014-2016 LinkedIn Corp. (pinot-core@linkedin.com)\n+ *\n+ * Licensed under the Apache License, Version 2.0 (the \"License\");\n+ * you may not use this file except in compliance with the License.\n+ * You may obtain a copy of the License at\n+ *\n+ *         http://www.apache.org/licenses/LICENSE-2.0\n+ *\n+ * Unless required by applicable law or agreed to in writing, software\n+ * distributed under the License is distributed on an \"AS IS\" BASIS,\n+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+ * See the License for the specific language governing permissions and\n+ * limitations under the License.\n+ */\n+package com.linkedin.pinot.core.operator.transform;\n+\n+import com.google.common.base.Preconditions;\n+import com.linkedin.pinot.common.request.transform.TransformExpressionTree;\n+import com.linkedin.pinot.core.common.Block;\n+import com.linkedin.pinot.core.common.BlockDocIdIterator;\n+import com.linkedin.pinot.core.common.BlockDocIdSet;\n+import com.linkedin.pinot.core.common.BlockId;\n+import com.linkedin.pinot.core.common.Constants;\n+import com.linkedin.pinot.core.indexsegment.IndexSegment;\n+import com.linkedin.pinot.core.operator.BaseOperator;\n+import com.linkedin.pinot.core.operator.MProjectionOperator;\n+import com.linkedin.pinot.core.operator.blocks.IntermediateResultsBlock;\n+import com.linkedin.pinot.core.operator.blocks.ProjectionBlock;\n+import com.linkedin.pinot.core.operator.docidsets.DocIdSetBlock;\n+import com.linkedin.pinot.core.plan.DocIdSetPlanNode;\n+\n+\n+/**\n+ * Class for evaluating transform expressions.\n+ */\n+public class TransformExpressionOperator extends BaseOperator {\n+  private static final String OPERATOR_NAME = \"TransformExpressionOperator\";\n+\n+  private final IndexSegment _indexSegment;\n+  private final MProjectionOperator _projectionOperator;\n+  TransformExpressionEvaluator _expressionEvaluator;\n+  private int _nextBlockCallCounter = 0;\n+\n+  // Reusable thread-local int array to store the doc ids.\n+  private static final ThreadLocal<int[]> REUSABLE_DOC_ID_SET = new ThreadLocal<int[]>() {\n+    @Override\n+    protected int[] initialValue() {\n+      return new int[DocIdSetPlanNode.MAX_DOC_PER_CALL];\n+    }\n+  };\n+\n+  /**\n+   * Constructor for the class\n+   *\n+   * @param indexSegment Segment to process\n+   * @param projectionOperator Projection operator\n+   * @param expressionTree Expression tree to evaluate\n+   */\n+  public TransformExpressionOperator(IndexSegment indexSegment, MProjectionOperator projectionOperator,\n+      TransformExpressionTree expressionTree) {\n+\n+    Preconditions.checkArgument((indexSegment != null));\n+    Preconditions.checkArgument((projectionOperator != null));\n+    Preconditions.checkArgument((expressionTree != null));\n+\n+    _indexSegment = indexSegment;\n+    _projectionOperator = projectionOperator;\n+\n+    _expressionEvaluator = new DefaultExpressionEvaluator(indexSegment, expressionTree);\n+  }\n+\n+  @Override\n+  public Block getNextBlock() {\n+    return getNextBlock(new BlockId(_nextBlockCallCounter++));\n+  }\n+\n+  @Override\n+  public Block getNextBlock(BlockId blockId) {\n+    // Only one block supported, currently.\n+    if (blockId.getId() > 0) {\n+      return null;\n+    }\n+\n+    int numDocsScanned = 0;\n+    final long startTimeMillis = System.currentTimeMillis();\n+\n+    _expressionEvaluator.init();\n+    while (_projectionOperator.nextBlock() != null) {\n+      ProjectionBlock currentBlock = _projectionOperator.getCurrentBlock();\n+      Block docIdSetBlock = currentBlock.getDocIdSetBlock();\n+      numDocsScanned = processBlock(currentBlock, docIdSetBlock, numDocsScanned);","path":"pinot-core/src/main/java/com/linkedin/pinot/core/operator/transform/TransformExpressionOperator.java","position":92,"original_position":92,"commit_id":"d3fb3625e5f4a39aae5c3261cb012dc76fb65ac4","original_commit_id":"d3fb3625e5f4a39aae5c3261cb012dc76fb65ac4","user":{"login":"Jackie-Jiang","id":17555551,"avatar_url":"https://avatars.githubusercontent.com/u/17555551?v=3","gravatar_id":"","url":"https://api.github.com/users/Jackie-Jiang","html_url":"https://github.com/Jackie-Jiang","followers_url":"https://api.github.com/users/Jackie-Jiang/followers","following_url":"https://api.github.com/users/Jackie-Jiang/following{/other_user}","gists_url":"https://api.github.com/users/Jackie-Jiang/gists{/gist_id}","starred_url":"https://api.github.com/users/Jackie-Jiang/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Jackie-Jiang/subscriptions","organizations_url":"https://api.github.com/users/Jackie-Jiang/orgs","repos_url":"https://api.github.com/users/Jackie-Jiang/repos","events_url":"https://api.github.com/users/Jackie-Jiang/events{/privacy}","received_events_url":"https://api.github.com/users/Jackie-Jiang/received_events","type":"User","site_admin":false},"body":"No need to pass in docIdSetBlock because whether it can be used depends on the class of currentBlock.\r\nAlso, no need to pass in numDocsScanned, just return the docsScanned for the currentBlock, and add it to the numDocsScanned.","created_at":"2016-10-17T08:41:15Z","updated_at":"2016-10-17T08:52:54Z","html_url":"https://github.com/linkedin/pinot/pull/648#discussion_r83598087","pull_request_url":"https://api.github.com/repos/linkedin/pinot/pulls/648","_links":{"self":{"href":"https://api.github.com/repos/linkedin/pinot/pulls/comments/83598087"},"html":{"href":"https://github.com/linkedin/pinot/pull/648#discussion_r83598087"},"pull_request":{"href":"https://api.github.com/repos/linkedin/pinot/pulls/648"}}},"pull_request":{"url":"https://api.github.com/repos/linkedin/pinot/pulls/648","id":88203902,"html_url":"https://github.com/linkedin/pinot/pull/648","diff_url":"https://github.com/linkedin/pinot/pull/648.diff","patch_url":"https://github.com/linkedin/pinot/pull/648.patch","issue_url":"https://api.github.com/repos/linkedin/pinot/issues/648","number":648,"state":"open","locked":false,"title":"UDF and Expressions support.","user":{"login":"mayankshriv","id":12615885,"avatar_url":"https://avatars.githubusercontent.com/u/12615885?v=3","gravatar_id":"","url":"https://api.github.com/users/mayankshriv","html_url":"https://github.com/mayankshriv","followers_url":"https://api.github.com/users/mayankshriv/followers","following_url":"https://api.github.com/users/mayankshriv/following{/other_user}","gists_url":"https://api.github.com/users/mayankshriv/gists{/gist_id}","starred_url":"https://api.github.com/users/mayankshriv/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mayankshriv/subscriptions","organizations_url":"https://api.github.com/users/mayankshriv/orgs","repos_url":"https://api.github.com/users/mayankshriv/repos","events_url":"https://api.github.com/users/mayankshriv/events{/privacy}","received_events_url":"https://api.github.com/users/mayankshriv/received_events","type":"User","site_admin":false},"body":"1. Added TransformOperator and TransformExpressionEvaluator\nfunctionalities.\n2. Added basic built-in functions (add/sub/mult/div).\n3. Added Server config for user to specify UDF class.\n4. Added unit test for new code.","created_at":"2016-10-06T06:12:17Z","updated_at":"2016-10-17T08:52:54Z","closed_at":null,"merged_at":null,"merge_commit_sha":"ecba00cd8cf109c1d8c356f3f6563b5926268b4e","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/linkedin/pinot/pulls/648/commits","review_comments_url":"https://api.github.com/repos/linkedin/pinot/pulls/648/comments","review_comment_url":"https://api.github.com/repos/linkedin/pinot/pulls/comments{/number}","comments_url":"https://api.github.com/repos/linkedin/pinot/issues/648/comments","statuses_url":"https://api.github.com/repos/linkedin/pinot/statuses/d3fb3625e5f4a39aae5c3261cb012dc76fb65ac4","head":{"label":"linkedin:udf-pr","ref":"udf-pr","sha":"d3fb3625e5f4a39aae5c3261cb012dc76fb65ac4","user":{"login":"linkedin","id":357098,"avatar_url":"https://avatars.githubusercontent.com/u/357098?v=3","gravatar_id":"","url":"https://api.github.com/users/linkedin","html_url":"https://github.com/linkedin","followers_url":"https://api.github.com/users/linkedin/followers","following_url":"https://api.github.com/users/linkedin/following{/other_user}","gists_url":"https://api.github.com/users/linkedin/gists{/gist_id}","starred_url":"https://api.github.com/users/linkedin/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/linkedin/subscriptions","organizations_url":"https://api.github.com/users/linkedin/orgs","repos_url":"https://api.github.com/users/linkedin/repos","events_url":"https://api.github.com/users/linkedin/events{/privacy}","received_events_url":"https://api.github.com/users/linkedin/received_events","type":"Organization","site_admin":false},"repo":{"id":19961085,"name":"pinot","full_name":"linkedin/pinot","owner":{"login":"linkedin","id":357098,"avatar_url":"https://avatars.githubusercontent.com/u/357098?v=3","gravatar_id":"","url":"https://api.github.com/users/linkedin","html_url":"https://github.com/linkedin","followers_url":"https://api.github.com/users/linkedin/followers","following_url":"https://api.github.com/users/linkedin/following{/other_user}","gists_url":"https://api.github.com/users/linkedin/gists{/gist_id}","starred_url":"https://api.github.com/users/linkedin/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/linkedin/subscriptions","organizations_url":"https://api.github.com/users/linkedin/orgs","repos_url":"https://api.github.com/users/linkedin/repos","events_url":"https://api.github.com/users/linkedin/events{/privacy}","received_events_url":"https://api.github.com/users/linkedin/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/linkedin/pinot","description":"A realtime distributed OLAP datastore","fork":false,"url":"https://api.github.com/repos/linkedin/pinot","forks_url":"https://api.github.com/repos/linkedin/pinot/forks","keys_url":"https://api.github.com/repos/linkedin/pinot/keys{/key_id}","collaborators_url":"https://api.github.com/repos/linkedin/pinot/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/linkedin/pinot/teams","hooks_url":"https://api.github.com/repos/linkedin/pinot/hooks","issue_events_url":"https://api.github.com/repos/linkedin/pinot/issues/events{/number}","events_url":"https://api.github.com/repos/linkedin/pinot/events","assignees_url":"https://api.github.com/repos/linkedin/pinot/assignees{/user}","branches_url":"https://api.github.com/repos/linkedin/pinot/branches{/branch}","tags_url":"https://api.github.com/repos/linkedin/pinot/tags","blobs_url":"https://api.github.com/repos/linkedin/pinot/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/linkedin/pinot/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/linkedin/pinot/git/refs{/sha}","trees_url":"https://api.github.com/repos/linkedin/pinot/git/trees{/sha}","statuses_url":"https://api.github.com/repos/linkedin/pinot/statuses/{sha}","languages_url":"https://api.github.com/repos/linkedin/pinot/languages","stargazers_url":"https://api.github.com/repos/linkedin/pinot/stargazers","contributors_url":"https://api.github.com/repos/linkedin/pinot/contributors","subscribers_url":"https://api.github.com/repos/linkedin/pinot/subscribers","subscription_url":"https://api.github.com/repos/linkedin/pinot/subscription","commits_url":"https://api.github.com/repos/linkedin/pinot/commits{/sha}","git_commits_url":"https://api.github.com/repos/linkedin/pinot/git/commits{/sha}","comments_url":"https://api.github.com/repos/linkedin/pinot/comments{/number}","issue_comment_url":"https://api.github.com/repos/linkedin/pinot/issues/comments{/number}","contents_url":"https://api.github.com/repos/linkedin/pinot/contents/{+path}","compare_url":"https://api.github.com/repos/linkedin/pinot/compare/{base}...{head}","merges_url":"https://api.github.com/repos/linkedin/pinot/merges","archive_url":"https://api.github.com/repos/linkedin/pinot/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/linkedin/pinot/downloads","issues_url":"https://api.github.com/repos/linkedin/pinot/issues{/number}","pulls_url":"https://api.github.com/repos/linkedin/pinot/pulls{/number}","milestones_url":"https://api.github.com/repos/linkedin/pinot/milestones{/number}","notifications_url":"https://api.github.com/repos/linkedin/pinot/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/linkedin/pinot/labels{/name}","releases_url":"https://api.github.com/repos/linkedin/pinot/releases{/id}","deployments_url":"https://api.github.com/repos/linkedin/pinot/deployments","created_at":"2014-05-19T23:27:48Z","updated_at":"2016-10-16T18:09:14Z","pushed_at":"2016-10-15T07:13:59Z","git_url":"git://github.com/linkedin/pinot.git","ssh_url":"git@github.com:linkedin/pinot.git","clone_url":"https://github.com/linkedin/pinot.git","svn_url":"https://github.com/linkedin/pinot","homepage":"https://github.com/linkedin/pinot/wiki","size":146028,"stargazers_count":1351,"watchers_count":1351,"language":"Java","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":225,"mirror_url":null,"open_issues_count":88,"forks":225,"open_issues":88,"watchers":1351,"default_branch":"master"}},"base":{"label":"linkedin:master","ref":"master","sha":"4ad2858ce2a94c88193b22787199ad75bca8c38d","user":{"login":"linkedin","id":357098,"avatar_url":"https://avatars.githubusercontent.com/u/357098?v=3","gravatar_id":"","url":"https://api.github.com/users/linkedin","html_url":"https://github.com/linkedin","followers_url":"https://api.github.com/users/linkedin/followers","following_url":"https://api.github.com/users/linkedin/following{/other_user}","gists_url":"https://api.github.com/users/linkedin/gists{/gist_id}","starred_url":"https://api.github.com/users/linkedin/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/linkedin/subscriptions","organizations_url":"https://api.github.com/users/linkedin/orgs","repos_url":"https://api.github.com/users/linkedin/repos","events_url":"https://api.github.com/users/linkedin/events{/privacy}","received_events_url":"https://api.github.com/users/linkedin/received_events","type":"Organization","site_admin":false},"repo":{"id":19961085,"name":"pinot","full_name":"linkedin/pinot","owner":{"login":"linkedin","id":357098,"avatar_url":"https://avatars.githubusercontent.com/u/357098?v=3","gravatar_id":"","url":"https://api.github.com/users/linkedin","html_url":"https://github.com/linkedin","followers_url":"https://api.github.com/users/linkedin/followers","following_url":"https://api.github.com/users/linkedin/following{/other_user}","gists_url":"https://api.github.com/users/linkedin/gists{/gist_id}","starred_url":"https://api.github.com/users/linkedin/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/linkedin/subscriptions","organizations_url":"https://api.github.com/users/linkedin/orgs","repos_url":"https://api.github.com/users/linkedin/repos","events_url":"https://api.github.com/users/linkedin/events{/privacy}","received_events_url":"https://api.github.com/users/linkedin/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/linkedin/pinot","description":"A realtime distributed OLAP datastore","fork":false,"url":"https://api.github.com/repos/linkedin/pinot","forks_url":"https://api.github.com/repos/linkedin/pinot/forks","keys_url":"https://api.github.com/repos/linkedin/pinot/keys{/key_id}","collaborators_url":"https://api.github.com/repos/linkedin/pinot/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/linkedin/pinot/teams","hooks_url":"https://api.github.com/repos/linkedin/pinot/hooks","issue_events_url":"https://api.github.com/repos/linkedin/pinot/issues/events{/number}","events_url":"https://api.github.com/repos/linkedin/pinot/events","assignees_url":"https://api.github.com/repos/linkedin/pinot/assignees{/user}","branches_url":"https://api.github.com/repos/linkedin/pinot/branches{/branch}","tags_url":"https://api.github.com/repos/linkedin/pinot/tags","blobs_url":"https://api.github.com/repos/linkedin/pinot/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/linkedin/pinot/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/linkedin/pinot/git/refs{/sha}","trees_url":"https://api.github.com/repos/linkedin/pinot/git/trees{/sha}","statuses_url":"https://api.github.com/repos/linkedin/pinot/statuses/{sha}","languages_url":"https://api.github.com/repos/linkedin/pinot/languages","stargazers_url":"https://api.github.com/repos/linkedin/pinot/stargazers","contributors_url":"https://api.github.com/repos/linkedin/pinot/contributors","subscribers_url":"https://api.github.com/repos/linkedin/pinot/subscribers","subscription_url":"https://api.github.com/repos/linkedin/pinot/subscription","commits_url":"https://api.github.com/repos/linkedin/pinot/commits{/sha}","git_commits_url":"https://api.github.com/repos/linkedin/pinot/git/commits{/sha}","comments_url":"https://api.github.com/repos/linkedin/pinot/comments{/number}","issue_comment_url":"https://api.github.com/repos/linkedin/pinot/issues/comments{/number}","contents_url":"https://api.github.com/repos/linkedin/pinot/contents/{+path}","compare_url":"https://api.github.com/repos/linkedin/pinot/compare/{base}...{head}","merges_url":"https://api.github.com/repos/linkedin/pinot/merges","archive_url":"https://api.github.com/repos/linkedin/pinot/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/linkedin/pinot/downloads","issues_url":"https://api.github.com/repos/linkedin/pinot/issues{/number}","pulls_url":"https://api.github.com/repos/linkedin/pinot/pulls{/number}","milestones_url":"https://api.github.com/repos/linkedin/pinot/milestones{/number}","notifications_url":"https://api.github.com/repos/linkedin/pinot/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/linkedin/pinot/labels{/name}","releases_url":"https://api.github.com/repos/linkedin/pinot/releases{/id}","deployments_url":"https://api.github.com/repos/linkedin/pinot/deployments","created_at":"2014-05-19T23:27:48Z","updated_at":"2016-10-16T18:09:14Z","pushed_at":"2016-10-15T07:13:59Z","git_url":"git://github.com/linkedin/pinot.git","ssh_url":"git@github.com:linkedin/pinot.git","clone_url":"https://github.com/linkedin/pinot.git","svn_url":"https://github.com/linkedin/pinot","homepage":"https://github.com/linkedin/pinot/wiki","size":146028,"stargazers_count":1351,"watchers_count":1351,"language":"Java","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":225,"mirror_url":null,"open_issues_count":88,"forks":225,"open_issues":88,"watchers":1351,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/linkedin/pinot/pulls/648"},"html":{"href":"https://github.com/linkedin/pinot/pull/648"},"issue":{"href":"https://api.github.com/repos/linkedin/pinot/issues/648"},"comments":{"href":"https://api.github.com/repos/linkedin/pinot/issues/648/comments"},"review_comments":{"href":"https://api.github.com/repos/linkedin/pinot/pulls/648/comments"},"review_comment":{"href":"https://api.github.com/repos/linkedin/pinot/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/linkedin/pinot/pulls/648/commits"},"statuses":{"href":"https://api.github.com/repos/linkedin/pinot/statuses/d3fb3625e5f4a39aae5c3261cb012dc76fb65ac4"}}}}
{"actor":{"display_login":"Jackie-Jiang"}}