{"action":"created","comment":{"url":"https://api.github.com/repos/apache/carbondata/pulls/comments/178568134","pull_request_review_id":108644432,"id":178568134,"diff_hunk":"@@ -0,0 +1,67 @@\n+/*\n+ * Licensed to the Apache Software Foundation (ASF) under one or more\n+ * contributor license agreements.  See the NOTICE file distributed with\n+ * this work for additional information regarding copyright ownership.\n+ * The ASF licenses this file to You under the Apache License, Version 2.0\n+ * (the \"License\"); you may not use this file except in compliance with\n+ * the License.  You may obtain a copy of the License at\n+ *\n+ *    http://www.apache.org/licenses/LICENSE-2.0\n+ *\n+ * Unless required by applicable law or agreed to in writing, software\n+ * distributed under the License is distributed on an \"AS IS\" BASIS,\n+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+ * See the License for the specific language governing permissions and\n+ * limitations under the License.\n+ */\n+package org.apache.carbondata.core.scan.executor.impl;\n+\n+import java.io.IOException;\n+import java.util.List;\n+import java.util.concurrent.ExecutorService;\n+import java.util.concurrent.Executors;\n+\n+import org.apache.carbondata.common.CarbonIterator;\n+import org.apache.carbondata.common.logging.LogService;\n+import org.apache.carbondata.common.logging.LogServiceFactory;\n+import org.apache.carbondata.core.constants.CarbonCommonConstants;\n+import org.apache.carbondata.core.scan.executor.exception.QueryExecutionException;\n+import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;\n+import org.apache.carbondata.core.scan.model.QueryModel;\n+import org.apache.carbondata.core.scan.result.iterator.SearchModeResultIterator;\n+import org.apache.carbondata.core.util.CarbonProperties;\n+\n+/**\n+ * Below class will be used to execute the detail query and returns columnar vectors.\n+ */\n+public class SearchModeVectorDetailQueryExecutor extends AbstractQueryExecutor<Object> {\n+  private static final LogService LOGGER =\n+          LogServiceFactory.getLogService(SearchModeVectorDetailQueryExecutor.class.getName());\n+  private static ExecutorService executorService;\n+\n+  static {\n+    int nThread;\n+    try {\n+      nThread = Integer.parseInt(CarbonProperties.getInstance()\n+              .getProperty(CarbonCommonConstants.CARBON_SEARCH_MODE_THREAD,\n+                      CarbonCommonConstants.CARBON_SEARCH_MODE_THREAD_DEFAULT));\n+    } catch (NumberFormatException e) {\n+      nThread = Integer.parseInt(CarbonCommonConstants.CARBON_SEARCH_MODE_THREAD_DEFAULT);\n+      LOGGER.warn(\"The carbon.search.mode.thread is invalid. Using the default value \" + nThread);\n+    }\n+    executorService = Executors.newFixedThreadPool(nThread);","path":"core/src/main/java/org/apache/carbondata/core/scan/executor/impl/SearchModeVectorDetailQueryExecutor.java","position":52,"original_position":52,"commit_id":"58515280ac8563aab3b29e476320d947121ee1e9","original_commit_id":"58515280ac8563aab3b29e476320d947121ee1e9","user":{"login":"kevinjmh","id":3809732,"avatar_url":"https://avatars1.githubusercontent.com/u/3809732?v=4","gravatar_id":"","url":"https://api.github.com/users/kevinjmh","html_url":"https://github.com/kevinjmh","followers_url":"https://api.github.com/users/kevinjmh/followers","following_url":"https://api.github.com/users/kevinjmh/following{/other_user}","gists_url":"https://api.github.com/users/kevinjmh/gists{/gist_id}","starred_url":"https://api.github.com/users/kevinjmh/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/kevinjmh/subscriptions","organizations_url":"https://api.github.com/users/kevinjmh/orgs","repos_url":"https://api.github.com/users/kevinjmh/repos","events_url":"https://api.github.com/users/kevinjmh/events{/privacy}","received_events_url":"https://api.github.com/users/kevinjmh/received_events","type":"User","site_admin":false},"body":"Setting number of threads is a way to limit the parallelism in case of dealing huge number of splits in a task. As for static ExecutorService, it can avoid to create a new thread pool for every query and keep the thread pool standby for next query. ","created_at":"2018-04-02T15:27:32Z","updated_at":"2018-04-02T15:27:32Z","html_url":"https://github.com/apache/carbondata/pull/2123#discussion_r178568134","pull_request_url":"https://api.github.com/repos/apache/carbondata/pulls/2123","author_association":"CONTRIBUTOR","_links":{"self":{"href":"https://api.github.com/repos/apache/carbondata/pulls/comments/178568134"},"html":{"href":"https://github.com/apache/carbondata/pull/2123#discussion_r178568134"},"pull_request":{"href":"https://api.github.com/repos/apache/carbondata/pulls/2123"}},"in_reply_to_id":178560655},"pull_request":{"url":"https://api.github.com/repos/apache/carbondata/pulls/2123","id":178664167,"html_url":"https://github.com/apache/carbondata/pull/2123","diff_url":"https://github.com/apache/carbondata/pull/2123.diff","patch_url":"https://github.com/apache/carbondata/pull/2123.patch","issue_url":"https://api.github.com/repos/apache/carbondata/issues/2123","number":2123,"state":"open","locked":false,"title":"[CARBONDATA-2297] Support SEARCH_MODE for basic filter query","user":{"login":"kevinjmh","id":3809732,"avatar_url":"https://avatars1.githubusercontent.com/u/3809732?v=4","gravatar_id":"","url":"https://api.github.com/users/kevinjmh","html_url":"https://github.com/kevinjmh","followers_url":"https://api.github.com/users/kevinjmh/followers","following_url":"https://api.github.com/users/kevinjmh/following{/other_user}","gists_url":"https://api.github.com/users/kevinjmh/gists{/gist_id}","starred_url":"https://api.github.com/users/kevinjmh/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/kevinjmh/subscriptions","organizations_url":"https://api.github.com/users/kevinjmh/orgs","repos_url":"https://api.github.com/users/kevinjmh/repos","events_url":"https://api.github.com/users/kevinjmh/events{/privacy}","received_events_url":"https://api.github.com/users/kevinjmh/received_events","type":"User","site_admin":false},"body":"Be sure to do all of the following checklist to help us incorporate \r\nyour contribution quickly and easily:\r\n\r\n - [ ] Any interfaces changed?\r\n \r\n - [ ] Any backward compatibility impacted?\r\n \r\n - [ ] Document update required?\r\n\r\n - [x] Testing done\r\n        Please provide details on \r\n        - Whether new unit test cases have been added or why no new tests are required?\r\n        - How it is tested? Please attach test report.\r\n        - Is it a performance related change? Please attach the performance test report.\r\n        - Any additional information to help reviewers in testing this change.\r\n       \r\n - [ ] For large changes, please consider breaking it into sub-tasks under an umbrella JIRA. \r\n\r\n---\r\n\r\n### Testing\r\n\r\n#### Environment: \r\n3 nodes, 20 cores per node\r\n\r\n#### Query Template:\r\n`SELECT col1 FROM table1 WHERE col2 = \"sometext\"`\r\n\r\n#### Test Result\r\nAfter running serveral tests, we find that search mode performs well if filter column is not in sort columns which can easily make `count(task) > count(available core)`(spark.task.cpus=1)*(See Test 1.1/1.3/2.1)* . On the other hand, it may degrade the query performance a bit if filter column is in sort columns *(See Test 1.2/1.4/2.2)*, but superiority shows as block hit increasing *(See Test 2.2/2.3/2.4)*\r\n\r\nDetail result as followed:\r\n##### 1. Interactive Query Test (Single query at a time)\r\n> Test 1.3&1.4 simply reload the data of Test 1.1&1.2 to add new segment;\r\n\r\n|Test No. | Filter column is SortColumn | Block Hit |  \\# of Task (origin way) | Background Query Time (origin way) | \\# of Task (search mode) | Background Query Time (search mode) |\r\n|-|-|-|-|-|-|-|\r\n| 1.1 | N | 85 | 85 | 0.9~1s | 3 | 0.5~0.6s |\r\n| 1.2 | Y | 3 | 3 | 71~76ms | 3 | 67~81ms |\r\n| 1.3 | N | 170 | 170 | 2s | 3 | 1s |\r\n| 1.4 | Y | 6 | 6 | 74~92ms | 3 | 88ms~0.1s |\r\n\r\n\r\n##### 2. Concurrent Query Test (20 queries at a time)\r\n\r\n|Test No. | Filter column is SortColumn | Block Hit |  \\# of Task (origin way) | Background Query Time(ms) (origin way) | \\# of Task (search mode) | Background Query Time(ms) (search mode) | Description |\r\n|-|-|-|-|-|-|-|-|\r\n| 2.1 | N | 85 | 85 | 22399 | 3 | 12560 | - |\r\n| 2.2 | Y | 3 | 3 | 6557 | 3 | 6915 | data is same as 2.1 |\r\n| 2.3 | Y | 6 | 6| 6863 | 3 | 7034 | reload data to add new segment |\r\n| 2.4 | Y | 24 | 24 | 7468 | 3 | 6929 | divide data of 2.1 to 8 parts and load seperately | \r\n\r\n\r\n\r\n\r\n","created_at":"2018-03-31T03:38:25Z","updated_at":"2018-04-02T15:27:32Z","closed_at":null,"merged_at":null,"merge_commit_sha":"b19bf1959c9e859a47a160405e347d9cafdb97b9","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/apache/carbondata/pulls/2123/commits","review_comments_url":"https://api.github.com/repos/apache/carbondata/pulls/2123/comments","review_comment_url":"https://api.github.com/repos/apache/carbondata/pulls/comments{/number}","comments_url":"https://api.github.com/repos/apache/carbondata/issues/2123/comments","statuses_url":"https://api.github.com/repos/apache/carbondata/statuses/58515280ac8563aab3b29e476320d947121ee1e9","head":{"label":"kevinjmh:searchmode","ref":"searchmode","sha":"58515280ac8563aab3b29e476320d947121ee1e9","user":{"login":"kevinjmh","id":3809732,"avatar_url":"https://avatars1.githubusercontent.com/u/3809732?v=4","gravatar_id":"","url":"https://api.github.com/users/kevinjmh","html_url":"https://github.com/kevinjmh","followers_url":"https://api.github.com/users/kevinjmh/followers","following_url":"https://api.github.com/users/kevinjmh/following{/other_user}","gists_url":"https://api.github.com/users/kevinjmh/gists{/gist_id}","starred_url":"https://api.github.com/users/kevinjmh/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/kevinjmh/subscriptions","organizations_url":"https://api.github.com/users/kevinjmh/orgs","repos_url":"https://api.github.com/users/kevinjmh/repos","events_url":"https://api.github.com/users/kevinjmh/events{/privacy}","received_events_url":"https://api.github.com/users/kevinjmh/received_events","type":"User","site_admin":false},"repo":{"id":113741552,"name":"carbondata","full_name":"kevinjmh/carbondata","owner":{"login":"kevinjmh","id":3809732,"avatar_url":"https://avatars1.githubusercontent.com/u/3809732?v=4","gravatar_id":"","url":"https://api.github.com/users/kevinjmh","html_url":"https://github.com/kevinjmh","followers_url":"https://api.github.com/users/kevinjmh/followers","following_url":"https://api.github.com/users/kevinjmh/following{/other_user}","gists_url":"https://api.github.com/users/kevinjmh/gists{/gist_id}","starred_url":"https://api.github.com/users/kevinjmh/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/kevinjmh/subscriptions","organizations_url":"https://api.github.com/users/kevinjmh/orgs","repos_url":"https://api.github.com/users/kevinjmh/repos","events_url":"https://api.github.com/users/kevinjmh/events{/privacy}","received_events_url":"https://api.github.com/users/kevinjmh/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/kevinjmh/carbondata","description":"Mirror of Apache CarbonData","fork":true,"url":"https://api.github.com/repos/kevinjmh/carbondata","forks_url":"https://api.github.com/repos/kevinjmh/carbondata/forks","keys_url":"https://api.github.com/repos/kevinjmh/carbondata/keys{/key_id}","collaborators_url":"https://api.github.com/repos/kevinjmh/carbondata/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/kevinjmh/carbondata/teams","hooks_url":"https://api.github.com/repos/kevinjmh/carbondata/hooks","issue_events_url":"https://api.github.com/repos/kevinjmh/carbondata/issues/events{/number}","events_url":"https://api.github.com/repos/kevinjmh/carbondata/events","assignees_url":"https://api.github.com/repos/kevinjmh/carbondata/assignees{/user}","branches_url":"https://api.github.com/repos/kevinjmh/carbondata/branches{/branch}","tags_url":"https://api.github.com/repos/kevinjmh/carbondata/tags","blobs_url":"https://api.github.com/repos/kevinjmh/carbondata/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/kevinjmh/carbondata/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/kevinjmh/carbondata/git/refs{/sha}","trees_url":"https://api.github.com/repos/kevinjmh/carbondata/git/trees{/sha}","statuses_url":"https://api.github.com/repos/kevinjmh/carbondata/statuses/{sha}","languages_url":"https://api.github.com/repos/kevinjmh/carbondata/languages","stargazers_url":"https://api.github.com/repos/kevinjmh/carbondata/stargazers","contributors_url":"https://api.github.com/repos/kevinjmh/carbondata/contributors","subscribers_url":"https://api.github.com/repos/kevinjmh/carbondata/subscribers","subscription_url":"https://api.github.com/repos/kevinjmh/carbondata/subscription","commits_url":"https://api.github.com/repos/kevinjmh/carbondata/commits{/sha}","git_commits_url":"https://api.github.com/repos/kevinjmh/carbondata/git/commits{/sha}","comments_url":"https://api.github.com/repos/kevinjmh/carbondata/comments{/number}","issue_comment_url":"https://api.github.com/repos/kevinjmh/carbondata/issues/comments{/number}","contents_url":"https://api.github.com/repos/kevinjmh/carbondata/contents/{+path}","compare_url":"https://api.github.com/repos/kevinjmh/carbondata/compare/{base}...{head}","merges_url":"https://api.github.com/repos/kevinjmh/carbondata/merges","archive_url":"https://api.github.com/repos/kevinjmh/carbondata/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/kevinjmh/carbondata/downloads","issues_url":"https://api.github.com/repos/kevinjmh/carbondata/issues{/number}","pulls_url":"https://api.github.com/repos/kevinjmh/carbondata/pulls{/number}","milestones_url":"https://api.github.com/repos/kevinjmh/carbondata/milestones{/number}","notifications_url":"https://api.github.com/repos/kevinjmh/carbondata/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/kevinjmh/carbondata/labels{/name}","releases_url":"https://api.github.com/repos/kevinjmh/carbondata/releases{/id}","deployments_url":"https://api.github.com/repos/kevinjmh/carbondata/deployments","created_at":"2017-12-10T10:37:41Z","updated_at":"2018-03-30T11:29:18Z","pushed_at":"2018-04-02T14:12:59Z","git_url":"git://github.com/kevinjmh/carbondata.git","ssh_url":"git@github.com:kevinjmh/carbondata.git","clone_url":"https://github.com/kevinjmh/carbondata.git","svn_url":"https://github.com/kevinjmh/carbondata","homepage":"","size":55827,"stargazers_count":0,"watchers_count":0,"language":"Scala","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":0,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0"},"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"apache:master","ref":"master","sha":"7b56126b726bfc47504513d3994a2849f6412016","user":{"login":"apache","id":47359,"avatar_url":"https://avatars0.githubusercontent.com/u/47359?v=4","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"repo":{"id":62117818,"name":"carbondata","full_name":"apache/carbondata","owner":{"login":"apache","id":47359,"avatar_url":"https://avatars0.githubusercontent.com/u/47359?v=4","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/apache/carbondata","description":"Mirror of Apache CarbonData","fork":false,"url":"https://api.github.com/repos/apache/carbondata","forks_url":"https://api.github.com/repos/apache/carbondata/forks","keys_url":"https://api.github.com/repos/apache/carbondata/keys{/key_id}","collaborators_url":"https://api.github.com/repos/apache/carbondata/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/apache/carbondata/teams","hooks_url":"https://api.github.com/repos/apache/carbondata/hooks","issue_events_url":"https://api.github.com/repos/apache/carbondata/issues/events{/number}","events_url":"https://api.github.com/repos/apache/carbondata/events","assignees_url":"https://api.github.com/repos/apache/carbondata/assignees{/user}","branches_url":"https://api.github.com/repos/apache/carbondata/branches{/branch}","tags_url":"https://api.github.com/repos/apache/carbondata/tags","blobs_url":"https://api.github.com/repos/apache/carbondata/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/apache/carbondata/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/apache/carbondata/git/refs{/sha}","trees_url":"https://api.github.com/repos/apache/carbondata/git/trees{/sha}","statuses_url":"https://api.github.com/repos/apache/carbondata/statuses/{sha}","languages_url":"https://api.github.com/repos/apache/carbondata/languages","stargazers_url":"https://api.github.com/repos/apache/carbondata/stargazers","contributors_url":"https://api.github.com/repos/apache/carbondata/contributors","subscribers_url":"https://api.github.com/repos/apache/carbondata/subscribers","subscription_url":"https://api.github.com/repos/apache/carbondata/subscription","commits_url":"https://api.github.com/repos/apache/carbondata/commits{/sha}","git_commits_url":"https://api.github.com/repos/apache/carbondata/git/commits{/sha}","comments_url":"https://api.github.com/repos/apache/carbondata/comments{/number}","issue_comment_url":"https://api.github.com/repos/apache/carbondata/issues/comments{/number}","contents_url":"https://api.github.com/repos/apache/carbondata/contents/{+path}","compare_url":"https://api.github.com/repos/apache/carbondata/compare/{base}...{head}","merges_url":"https://api.github.com/repos/apache/carbondata/merges","archive_url":"https://api.github.com/repos/apache/carbondata/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/apache/carbondata/downloads","issues_url":"https://api.github.com/repos/apache/carbondata/issues{/number}","pulls_url":"https://api.github.com/repos/apache/carbondata/pulls{/number}","milestones_url":"https://api.github.com/repos/apache/carbondata/milestones{/number}","notifications_url":"https://api.github.com/repos/apache/carbondata/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/apache/carbondata/labels{/name}","releases_url":"https://api.github.com/repos/apache/carbondata/releases{/id}","deployments_url":"https://api.github.com/repos/apache/carbondata/deployments","created_at":"2016-06-28T07:00:06Z","updated_at":"2018-04-02T12:55:17Z","pushed_at":"2018-04-02T15:04:49Z","git_url":"git://github.com/apache/carbondata.git","ssh_url":"git@github.com:apache/carbondata.git","clone_url":"https://github.com/apache/carbondata.git","svn_url":"https://github.com/apache/carbondata","homepage":"","size":56310,"stargazers_count":638,"watchers_count":638,"language":"Scala","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":403,"mirror_url":null,"archived":false,"open_issues_count":99,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0"},"forks":403,"open_issues":99,"watchers":638,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/apache/carbondata/pulls/2123"},"html":{"href":"https://github.com/apache/carbondata/pull/2123"},"issue":{"href":"https://api.github.com/repos/apache/carbondata/issues/2123"},"comments":{"href":"https://api.github.com/repos/apache/carbondata/issues/2123/comments"},"review_comments":{"href":"https://api.github.com/repos/apache/carbondata/pulls/2123/comments"},"review_comment":{"href":"https://api.github.com/repos/apache/carbondata/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/apache/carbondata/pulls/2123/commits"},"statuses":{"href":"https://api.github.com/repos/apache/carbondata/statuses/58515280ac8563aab3b29e476320d947121ee1e9"}},"author_association":"CONTRIBUTOR"}}
{"actor":{"display_login":"kevinjmh"}}