{"action":"created","comment":{"url":"https://api.github.com/repos/linkedin/pinot/pulls/comments/88119314","pull_request_review_id":8701621,"id":88119314,"diff_hunk":"@@ -0,0 +1,168 @@\n+/**\n+ * Copyright (C) 2014-2016 LinkedIn Corp. (pinot-core@linkedin.com)\n+ *\n+ * Licensed under the Apache License, Version 2.0 (the \"License\");\n+ * you may not use this file except in compliance with the License.\n+ * You may obtain a copy of the License at\n+ *\n+ *         http://www.apache.org/licenses/LICENSE-2.0\n+ *\n+ * Unless required by applicable law or agreed to in writing, software\n+ * distributed under the License is distributed on an \"AS IS\" BASIS,\n+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+ * See the License for the specific language governing permissions and\n+ * limitations under the License.\n+ */\n+package com.linkedin.pinot.tools.perf;\n+\n+import com.google.common.base.Preconditions;\n+import com.linkedin.pinot.core.segment.index.SegmentMetadataImpl;\n+import com.linkedin.pinot.tools.AbstractBaseCommand;\n+import com.linkedin.pinot.tools.Command;\n+import java.io.File;\n+import java.util.Arrays;\n+import java.util.List;\n+import java.util.concurrent.ExecutorService;\n+import java.util.concurrent.Executors;\n+import java.util.concurrent.TimeUnit;\n+import org.kohsuke.args4j.Option;\n+import org.slf4j.Logger;\n+import org.slf4j.LoggerFactory;\n+\n+\n+@SuppressWarnings(\"FieldCanBeLocal\")\n+public class PerfBenchmarkRunner extends AbstractBaseCommand implements Command {\n+  private static final Logger LOGGER = LoggerFactory.getLogger(PerfBenchmarkRunner.class);\n+\n+  @Option(name = \"-mode\", required = true, metaVar = \"<String>\",\n+      usage = \"Mode of the PerfBenchmarkRunner (startAll|startAllButServer|startServerWithPreLoadedSegments).\")\n+  private String _mode;\n+\n+  @Option(name = \"-dataDir\", required = false, metaVar = \"<String>\", usage = \"Path to data directory.\")\n+  private String _dataDir;\n+\n+  @Option(name = \"-tempDir\", required = false, metaVar = \"<String>\",\n+      usage = \"Path to temporary directory to start the cluster\")\n+  private String _tempDir = \"/tmp/\";\n+\n+  @Option(name = \"-loadMode\", required = false, metaVar = \"<String>\", usage = \"Load mode of the segments (HEAP|MMAP).\")\n+  private String _loadMode = \"HEAP\";\n+\n+  @Option(name = \"-segmentFormatVersion\", required = false, metaVar = \"<String>\",\n+      usage = \"Segment format version to be loaded (v1|v3).\")\n+  private String _segmentFormatVersion = \"v1\";\n+\n+  @Option(name = \"-batchLoad\", required = false, metaVar = \"<boolean>\", usage = \"Batch load multiple tables.\")\n+  private boolean _isBatchLoad;\n+\n+  @Option(name = \"-numThreads\", required = false, metaVar = \"<int>\",\n+      usage = \"Number of threads for batch load (default 10).\")\n+  private int _numThreads = 10;\n+\n+  @Option(name = \"-timeoutInSeconds\", required = false, metaVar = \"<int>\",\n+      usage = \"Timeout in seconds for batch load (default 60).\")\n+  private int _timeoutInSeconds = 60;\n+\n+  @Option(name = \"-tableNames\", required = false, metaVar = \"<String>\",\n+      usage = \"Comma separated table names to be loaded (non-batch load).\")\n+  private String _tableNames;\n+\n+  @Option(name = \"-invertedIndexColumns\", required = false, metaVar = \"<String>\",\n+      usage = \"Comma separated inverted index columns to be created (non-batch load).\")\n+  private String _invertedIndexColumns;\n+\n+  @Option(name = \"-help\", required = false, help = true, aliases = {\"-h\", \"--h\", \"--help\"},\n+      usage = \"Print this message.\")\n+  private boolean _help = false;\n+\n+  @Override\n+  public boolean getHelp() {\n+    return _help;\n+  }\n+\n+  @Override\n+  public String getName() {\n+    return getClass().getSimpleName();\n+  }\n+\n+  @Override\n+  public String description() {\n+    return \"Start Pinot cluster with optional preloaded segments\";\n+  }\n+\n+  @Override\n+  public boolean execute()\n+      throws Exception {\n+    if (_mode.equalsIgnoreCase(\"startAll\") || _mode.equalsIgnoreCase(\"startAllButServer\")) {\n+      startAllButServer();\n+    }\n+    if (_mode.equalsIgnoreCase(\"startAll\") || _mode.equalsIgnoreCase(\"startServerWithPreLoadedSegments\")) {\n+      startServerWithPreLoadedSegments();\n+    }\n+    return true;\n+  }\n+\n+  public static void startAllButServer()\n+      throws Exception {\n+    PerfBenchmarkDriverConf perfBenchmarkDriverConf = new PerfBenchmarkDriverConf();\n+    perfBenchmarkDriverConf.setStartServer(false);\n+    PerfBenchmarkDriver driver = new PerfBenchmarkDriver(perfBenchmarkDriverConf);\n+    driver.run();\n+  }\n+\n+  private void startServerWithPreLoadedSegments()\n+      throws Exception {\n+    PerfBenchmarkDriverConf perfBenchmarkDriverConf = new PerfBenchmarkDriverConf();\n+    perfBenchmarkDriverConf.setStartZookeeper(false);\n+    perfBenchmarkDriverConf.setStartController(false);\n+    perfBenchmarkDriverConf.setStartBroker(false);\n+    perfBenchmarkDriverConf.setServerInstanceDataDir(_dataDir);\n+    final PerfBenchmarkDriver driver =\n+        new PerfBenchmarkDriver(perfBenchmarkDriverConf, _tempDir, _loadMode, _segmentFormatVersion, false);\n+    driver.run();\n+\n+    if (_isBatchLoad) {\n+      String[] tableNames = new File(_dataDir).list();\n+      Preconditions.checkNotNull(tableNames);\n+      ExecutorService executorService = Executors.newFixedThreadPool(_numThreads);\n+      for (final String tableName : tableNames) {\n+        executorService.submit(new Runnable() {\n+          @Override\n+          public void run() {\n+            try {\n+              loadTable(driver, _dataDir, tableName, null);\n+            } catch (Exception e) {\n+              LOGGER.error(\"Caught exception while loading table: {}\", tableName, e);\n+            }\n+          }\n+        });\n+      }\n+      executorService.shutdown();\n+      executorService.awaitTermination(_timeoutInSeconds, TimeUnit.SECONDS);\n+    } else {\n+      List<String> invertedIndexColumns = null;\n+      if (_invertedIndexColumns != null) {\n+        invertedIndexColumns = Arrays.asList(_invertedIndexColumns.split(\",\"));\n+      }\n+      for (String tableName : _tableNames.split(\",\")) {\n+        loadTable(driver, _dataDir, tableName, invertedIndexColumns);\n+      }\n+    }\n+  }\n+\n+  public static void loadTable(PerfBenchmarkDriver driver, String dataDir, String tableName,\n+      List<String> invertedIndexColumns)\n+      throws Exception {\n+    boolean tableConfigured = false;\n+    File[] segments = new File(dataDir, tableName).listFiles();\n+    Preconditions.checkNotNull(segments);\n+    for (File segment : segments) {\n+      SegmentMetadataImpl segmentMetadata = new SegmentMetadataImpl(segment);\n+      if (!tableConfigured) {\n+        driver.configureTable(segmentMetadata.getTableName(), invertedIndexColumns);\n+        tableConfigured = true;\n+      }\n+      driver.addSegment(segmentMetadata);","path":"pinot-tools/src/main/java/com/linkedin/pinot/tools/perf/PerfBenchmarkRunner.java","position":165,"original_position":165,"commit_id":"0c6eaf3f05feb3a77e582a8655f5d60be187daa8","original_commit_id":"0c6eaf3f05feb3a77e582a8655f5d60be187daa8","user":{"login":"kishoreg","id":850791,"avatar_url":"https://avatars.githubusercontent.com/u/850791?v=3","gravatar_id":"","url":"https://api.github.com/users/kishoreg","html_url":"https://github.com/kishoreg","followers_url":"https://api.github.com/users/kishoreg/followers","following_url":"https://api.github.com/users/kishoreg/following{/other_user}","gists_url":"https://api.github.com/users/kishoreg/gists{/gist_id}","starred_url":"https://api.github.com/users/kishoreg/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/kishoreg/subscriptions","organizations_url":"https://api.github.com/users/kishoreg/orgs","repos_url":"https://api.github.com/users/kishoreg/repos","events_url":"https://api.github.com/users/kishoreg/events{/privacy}","received_events_url":"https://api.github.com/users/kishoreg/received_events","type":"User","site_admin":false},"body":"More than multi threaded, I think this was the slowest part. Since we do one segment at a time. We should have a method to do addSegments(segmentMetadataList) or something. This will ensure that there is only one write to ZK.","created_at":"2016-11-15T21:37:56Z","updated_at":"2016-11-15T21:38:02Z","html_url":"https://github.com/linkedin/pinot/pull/817#discussion_r88119314","pull_request_url":"https://api.github.com/repos/linkedin/pinot/pulls/817","_links":{"self":{"href":"https://api.github.com/repos/linkedin/pinot/pulls/comments/88119314"},"html":{"href":"https://github.com/linkedin/pinot/pull/817#discussion_r88119314"},"pull_request":{"href":"https://api.github.com/repos/linkedin/pinot/pulls/817"}}},"pull_request":{"url":"https://api.github.com/repos/linkedin/pinot/pulls/817","id":93840215,"html_url":"https://github.com/linkedin/pinot/pull/817","diff_url":"https://github.com/linkedin/pinot/pull/817.diff","patch_url":"https://github.com/linkedin/pinot/pull/817.patch","issue_url":"https://api.github.com/repos/linkedin/pinot/issues/817","number":817,"state":"open","locked":false,"title":"Add PerfBenchmarkRunner as a tool inside PinotToolLauncher","user":{"login":"Jackie-Jiang","id":17555551,"avatar_url":"https://avatars.githubusercontent.com/u/17555551?v=3","gravatar_id":"","url":"https://api.github.com/users/Jackie-Jiang","html_url":"https://github.com/Jackie-Jiang","followers_url":"https://api.github.com/users/Jackie-Jiang/followers","following_url":"https://api.github.com/users/Jackie-Jiang/following{/other_user}","gists_url":"https://api.github.com/users/Jackie-Jiang/gists{/gist_id}","starred_url":"https://api.github.com/users/Jackie-Jiang/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Jackie-Jiang/subscriptions","organizations_url":"https://api.github.com/users/Jackie-Jiang/orgs","repos_url":"https://api.github.com/users/Jackie-Jiang/repos","events_url":"https://api.github.com/users/Jackie-Jiang/events{/privacy}","received_events_url":"https://api.github.com/users/Jackie-Jiang/received_events","type":"User","site_admin":false},"body":"1. Adding multi-thread batch loading option.\n2. Refactor PerfBenchmarkDriver and add more custom configuratioins.\n3. Move PerfBenchmarkDriver related classed to pinot-tool to solve dependency issue.","created_at":"2016-11-15T20:18:49Z","updated_at":"2016-11-15T21:38:02Z","closed_at":null,"merged_at":null,"merge_commit_sha":"6b27ddaa0a2b8cdb790bb28ab2609cfe331e8bdc","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/linkedin/pinot/pulls/817/commits","review_comments_url":"https://api.github.com/repos/linkedin/pinot/pulls/817/comments","review_comment_url":"https://api.github.com/repos/linkedin/pinot/pulls/comments{/number}","comments_url":"https://api.github.com/repos/linkedin/pinot/issues/817/comments","statuses_url":"https://api.github.com/repos/linkedin/pinot/statuses/0c6eaf3f05feb3a77e582a8655f5d60be187daa8","head":{"label":"linkedin:perf_runner_tool","ref":"perf_runner_tool","sha":"0c6eaf3f05feb3a77e582a8655f5d60be187daa8","user":{"login":"linkedin","id":357098,"avatar_url":"https://avatars.githubusercontent.com/u/357098?v=3","gravatar_id":"","url":"https://api.github.com/users/linkedin","html_url":"https://github.com/linkedin","followers_url":"https://api.github.com/users/linkedin/followers","following_url":"https://api.github.com/users/linkedin/following{/other_user}","gists_url":"https://api.github.com/users/linkedin/gists{/gist_id}","starred_url":"https://api.github.com/users/linkedin/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/linkedin/subscriptions","organizations_url":"https://api.github.com/users/linkedin/orgs","repos_url":"https://api.github.com/users/linkedin/repos","events_url":"https://api.github.com/users/linkedin/events{/privacy}","received_events_url":"https://api.github.com/users/linkedin/received_events","type":"Organization","site_admin":false},"repo":{"id":19961085,"name":"pinot","full_name":"linkedin/pinot","owner":{"login":"linkedin","id":357098,"avatar_url":"https://avatars.githubusercontent.com/u/357098?v=3","gravatar_id":"","url":"https://api.github.com/users/linkedin","html_url":"https://github.com/linkedin","followers_url":"https://api.github.com/users/linkedin/followers","following_url":"https://api.github.com/users/linkedin/following{/other_user}","gists_url":"https://api.github.com/users/linkedin/gists{/gist_id}","starred_url":"https://api.github.com/users/linkedin/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/linkedin/subscriptions","organizations_url":"https://api.github.com/users/linkedin/orgs","repos_url":"https://api.github.com/users/linkedin/repos","events_url":"https://api.github.com/users/linkedin/events{/privacy}","received_events_url":"https://api.github.com/users/linkedin/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/linkedin/pinot","description":"A realtime distributed OLAP datastore","fork":false,"url":"https://api.github.com/repos/linkedin/pinot","forks_url":"https://api.github.com/repos/linkedin/pinot/forks","keys_url":"https://api.github.com/repos/linkedin/pinot/keys{/key_id}","collaborators_url":"https://api.github.com/repos/linkedin/pinot/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/linkedin/pinot/teams","hooks_url":"https://api.github.com/repos/linkedin/pinot/hooks","issue_events_url":"https://api.github.com/repos/linkedin/pinot/issues/events{/number}","events_url":"https://api.github.com/repos/linkedin/pinot/events","assignees_url":"https://api.github.com/repos/linkedin/pinot/assignees{/user}","branches_url":"https://api.github.com/repos/linkedin/pinot/branches{/branch}","tags_url":"https://api.github.com/repos/linkedin/pinot/tags","blobs_url":"https://api.github.com/repos/linkedin/pinot/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/linkedin/pinot/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/linkedin/pinot/git/refs{/sha}","trees_url":"https://api.github.com/repos/linkedin/pinot/git/trees{/sha}","statuses_url":"https://api.github.com/repos/linkedin/pinot/statuses/{sha}","languages_url":"https://api.github.com/repos/linkedin/pinot/languages","stargazers_url":"https://api.github.com/repos/linkedin/pinot/stargazers","contributors_url":"https://api.github.com/repos/linkedin/pinot/contributors","subscribers_url":"https://api.github.com/repos/linkedin/pinot/subscribers","subscription_url":"https://api.github.com/repos/linkedin/pinot/subscription","commits_url":"https://api.github.com/repos/linkedin/pinot/commits{/sha}","git_commits_url":"https://api.github.com/repos/linkedin/pinot/git/commits{/sha}","comments_url":"https://api.github.com/repos/linkedin/pinot/comments{/number}","issue_comment_url":"https://api.github.com/repos/linkedin/pinot/issues/comments{/number}","contents_url":"https://api.github.com/repos/linkedin/pinot/contents/{+path}","compare_url":"https://api.github.com/repos/linkedin/pinot/compare/{base}...{head}","merges_url":"https://api.github.com/repos/linkedin/pinot/merges","archive_url":"https://api.github.com/repos/linkedin/pinot/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/linkedin/pinot/downloads","issues_url":"https://api.github.com/repos/linkedin/pinot/issues{/number}","pulls_url":"https://api.github.com/repos/linkedin/pinot/pulls{/number}","milestones_url":"https://api.github.com/repos/linkedin/pinot/milestones{/number}","notifications_url":"https://api.github.com/repos/linkedin/pinot/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/linkedin/pinot/labels{/name}","releases_url":"https://api.github.com/repos/linkedin/pinot/releases{/id}","deployments_url":"https://api.github.com/repos/linkedin/pinot/deployments","created_at":"2014-05-19T23:27:48Z","updated_at":"2016-11-14T16:11:50Z","pushed_at":"2016-11-15T20:18:49Z","git_url":"git://github.com/linkedin/pinot.git","ssh_url":"git@github.com:linkedin/pinot.git","clone_url":"https://github.com/linkedin/pinot.git","svn_url":"https://github.com/linkedin/pinot","homepage":"https://github.com/linkedin/pinot/wiki","size":147805,"stargazers_count":1369,"watchers_count":1369,"language":"Java","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":230,"mirror_url":null,"open_issues_count":94,"forks":230,"open_issues":94,"watchers":1369,"default_branch":"master"}},"base":{"label":"linkedin:master","ref":"master","sha":"70a172de8fe4017188888ff8421b4f7e77dff450","user":{"login":"linkedin","id":357098,"avatar_url":"https://avatars.githubusercontent.com/u/357098?v=3","gravatar_id":"","url":"https://api.github.com/users/linkedin","html_url":"https://github.com/linkedin","followers_url":"https://api.github.com/users/linkedin/followers","following_url":"https://api.github.com/users/linkedin/following{/other_user}","gists_url":"https://api.github.com/users/linkedin/gists{/gist_id}","starred_url":"https://api.github.com/users/linkedin/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/linkedin/subscriptions","organizations_url":"https://api.github.com/users/linkedin/orgs","repos_url":"https://api.github.com/users/linkedin/repos","events_url":"https://api.github.com/users/linkedin/events{/privacy}","received_events_url":"https://api.github.com/users/linkedin/received_events","type":"Organization","site_admin":false},"repo":{"id":19961085,"name":"pinot","full_name":"linkedin/pinot","owner":{"login":"linkedin","id":357098,"avatar_url":"https://avatars.githubusercontent.com/u/357098?v=3","gravatar_id":"","url":"https://api.github.com/users/linkedin","html_url":"https://github.com/linkedin","followers_url":"https://api.github.com/users/linkedin/followers","following_url":"https://api.github.com/users/linkedin/following{/other_user}","gists_url":"https://api.github.com/users/linkedin/gists{/gist_id}","starred_url":"https://api.github.com/users/linkedin/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/linkedin/subscriptions","organizations_url":"https://api.github.com/users/linkedin/orgs","repos_url":"https://api.github.com/users/linkedin/repos","events_url":"https://api.github.com/users/linkedin/events{/privacy}","received_events_url":"https://api.github.com/users/linkedin/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/linkedin/pinot","description":"A realtime distributed OLAP datastore","fork":false,"url":"https://api.github.com/repos/linkedin/pinot","forks_url":"https://api.github.com/repos/linkedin/pinot/forks","keys_url":"https://api.github.com/repos/linkedin/pinot/keys{/key_id}","collaborators_url":"https://api.github.com/repos/linkedin/pinot/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/linkedin/pinot/teams","hooks_url":"https://api.github.com/repos/linkedin/pinot/hooks","issue_events_url":"https://api.github.com/repos/linkedin/pinot/issues/events{/number}","events_url":"https://api.github.com/repos/linkedin/pinot/events","assignees_url":"https://api.github.com/repos/linkedin/pinot/assignees{/user}","branches_url":"https://api.github.com/repos/linkedin/pinot/branches{/branch}","tags_url":"https://api.github.com/repos/linkedin/pinot/tags","blobs_url":"https://api.github.com/repos/linkedin/pinot/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/linkedin/pinot/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/linkedin/pinot/git/refs{/sha}","trees_url":"https://api.github.com/repos/linkedin/pinot/git/trees{/sha}","statuses_url":"https://api.github.com/repos/linkedin/pinot/statuses/{sha}","languages_url":"https://api.github.com/repos/linkedin/pinot/languages","stargazers_url":"https://api.github.com/repos/linkedin/pinot/stargazers","contributors_url":"https://api.github.com/repos/linkedin/pinot/contributors","subscribers_url":"https://api.github.com/repos/linkedin/pinot/subscribers","subscription_url":"https://api.github.com/repos/linkedin/pinot/subscription","commits_url":"https://api.github.com/repos/linkedin/pinot/commits{/sha}","git_commits_url":"https://api.github.com/repos/linkedin/pinot/git/commits{/sha}","comments_url":"https://api.github.com/repos/linkedin/pinot/comments{/number}","issue_comment_url":"https://api.github.com/repos/linkedin/pinot/issues/comments{/number}","contents_url":"https://api.github.com/repos/linkedin/pinot/contents/{+path}","compare_url":"https://api.github.com/repos/linkedin/pinot/compare/{base}...{head}","merges_url":"https://api.github.com/repos/linkedin/pinot/merges","archive_url":"https://api.github.com/repos/linkedin/pinot/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/linkedin/pinot/downloads","issues_url":"https://api.github.com/repos/linkedin/pinot/issues{/number}","pulls_url":"https://api.github.com/repos/linkedin/pinot/pulls{/number}","milestones_url":"https://api.github.com/repos/linkedin/pinot/milestones{/number}","notifications_url":"https://api.github.com/repos/linkedin/pinot/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/linkedin/pinot/labels{/name}","releases_url":"https://api.github.com/repos/linkedin/pinot/releases{/id}","deployments_url":"https://api.github.com/repos/linkedin/pinot/deployments","created_at":"2014-05-19T23:27:48Z","updated_at":"2016-11-14T16:11:50Z","pushed_at":"2016-11-15T20:18:49Z","git_url":"git://github.com/linkedin/pinot.git","ssh_url":"git@github.com:linkedin/pinot.git","clone_url":"https://github.com/linkedin/pinot.git","svn_url":"https://github.com/linkedin/pinot","homepage":"https://github.com/linkedin/pinot/wiki","size":147805,"stargazers_count":1369,"watchers_count":1369,"language":"Java","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":230,"mirror_url":null,"open_issues_count":94,"forks":230,"open_issues":94,"watchers":1369,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/linkedin/pinot/pulls/817"},"html":{"href":"https://github.com/linkedin/pinot/pull/817"},"issue":{"href":"https://api.github.com/repos/linkedin/pinot/issues/817"},"comments":{"href":"https://api.github.com/repos/linkedin/pinot/issues/817/comments"},"review_comments":{"href":"https://api.github.com/repos/linkedin/pinot/pulls/817/comments"},"review_comment":{"href":"https://api.github.com/repos/linkedin/pinot/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/linkedin/pinot/pulls/817/commits"},"statuses":{"href":"https://api.github.com/repos/linkedin/pinot/statuses/0c6eaf3f05feb3a77e582a8655f5d60be187daa8"}}}}
{"actor":{"display_login":"kishoreg"}}