{"action":"created","comment":{"url":"https://api.github.com/repos/hyrise/hyrise/pulls/comments/189515202","pull_request_review_id":121707452,"id":189515202,"diff_hunk":"@@ -0,0 +1,449 @@\n+#include <random>\n+\n+#if __has_include(<fs>)\n+#include <filesystem>\n+namespace fs = std::filesystem;\n+#else\n+#include <experimental/filesystem>\n+#include <scheduler/current_scheduler.hpp>\n+namespace fs = std::experimental::filesystem;\n+#endif\n+\n+#include <json.hpp>\n+#include <storage/chunk_encoder.hpp>\n+\n+#include \"benchmark_runner.hpp\"\n+#include \"constant_mappings.hpp\"\n+#include \"import_export/csv_parser.hpp\"\n+#include \"planviz/lqp_visualizer.hpp\"\n+#include \"planviz/sql_query_plan_visualizer.hpp\"\n+#include \"sql/sql_pipeline_builder.hpp\"\n+#include \"storage/storage_manager.hpp\"\n+#include \"tpch/tpch_db_generator.hpp\"\n+#include \"tpch/tpch_queries.hpp\"\n+#include \"utils/load_table.hpp\"\n+\n+namespace opossum {\n+\n+BenchmarkRunner::BenchmarkRunner(BenchmarkConfig config, const NamedQueries& queries, const nlohmann::json& context)\n+    : _config(std::move(config)), _queries(queries), _context(context) {}\n+\n+void BenchmarkRunner::run() {\n+  _config.out << \"\\n- Starting Benchmark..\" << std::endl;\n+\n+  // Run the queries in the selected mode\n+  switch (_config.benchmark_mode) {\n+    case BenchmarkMode::IndividualQueries: {\n+      _benchmark_individual_queries();\n+      break;\n+    }\n+    case BenchmarkMode::PermutedQuerySets: {\n+      _benchmark_permuted_query_sets();\n+      break;\n+    }\n+  }\n+\n+  // Create report\n+  if (_config.output_file_path) {\n+    std::ofstream output_file(*_config.output_file_path);\n+    _create_report(output_file);\n+  } else {\n+    _create_report(std::cout);\n+  }\n+\n+  // Visualize query plans\n+  if (_config.enable_visualization) {\n+    for (const auto& name_and_plans : _query_plans) {\n+      const auto& name = name_and_plans.first;\n+      const auto& lqps = name_and_plans.second.lqps;\n+      const auto& pqps = name_and_plans.second.pqps;\n+\n+      GraphvizConfig graphviz_config;\n+      graphviz_config.format = \"svg\";\n+\n+      for (auto lqp_idx = size_t{0}; lqp_idx < lqps.size(); ++lqp_idx) {\n+        const auto file_prefix = name + \"-LQP-\" + std::to_string(lqp_idx);\n+        LQPVisualizer{graphviz_config, {}, {}, {}}.visualize({lqps[lqp_idx]}, file_prefix + \".dot\",\n+                                                             file_prefix + \".svg\");\n+      }\n+      for (auto pqp_idx = size_t{0}; pqp_idx < pqps.size(); ++pqp_idx) {\n+        const auto file_prefix = name + \"-PQP-\" + std::to_string(pqp_idx);\n+        SQLQueryPlanVisualizer{graphviz_config, {}, {}, {}}.visualize(*pqps[pqp_idx], file_prefix + \".dot\",\n+                                                                      file_prefix + \".svg\");\n+      }\n+    }\n+  }\n+}\n+\n+void BenchmarkRunner::_benchmark_permuted_query_sets() {\n+  // Init results\n+  for (const auto& named_query : _queries) {\n+    const auto& name = named_query.first;\n+    _query_results_by_query_name.emplace(name, QueryBenchmarkResult{});\n+  }\n+\n+  auto mutable_named_queries = _queries;\n+\n+  // For shuffling the query order\n+  std::random_device random_device;\n+  std::mt19937 random_generator(random_device());\n+\n+  BenchmarkState state{_config.max_num_query_runs, _config.max_duration};\n+  while (state.keep_running()) {\n+    std::shuffle(mutable_named_queries.begin(), mutable_named_queries.end(), random_generator);\n+\n+    for (const auto& named_query : mutable_named_queries) {\n+      const auto query_benchmark_begin = std::chrono::steady_clock::now();\n+\n+      // Execute the query, we don't care about the results\n+      _execute_query(named_query);\n+\n+      const auto query_benchmark_end = std::chrono::steady_clock::now();\n+\n+      auto& query_benchmark_result = _query_results_by_query_name.at(named_query.first);\n+      query_benchmark_result.duration += query_benchmark_end - query_benchmark_begin;\n+      query_benchmark_result.num_iterations++;\n+    }\n+  }\n+}\n+\n+void BenchmarkRunner::_benchmark_individual_queries() {\n+  for (const auto& named_query : _queries) {\n+    const auto& name = named_query.first;\n+    _config.out << \"- Benchmarking Query \" << name << std::endl;\n+\n+    BenchmarkState state{_config.max_num_query_runs, _config.max_duration};\n+    while (state.keep_running()) {\n+      _execute_query(named_query);\n+    }\n+\n+    QueryBenchmarkResult result;\n+    result.num_iterations = state.num_iterations;\n+    result.duration = std::chrono::high_resolution_clock::now() - state.begin;\n+\n+    _query_results_by_query_name.emplace(name, result);\n+  }\n+}\n+void BenchmarkRunner::_execute_query(const NamedQuery& named_query) {\n+  const auto& name = named_query.first;\n+  const auto& sql = named_query.second;\n+\n+  auto pipeline = SQLPipelineBuilder{sql}.with_mvcc(_config.use_mvcc).create_pipeline();\n+  // Execute the query, we don't care about the results\n+  pipeline.get_result_table();\n+\n+  // If necessary, keep plans for visualization\n+  if (_config.enable_visualization) {\n+    const auto query_plans_iter = _query_plans.find(name);\n+    if (query_plans_iter == _query_plans.end()) {\n+      Assert(pipeline.get_query_plans().size() == 1, \"Expected exactly one SQLQueryPlan\");\n+      QueryPlans plans{pipeline.get_optimized_logical_plans(), pipeline.get_query_plans()};\n+      _query_plans.emplace(name, plans);\n+    }\n+  }\n+}\n+\n+void BenchmarkRunner::_create_report(std::ostream& stream) const {\n+  nlohmann::json benchmarks;\n+\n+  for (const auto& named_query : _queries) {\n+    const auto& name = named_query.first;\n+    const auto& query_result = _query_results_by_query_name.at(name);\n+\n+    const auto duration_ns = std::chrono::duration_cast<std::chrono::nanoseconds>(query_result.duration).count();\n+    const auto duration_seconds = static_cast<float>(duration_ns) / 1'000'000'000;\n+    const auto items_per_second = static_cast<float>(query_result.num_iterations) / duration_seconds;\n+    const auto time_per_query = duration_ns / query_result.num_iterations;\n+\n+    nlohmann::json benchmark{","path":"src/benchmarklib/benchmark_runner.cpp","position":160,"original_position":158,"commit_id":"efec5f8051cbbc85b7112d95e1addf8c1c86fd72","original_commit_id":"728e21989172a682b1a466332035ecf60988ba9d","user":{"login":"mrks","id":575106,"avatar_url":"https://avatars2.githubusercontent.com/u/575106?v=4","gravatar_id":"","url":"https://api.github.com/users/mrks","html_url":"https://github.com/mrks","followers_url":"https://api.github.com/users/mrks/followers","following_url":"https://api.github.com/users/mrks/following{/other_user}","gists_url":"https://api.github.com/users/mrks/gists{/gist_id}","starred_url":"https://api.github.com/users/mrks/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mrks/subscriptions","organizations_url":"https://api.github.com/users/mrks/orgs","repos_url":"https://api.github.com/users/mrks/repos","events_url":"https://api.github.com/users/mrks/events{/privacy}","received_events_url":"https://api.github.com/users/mrks/received_events","type":"User","site_admin":false},"body":"How hacky is that? I see a value in having the commit id so that when we find old benchmark results somewhere, we know which version they used.","created_at":"2018-05-21T07:41:22Z","updated_at":"2018-05-21T07:41:22Z","html_url":"https://github.com/hyrise/hyrise/pull/876#discussion_r189515202","pull_request_url":"https://api.github.com/repos/hyrise/hyrise/pulls/876","author_association":"MEMBER","_links":{"self":{"href":"https://api.github.com/repos/hyrise/hyrise/pulls/comments/189515202"},"html":{"href":"https://github.com/hyrise/hyrise/pull/876#discussion_r189515202"},"pull_request":{"href":"https://api.github.com/repos/hyrise/hyrise/pulls/876"}},"in_reply_to_id":187937987},"pull_request":{"url":"https://api.github.com/repos/hyrise/hyrise/pulls/876","id":187167061,"html_url":"https://github.com/hyrise/hyrise/pull/876","diff_url":"https://github.com/hyrise/hyrise/pull/876.diff","patch_url":"https://github.com/hyrise/hyrise/pull/876.patch","issue_url":"https://api.github.com/repos/hyrise/hyrise/issues/876","number":876,"state":"open","locked":false,"title":"General Purpose Benchmark Runner","user":{"login":"lawben","id":7422656,"avatar_url":"https://avatars0.githubusercontent.com/u/7422656?v=4","gravatar_id":"","url":"https://api.github.com/users/lawben","html_url":"https://github.com/lawben","followers_url":"https://api.github.com/users/lawben/followers","following_url":"https://api.github.com/users/lawben/following{/other_user}","gists_url":"https://api.github.com/users/lawben/gists{/gist_id}","starred_url":"https://api.github.com/users/lawben/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/lawben/subscriptions","organizations_url":"https://api.github.com/users/lawben/orgs","repos_url":"https://api.github.com/users/lawben/repos","events_url":"https://api.github.com/users/lawben/events{/privacy}","received_events_url":"https://api.github.com/users/lawben/received_events","type":"User","site_admin":false},"body":"This PR ads a general benchmark runner, that runs on any given set of table- and query-files. \r\n\r\nThe TPCH benchmark is now just a wrapper around this. A lot of this code is not new, it is just moved and adapted slightly.  It's usage has not changed, except that there is an `encoding` option now. \r\n\r\nUsage: `build/hyriseBenchmarkRunner --tables=src/test/tables --queries=path/to/queries <all-other-options>`\r\n\r\nCloses #839 \r\n\r\nStill to come, but probably best handled in a separate PR:\r\n- Store tables in .bin for better performance\r\n- Make benchmark configurable via JSON\r\n- Customisable encoding for tables/columns","created_at":"2018-05-10T11:22:18Z","updated_at":"2018-05-21T07:41:22Z","closed_at":null,"merged_at":null,"merge_commit_sha":"4a6843f1d952c0ecd6fe744f810049b0b3600e44","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/hyrise/hyrise/pulls/876/commits","review_comments_url":"https://api.github.com/repos/hyrise/hyrise/pulls/876/comments","review_comment_url":"https://api.github.com/repos/hyrise/hyrise/pulls/comments{/number}","comments_url":"https://api.github.com/repos/hyrise/hyrise/issues/876/comments","statuses_url":"https://api.github.com/repos/hyrise/hyrise/statuses/efec5f8051cbbc85b7112d95e1addf8c1c86fd72","head":{"label":"hyrise:lawben/general_benchmark","ref":"lawben/general_benchmark","sha":"efec5f8051cbbc85b7112d95e1addf8c1c86fd72","user":{"login":"hyrise","id":3465095,"avatar_url":"https://avatars2.githubusercontent.com/u/3465095?v=4","gravatar_id":"","url":"https://api.github.com/users/hyrise","html_url":"https://github.com/hyrise","followers_url":"https://api.github.com/users/hyrise/followers","following_url":"https://api.github.com/users/hyrise/following{/other_user}","gists_url":"https://api.github.com/users/hyrise/gists{/gist_id}","starred_url":"https://api.github.com/users/hyrise/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/hyrise/subscriptions","organizations_url":"https://api.github.com/users/hyrise/orgs","repos_url":"https://api.github.com/users/hyrise/repos","events_url":"https://api.github.com/users/hyrise/events{/privacy}","received_events_url":"https://api.github.com/users/hyrise/received_events","type":"Organization","site_admin":false},"repo":{"id":87414843,"name":"hyrise","full_name":"hyrise/hyrise","owner":{"login":"hyrise","id":3465095,"avatar_url":"https://avatars2.githubusercontent.com/u/3465095?v=4","gravatar_id":"","url":"https://api.github.com/users/hyrise","html_url":"https://github.com/hyrise","followers_url":"https://api.github.com/users/hyrise/followers","following_url":"https://api.github.com/users/hyrise/following{/other_user}","gists_url":"https://api.github.com/users/hyrise/gists{/gist_id}","starred_url":"https://api.github.com/users/hyrise/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/hyrise/subscriptions","organizations_url":"https://api.github.com/users/hyrise/orgs","repos_url":"https://api.github.com/users/hyrise/repos","events_url":"https://api.github.com/users/hyrise/events{/privacy}","received_events_url":"https://api.github.com/users/hyrise/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/hyrise/hyrise","description":"A research in-memory database.","fork":false,"url":"https://api.github.com/repos/hyrise/hyrise","forks_url":"https://api.github.com/repos/hyrise/hyrise/forks","keys_url":"https://api.github.com/repos/hyrise/hyrise/keys{/key_id}","collaborators_url":"https://api.github.com/repos/hyrise/hyrise/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/hyrise/hyrise/teams","hooks_url":"https://api.github.com/repos/hyrise/hyrise/hooks","issue_events_url":"https://api.github.com/repos/hyrise/hyrise/issues/events{/number}","events_url":"https://api.github.com/repos/hyrise/hyrise/events","assignees_url":"https://api.github.com/repos/hyrise/hyrise/assignees{/user}","branches_url":"https://api.github.com/repos/hyrise/hyrise/branches{/branch}","tags_url":"https://api.github.com/repos/hyrise/hyrise/tags","blobs_url":"https://api.github.com/repos/hyrise/hyrise/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/hyrise/hyrise/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/hyrise/hyrise/git/refs{/sha}","trees_url":"https://api.github.com/repos/hyrise/hyrise/git/trees{/sha}","statuses_url":"https://api.github.com/repos/hyrise/hyrise/statuses/{sha}","languages_url":"https://api.github.com/repos/hyrise/hyrise/languages","stargazers_url":"https://api.github.com/repos/hyrise/hyrise/stargazers","contributors_url":"https://api.github.com/repos/hyrise/hyrise/contributors","subscribers_url":"https://api.github.com/repos/hyrise/hyrise/subscribers","subscription_url":"https://api.github.com/repos/hyrise/hyrise/subscription","commits_url":"https://api.github.com/repos/hyrise/hyrise/commits{/sha}","git_commits_url":"https://api.github.com/repos/hyrise/hyrise/git/commits{/sha}","comments_url":"https://api.github.com/repos/hyrise/hyrise/comments{/number}","issue_comment_url":"https://api.github.com/repos/hyrise/hyrise/issues/comments{/number}","contents_url":"https://api.github.com/repos/hyrise/hyrise/contents/{+path}","compare_url":"https://api.github.com/repos/hyrise/hyrise/compare/{base}...{head}","merges_url":"https://api.github.com/repos/hyrise/hyrise/merges","archive_url":"https://api.github.com/repos/hyrise/hyrise/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/hyrise/hyrise/downloads","issues_url":"https://api.github.com/repos/hyrise/hyrise/issues{/number}","pulls_url":"https://api.github.com/repos/hyrise/hyrise/pulls{/number}","milestones_url":"https://api.github.com/repos/hyrise/hyrise/milestones{/number}","notifications_url":"https://api.github.com/repos/hyrise/hyrise/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/hyrise/hyrise/labels{/name}","releases_url":"https://api.github.com/repos/hyrise/hyrise/releases{/id}","deployments_url":"https://api.github.com/repos/hyrise/hyrise/deployments","created_at":"2017-04-06T10:03:31Z","updated_at":"2018-05-18T21:32:47Z","pushed_at":"2018-05-20T21:51:28Z","git_url":"git://github.com/hyrise/hyrise.git","ssh_url":"git@github.com:hyrise/hyrise.git","clone_url":"https://github.com/hyrise/hyrise.git","svn_url":"https://github.com/hyrise/hyrise","homepage":"https://hpi.de/plattner/projects/hyrise.html","size":11869,"stargazers_count":69,"watchers_count":69,"language":"C++","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":22,"mirror_url":null,"archived":false,"open_issues_count":77,"license":{"key":"mit","name":"MIT License","spdx_id":"MIT","url":"https://api.github.com/licenses/mit"},"forks":22,"open_issues":77,"watchers":69,"default_branch":"master"}},"base":{"label":"hyrise:master","ref":"master","sha":"60fe75892811e89ca8ea52a46aeb3c6eb6441469","user":{"login":"hyrise","id":3465095,"avatar_url":"https://avatars2.githubusercontent.com/u/3465095?v=4","gravatar_id":"","url":"https://api.github.com/users/hyrise","html_url":"https://github.com/hyrise","followers_url":"https://api.github.com/users/hyrise/followers","following_url":"https://api.github.com/users/hyrise/following{/other_user}","gists_url":"https://api.github.com/users/hyrise/gists{/gist_id}","starred_url":"https://api.github.com/users/hyrise/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/hyrise/subscriptions","organizations_url":"https://api.github.com/users/hyrise/orgs","repos_url":"https://api.github.com/users/hyrise/repos","events_url":"https://api.github.com/users/hyrise/events{/privacy}","received_events_url":"https://api.github.com/users/hyrise/received_events","type":"Organization","site_admin":false},"repo":{"id":87414843,"name":"hyrise","full_name":"hyrise/hyrise","owner":{"login":"hyrise","id":3465095,"avatar_url":"https://avatars2.githubusercontent.com/u/3465095?v=4","gravatar_id":"","url":"https://api.github.com/users/hyrise","html_url":"https://github.com/hyrise","followers_url":"https://api.github.com/users/hyrise/followers","following_url":"https://api.github.com/users/hyrise/following{/other_user}","gists_url":"https://api.github.com/users/hyrise/gists{/gist_id}","starred_url":"https://api.github.com/users/hyrise/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/hyrise/subscriptions","organizations_url":"https://api.github.com/users/hyrise/orgs","repos_url":"https://api.github.com/users/hyrise/repos","events_url":"https://api.github.com/users/hyrise/events{/privacy}","received_events_url":"https://api.github.com/users/hyrise/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/hyrise/hyrise","description":"A research in-memory database.","fork":false,"url":"https://api.github.com/repos/hyrise/hyrise","forks_url":"https://api.github.com/repos/hyrise/hyrise/forks","keys_url":"https://api.github.com/repos/hyrise/hyrise/keys{/key_id}","collaborators_url":"https://api.github.com/repos/hyrise/hyrise/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/hyrise/hyrise/teams","hooks_url":"https://api.github.com/repos/hyrise/hyrise/hooks","issue_events_url":"https://api.github.com/repos/hyrise/hyrise/issues/events{/number}","events_url":"https://api.github.com/repos/hyrise/hyrise/events","assignees_url":"https://api.github.com/repos/hyrise/hyrise/assignees{/user}","branches_url":"https://api.github.com/repos/hyrise/hyrise/branches{/branch}","tags_url":"https://api.github.com/repos/hyrise/hyrise/tags","blobs_url":"https://api.github.com/repos/hyrise/hyrise/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/hyrise/hyrise/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/hyrise/hyrise/git/refs{/sha}","trees_url":"https://api.github.com/repos/hyrise/hyrise/git/trees{/sha}","statuses_url":"https://api.github.com/repos/hyrise/hyrise/statuses/{sha}","languages_url":"https://api.github.com/repos/hyrise/hyrise/languages","stargazers_url":"https://api.github.com/repos/hyrise/hyrise/stargazers","contributors_url":"https://api.github.com/repos/hyrise/hyrise/contributors","subscribers_url":"https://api.github.com/repos/hyrise/hyrise/subscribers","subscription_url":"https://api.github.com/repos/hyrise/hyrise/subscription","commits_url":"https://api.github.com/repos/hyrise/hyrise/commits{/sha}","git_commits_url":"https://api.github.com/repos/hyrise/hyrise/git/commits{/sha}","comments_url":"https://api.github.com/repos/hyrise/hyrise/comments{/number}","issue_comment_url":"https://api.github.com/repos/hyrise/hyrise/issues/comments{/number}","contents_url":"https://api.github.com/repos/hyrise/hyrise/contents/{+path}","compare_url":"https://api.github.com/repos/hyrise/hyrise/compare/{base}...{head}","merges_url":"https://api.github.com/repos/hyrise/hyrise/merges","archive_url":"https://api.github.com/repos/hyrise/hyrise/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/hyrise/hyrise/downloads","issues_url":"https://api.github.com/repos/hyrise/hyrise/issues{/number}","pulls_url":"https://api.github.com/repos/hyrise/hyrise/pulls{/number}","milestones_url":"https://api.github.com/repos/hyrise/hyrise/milestones{/number}","notifications_url":"https://api.github.com/repos/hyrise/hyrise/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/hyrise/hyrise/labels{/name}","releases_url":"https://api.github.com/repos/hyrise/hyrise/releases{/id}","deployments_url":"https://api.github.com/repos/hyrise/hyrise/deployments","created_at":"2017-04-06T10:03:31Z","updated_at":"2018-05-18T21:32:47Z","pushed_at":"2018-05-20T21:51:28Z","git_url":"git://github.com/hyrise/hyrise.git","ssh_url":"git@github.com:hyrise/hyrise.git","clone_url":"https://github.com/hyrise/hyrise.git","svn_url":"https://github.com/hyrise/hyrise","homepage":"https://hpi.de/plattner/projects/hyrise.html","size":11869,"stargazers_count":69,"watchers_count":69,"language":"C++","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":22,"mirror_url":null,"archived":false,"open_issues_count":77,"license":{"key":"mit","name":"MIT License","spdx_id":"MIT","url":"https://api.github.com/licenses/mit"},"forks":22,"open_issues":77,"watchers":69,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/hyrise/hyrise/pulls/876"},"html":{"href":"https://github.com/hyrise/hyrise/pull/876"},"issue":{"href":"https://api.github.com/repos/hyrise/hyrise/issues/876"},"comments":{"href":"https://api.github.com/repos/hyrise/hyrise/issues/876/comments"},"review_comments":{"href":"https://api.github.com/repos/hyrise/hyrise/pulls/876/comments"},"review_comment":{"href":"https://api.github.com/repos/hyrise/hyrise/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/hyrise/hyrise/pulls/876/commits"},"statuses":{"href":"https://api.github.com/repos/hyrise/hyrise/statuses/efec5f8051cbbc85b7112d95e1addf8c1c86fd72"}},"author_association":"CONTRIBUTOR"}}
{"actor":{"display_login":"mrks"}}