{"action":"created","comment":{"url":"https://api.github.com/repos/dotnet/cli/pulls/comments/61774801","id":61774801,"diff_hunk":"@@ -0,0 +1,238 @@\n+#!/usr/bin/env python\n+\n+import argparse\n+import glob\n+import os\n+import re\n+import subprocess\n+import sys\n+\n+SCRIPT_ROOT_PATH = os.path.dirname(os.path.realpath(__file__))\n+PERFTEST_JSON_PATH = os.path.join(SCRIPT_ROOT_PATH, 'project.json')\n+XUNITPERF_REPO_URL = 'https://github.com/microsoft/xunit-performance.git'\n+\n+script_args = None\n+\n+class FatalError(Exception):\n+    def __init__(self, message):\n+        self.message = message\n+\n+def check_requirements():\n+    try:\n+        run_command('git', '--version', quiet = True)\n+    except:\n+        raise FatalError(\"git not found, please make sure that it's installed and on path.\")\n+\n+    try:\n+        run_command('msbuild', '-version', quiet = True)\n+    except:\n+        raise FatalError(\"msbuild not found, please make sure that it's installed and on path.\")\n+\n+    if script_args.xunit_perf_path == None:\n+        raise FatalError(\"Don't know where to clone xunit-performance. Please specify --xunit-perf-path <path>. \" +\n+            \"You can also set/export XUNIT_PERFORMANCE_PATH to not have to set the value every time.\")\n+\n+def process_arguments():\n+    parser = argparse.ArgumentParser(\n+        description = \"Runs CLI perf tests. Requires 'git' and 'msbuild' to be on the PATH.\",\n+    )\n+\n+    parser.add_argument(\n+        'test_cli',\n+        help = \"full path to the dotnet.exe under test\",\n+    )\n+    parser.add_argument(\n+        '--runid', '--name', '-n',\n+        help = \"unique ID for this run\",\n+        required = True,\n+    )\n+    parser.add_argument(\n+        '--base', '--baseline', '-b',\n+        help = \"full path to the baseline dotnet.exe\",\n+        metavar = 'baseline_cli',\n+        dest = 'base_cli',\n+    )\n+    parser.add_argument(\n+        '--xunit-perf-path', '-x',\n+        help = \"\"\"Path to local copy of the xunit-performance repository.\n+            Required unless the environment variable XUNIT_PERFORMANCE_PATH is defined.\"\"\",\n+        default = os.environ.get('XUNIT_PERFORMANCE_PATH'),\n+        metavar = 'path',\n+    )\n+    parser.add_argument(\n+        '--rebuild', '--rebuild-tools', '-r',\n+        help = \"Rebuilds the test tools from scratch.\",\n+        action = 'store_true',\n+    )\n+    parser.add_argument(\n+        '--verbose', '-v',\n+        help = \"Shows the output of all commands run by this script\",\n+        action = 'store_true',\n+    )\n+\n+    global script_args\n+    script_args = parser.parse_args()\n+\n+def run_command(*vargs, **kwargs):\n+    title = kwargs['title'] if 'title' in kwargs else None\n+    from_dir = kwargs['from_dir'] if 'from_dir' in kwargs else None\n+    quiet = kwargs['quiet'] if 'quiet' in kwargs else False\n+\n+    quoted_args = map(lambda x: '\"{x}\"'.format(x=x) if ' ' in x else x, vargs)","path":"test/Performance/run-perftests.py","position":81,"original_position":81,"commit_id":"d722d8983b231705d8e17500ac398ba65b2892b7","original_commit_id":"d722d8983b231705d8e17500ac398ba65b2892b7","user":{"login":"brthor","id":14280892,"avatar_url":"https://avatars.githubusercontent.com/u/14280892?v=3","gravatar_id":"","url":"https://api.github.com/users/brthor","html_url":"https://github.com/brthor","followers_url":"https://api.github.com/users/brthor/followers","following_url":"https://api.github.com/users/brthor/following{/other_user}","gists_url":"https://api.github.com/users/brthor/gists{/gist_id}","starred_url":"https://api.github.com/users/brthor/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/brthor/subscriptions","organizations_url":"https://api.github.com/users/brthor/orgs","repos_url":"https://api.github.com/users/brthor/repos","events_url":"https://api.github.com/users/brthor/events{/privacy}","received_events_url":"https://api.github.com/users/brthor/received_events","type":"User","site_admin":false},"body":"Fair enough, this will cover 80% of the cases, and reimplenting that is a pain.. Sounds good.","created_at":"2016-05-02T17:52:25Z","updated_at":"2016-05-02T17:52:25Z","html_url":"https://github.com/dotnet/cli/pull/2746#discussion_r61774801","pull_request_url":"https://api.github.com/repos/dotnet/cli/pulls/2746","_links":{"self":{"href":"https://api.github.com/repos/dotnet/cli/pulls/comments/61774801"},"html":{"href":"https://github.com/dotnet/cli/pull/2746#discussion_r61774801"},"pull_request":{"href":"https://api.github.com/repos/dotnet/cli/pulls/2746"}}},"pull_request":{"url":"https://api.github.com/repos/dotnet/cli/pulls/2746","id":68310399,"html_url":"https://github.com/dotnet/cli/pull/2746","diff_url":"https://github.com/dotnet/cli/pull/2746.diff","patch_url":"https://github.com/dotnet/cli/pull/2746.patch","issue_url":"https://api.github.com/repos/dotnet/cli/issues/2746","number":2746,"state":"open","locked":false,"title":"Add script to run and compare CLI perf tests","user":{"login":"dpodder","id":2354388,"avatar_url":"https://avatars.githubusercontent.com/u/2354388?v=3","gravatar_id":"","url":"https://api.github.com/users/dpodder","html_url":"https://github.com/dpodder","followers_url":"https://api.github.com/users/dpodder/followers","following_url":"https://api.github.com/users/dpodder/following{/other_user}","gists_url":"https://api.github.com/users/dpodder/gists{/gist_id}","starred_url":"https://api.github.com/users/dpodder/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dpodder/subscriptions","organizations_url":"https://api.github.com/users/dpodder/orgs","repos_url":"https://api.github.com/users/dpodder/repos","events_url":"https://api.github.com/users/dpodder/events{/privacy}","received_events_url":"https://api.github.com/users/dpodder/received_events","type":"User","site_admin":false},"body":"`tests/Performance/run-perftests.py` is a Python3 script that fetches all\r\nof the dependencies needed to run the perf tests in test/Performance\r\nlocally. It can run the perf tests for a single dotnet.exe, or run two\r\ndotnet.exe instances sequentially and compare the results between them.\r\n\r\nBasic usage for a single test:\r\n```batch\r\n  run-perftests.py --name \"runid\" ...\\dotnet.exe\r\n```\r\n\r\nUsage for a comparison run:\r\n```batch\r\n  run-perftests.py --name \"runid\" ...\\dotnet.exe --base ...\\dotnet.exe\r\n```\r\n\r\nFor more detailed usage:\r\n```batch\r\n    run-perftests.py -h\r\n```\n\n<!-- Reviewable:start -->\n---\nThis change is [<img src=\"https://reviewable.io/review_button.svg\" height=\"35\" align=\"absmiddle\" alt=\"Reviewable\"/>](https://reviewable.io/reviews/dotnet/cli/2746)\n<!-- Reviewable:end -->\n","created_at":"2016-04-29T00:17:59Z","updated_at":"2016-05-02T17:52:25Z","closed_at":null,"merged_at":null,"merge_commit_sha":"1b7d1567235989140fabb2da5fbfe186bbcc9231","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/dotnet/cli/pulls/2746/commits","review_comments_url":"https://api.github.com/repos/dotnet/cli/pulls/2746/comments","review_comment_url":"https://api.github.com/repos/dotnet/cli/pulls/comments{/number}","comments_url":"https://api.github.com/repos/dotnet/cli/issues/2746/comments","statuses_url":"https://api.github.com/repos/dotnet/cli/statuses/d722d8983b231705d8e17500ac398ba65b2892b7","head":{"label":"dpodder:dapodd/perf-script","ref":"dapodd/perf-script","sha":"d722d8983b231705d8e17500ac398ba65b2892b7","user":{"login":"dpodder","id":2354388,"avatar_url":"https://avatars.githubusercontent.com/u/2354388?v=3","gravatar_id":"","url":"https://api.github.com/users/dpodder","html_url":"https://github.com/dpodder","followers_url":"https://api.github.com/users/dpodder/followers","following_url":"https://api.github.com/users/dpodder/following{/other_user}","gists_url":"https://api.github.com/users/dpodder/gists{/gist_id}","starred_url":"https://api.github.com/users/dpodder/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dpodder/subscriptions","organizations_url":"https://api.github.com/users/dpodder/orgs","repos_url":"https://api.github.com/users/dpodder/repos","events_url":"https://api.github.com/users/dpodder/events{/privacy}","received_events_url":"https://api.github.com/users/dpodder/received_events","type":"User","site_admin":false},"repo":{"id":57237207,"name":"cli","full_name":"dpodder/cli","owner":{"login":"dpodder","id":2354388,"avatar_url":"https://avatars.githubusercontent.com/u/2354388?v=3","gravatar_id":"","url":"https://api.github.com/users/dpodder","html_url":"https://github.com/dpodder","followers_url":"https://api.github.com/users/dpodder/followers","following_url":"https://api.github.com/users/dpodder/following{/other_user}","gists_url":"https://api.github.com/users/dpodder/gists{/gist_id}","starred_url":"https://api.github.com/users/dpodder/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dpodder/subscriptions","organizations_url":"https://api.github.com/users/dpodder/orgs","repos_url":"https://api.github.com/users/dpodder/repos","events_url":"https://api.github.com/users/dpodder/events{/privacy}","received_events_url":"https://api.github.com/users/dpodder/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/dpodder/cli","description":"This repo contains the .NET Core command-line (CLI) tools, used for building .NET Core apps and libraries through your development flow (compiling, NuGet package management, running, testing, ...).","fork":true,"url":"https://api.github.com/repos/dpodder/cli","forks_url":"https://api.github.com/repos/dpodder/cli/forks","keys_url":"https://api.github.com/repos/dpodder/cli/keys{/key_id}","collaborators_url":"https://api.github.com/repos/dpodder/cli/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/dpodder/cli/teams","hooks_url":"https://api.github.com/repos/dpodder/cli/hooks","issue_events_url":"https://api.github.com/repos/dpodder/cli/issues/events{/number}","events_url":"https://api.github.com/repos/dpodder/cli/events","assignees_url":"https://api.github.com/repos/dpodder/cli/assignees{/user}","branches_url":"https://api.github.com/repos/dpodder/cli/branches{/branch}","tags_url":"https://api.github.com/repos/dpodder/cli/tags","blobs_url":"https://api.github.com/repos/dpodder/cli/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/dpodder/cli/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/dpodder/cli/git/refs{/sha}","trees_url":"https://api.github.com/repos/dpodder/cli/git/trees{/sha}","statuses_url":"https://api.github.com/repos/dpodder/cli/statuses/{sha}","languages_url":"https://api.github.com/repos/dpodder/cli/languages","stargazers_url":"https://api.github.com/repos/dpodder/cli/stargazers","contributors_url":"https://api.github.com/repos/dpodder/cli/contributors","subscribers_url":"https://api.github.com/repos/dpodder/cli/subscribers","subscription_url":"https://api.github.com/repos/dpodder/cli/subscription","commits_url":"https://api.github.com/repos/dpodder/cli/commits{/sha}","git_commits_url":"https://api.github.com/repos/dpodder/cli/git/commits{/sha}","comments_url":"https://api.github.com/repos/dpodder/cli/comments{/number}","issue_comment_url":"https://api.github.com/repos/dpodder/cli/issues/comments{/number}","contents_url":"https://api.github.com/repos/dpodder/cli/contents/{+path}","compare_url":"https://api.github.com/repos/dpodder/cli/compare/{base}...{head}","merges_url":"https://api.github.com/repos/dpodder/cli/merges","archive_url":"https://api.github.com/repos/dpodder/cli/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/dpodder/cli/downloads","issues_url":"https://api.github.com/repos/dpodder/cli/issues{/number}","pulls_url":"https://api.github.com/repos/dpodder/cli/pulls{/number}","milestones_url":"https://api.github.com/repos/dpodder/cli/milestones{/number}","notifications_url":"https://api.github.com/repos/dpodder/cli/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/dpodder/cli/labels{/name}","releases_url":"https://api.github.com/repos/dpodder/cli/releases{/id}","deployments_url":"https://api.github.com/repos/dpodder/cli/deployments","created_at":"2016-04-27T18:40:40Z","updated_at":"2016-04-27T18:40:43Z","pushed_at":"2016-05-02T17:47:31Z","git_url":"git://github.com/dpodder/cli.git","ssh_url":"git@github.com:dpodder/cli.git","clone_url":"https://github.com/dpodder/cli.git","svn_url":"https://github.com/dpodder/cli","homepage":"http://dotnet.github.io/","size":20095,"stargazers_count":0,"watchers_count":0,"language":"C#","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"rel/1.0.0"}},"base":{"label":"dotnet:rel/1.0.0","ref":"rel/1.0.0","sha":"f8631fa4b731d4c903dbe8b0b5e5332eee40ecae","user":{"login":"dotnet","id":9141961,"avatar_url":"https://avatars.githubusercontent.com/u/9141961?v=3","gravatar_id":"","url":"https://api.github.com/users/dotnet","html_url":"https://github.com/dotnet","followers_url":"https://api.github.com/users/dotnet/followers","following_url":"https://api.github.com/users/dotnet/following{/other_user}","gists_url":"https://api.github.com/users/dotnet/gists{/gist_id}","starred_url":"https://api.github.com/users/dotnet/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dotnet/subscriptions","organizations_url":"https://api.github.com/users/dotnet/orgs","repos_url":"https://api.github.com/users/dotnet/repos","events_url":"https://api.github.com/users/dotnet/events{/privacy}","received_events_url":"https://api.github.com/users/dotnet/received_events","type":"Organization","site_admin":false},"repo":{"id":43568107,"name":"cli","full_name":"dotnet/cli","owner":{"login":"dotnet","id":9141961,"avatar_url":"https://avatars.githubusercontent.com/u/9141961?v=3","gravatar_id":"","url":"https://api.github.com/users/dotnet","html_url":"https://github.com/dotnet","followers_url":"https://api.github.com/users/dotnet/followers","following_url":"https://api.github.com/users/dotnet/following{/other_user}","gists_url":"https://api.github.com/users/dotnet/gists{/gist_id}","starred_url":"https://api.github.com/users/dotnet/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dotnet/subscriptions","organizations_url":"https://api.github.com/users/dotnet/orgs","repos_url":"https://api.github.com/users/dotnet/repos","events_url":"https://api.github.com/users/dotnet/events{/privacy}","received_events_url":"https://api.github.com/users/dotnet/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/dotnet/cli","description":"This repo contains the .NET Core command-line (CLI) tools, used for building .NET Core apps and libraries through your development flow (compiling, NuGet package management, running, testing, ...).","fork":false,"url":"https://api.github.com/repos/dotnet/cli","forks_url":"https://api.github.com/repos/dotnet/cli/forks","keys_url":"https://api.github.com/repos/dotnet/cli/keys{/key_id}","collaborators_url":"https://api.github.com/repos/dotnet/cli/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/dotnet/cli/teams","hooks_url":"https://api.github.com/repos/dotnet/cli/hooks","issue_events_url":"https://api.github.com/repos/dotnet/cli/issues/events{/number}","events_url":"https://api.github.com/repos/dotnet/cli/events","assignees_url":"https://api.github.com/repos/dotnet/cli/assignees{/user}","branches_url":"https://api.github.com/repos/dotnet/cli/branches{/branch}","tags_url":"https://api.github.com/repos/dotnet/cli/tags","blobs_url":"https://api.github.com/repos/dotnet/cli/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/dotnet/cli/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/dotnet/cli/git/refs{/sha}","trees_url":"https://api.github.com/repos/dotnet/cli/git/trees{/sha}","statuses_url":"https://api.github.com/repos/dotnet/cli/statuses/{sha}","languages_url":"https://api.github.com/repos/dotnet/cli/languages","stargazers_url":"https://api.github.com/repos/dotnet/cli/stargazers","contributors_url":"https://api.github.com/repos/dotnet/cli/contributors","subscribers_url":"https://api.github.com/repos/dotnet/cli/subscribers","subscription_url":"https://api.github.com/repos/dotnet/cli/subscription","commits_url":"https://api.github.com/repos/dotnet/cli/commits{/sha}","git_commits_url":"https://api.github.com/repos/dotnet/cli/git/commits{/sha}","comments_url":"https://api.github.com/repos/dotnet/cli/comments{/number}","issue_comment_url":"https://api.github.com/repos/dotnet/cli/issues/comments{/number}","contents_url":"https://api.github.com/repos/dotnet/cli/contents/{+path}","compare_url":"https://api.github.com/repos/dotnet/cli/compare/{base}...{head}","merges_url":"https://api.github.com/repos/dotnet/cli/merges","archive_url":"https://api.github.com/repos/dotnet/cli/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/dotnet/cli/downloads","issues_url":"https://api.github.com/repos/dotnet/cli/issues{/number}","pulls_url":"https://api.github.com/repos/dotnet/cli/pulls{/number}","milestones_url":"https://api.github.com/repos/dotnet/cli/milestones{/number}","notifications_url":"https://api.github.com/repos/dotnet/cli/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/dotnet/cli/labels{/name}","releases_url":"https://api.github.com/repos/dotnet/cli/releases{/id}","deployments_url":"https://api.github.com/repos/dotnet/cli/deployments","created_at":"2015-10-02T18:03:15Z","updated_at":"2016-05-02T16:33:36Z","pushed_at":"2016-05-02T17:48:00Z","git_url":"git://github.com/dotnet/cli.git","ssh_url":"git@github.com:dotnet/cli.git","clone_url":"https://github.com/dotnet/cli.git","svn_url":"https://github.com/dotnet/cli","homepage":"http://dotnet.github.io/","size":28756,"stargazers_count":907,"watchers_count":907,"language":"C#","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":182,"mirror_url":null,"open_issues_count":747,"forks":182,"open_issues":747,"watchers":907,"default_branch":"rel/1.0.0"}},"_links":{"self":{"href":"https://api.github.com/repos/dotnet/cli/pulls/2746"},"html":{"href":"https://github.com/dotnet/cli/pull/2746"},"issue":{"href":"https://api.github.com/repos/dotnet/cli/issues/2746"},"comments":{"href":"https://api.github.com/repos/dotnet/cli/issues/2746/comments"},"review_comments":{"href":"https://api.github.com/repos/dotnet/cli/pulls/2746/comments"},"review_comment":{"href":"https://api.github.com/repos/dotnet/cli/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/dotnet/cli/pulls/2746/commits"},"statuses":{"href":"https://api.github.com/repos/dotnet/cli/statuses/d722d8983b231705d8e17500ac398ba65b2892b7"}}}}