{"action":"created","comment":{"url":"https://api.github.com/repos/dotnet/coreclr/pulls/comments/168891843","pull_request_review_id":97342443,"id":168891843,"diff_hunk":"@@ -0,0 +1,177 @@\n+#!/usr/bin/env python3\n+\n+# This script is a simulation of what perf.groovy does in CI. It is not currently used for any official purpose, but I wanted\n+# to check it in hopes it might be useful to others.\n+\n+import ctypes\n+import json\n+import argparse\n+import os\n+import shutil\n+import subprocess\n+import sys\n+import glob\n+from sys import version_info\n+\n+##########################################################################\n+# Argument Parser\n+##########################################################################\n+\n+description = 'Tool to simulate CI perf jobs'\n+\n+parser = argparse.ArgumentParser(description=description)\n+\n+#job arguments\n+parser.add_argument('-job', dest='job', default=None, required=True, choices=['CoreCLR-Scenarios'])\n+parser.add_argument('-isPR', dest='isPR', default=True, choices=[True, False])\n+parser.add_argument('-arch', dest='arch', default='x64', choices=['x64', 'x86'])\n+parser.add_argument('-os', dest='operatingSystem', default='Windows_NT', choices=['Windows_NT', 'Ubuntu16.04', 'Ubuntu14.04', 'OSX'])\n+parser.add_argument('-optLevel', dest='optLevel', default='full_opt', choices=['full_opt', 'min_opt', 'tiered'])\n+parser.add_argument('-jitName', dest='jitName', default='ryujit', choices=['ryujit'])\n+parser.add_argument('-benchviewCommitName', dest='benchviewCommitName', default='FAKE_BV_COMMIT_NAME')\n+parser.add_argument('-bvUploadSasToken', dest='bvUploadSasToken', default='FAKE_BV_SAS_UPLOAD_TOKEN')\n+parser.add_argument('-uploadToBenchview', dest='uploadToBenchview', default=False, choices=[True, False])\n+\n+#ambient jenkins properties\n+parser.add_argument('-workspace', dest='workspace', default=os.path.abspath(os.getcwd()))\n+parser.add_argument('-gitBranchWithoutOrigin', dest='gitBranchWithoutOrigin', default='FAKE_BRANCHNAME')\n+parser.add_argument('-gitBranch', dest='gitBranch', default='origin/FAKE_BRANCHNAME')\n+parser.add_argument('-gitCommit', dest='gitCommit', default='FAKE_GIT_COMMIT')\n+\n+\n+\n+##########################################################################\n+# Helper Functions\n+##########################################################################\n+\n+def log(message):\n+    \"\"\" Print logging information\n+    Args:\n+        message (str): message to be printed\n+    \"\"\"\n+\n+    print('[%s]: %s' % (sys.argv[0], message))\n+\n+def is_supported_version() -> bool:\n+    \"\"\" Checkes that the version of python is at least 3.5\n+    Returns:\n+        bool : true if python version is 3.5 or higher\n+    \"\"\"\n+    return version_info.major > 2 and version_info.minor > 4\n+\n+def run_command(runArgs, environment):\n+    \"\"\" Run command specified by runArgs\n+    Args:\n+        runargs (str[]): list of arguments for subprocess\n+        environment(str{}): dict of environment variable\n+    \"\"\"\n+    log('RUNNING COMMAND: ' + runArgs)\n+    subprocess.check_call(runArgs, env=environment)\n+\n+##########################################################################\n+# CI jobs\n+##########################################################################\n+\n+class jenkins_properties:\n+    def __init__(self, workspace, git_branch_without_origin, git_branch, git_commit):\n+        self.workspace = workspace\n+        self.git_branch_without_origin = git_branch_without_origin\n+        self.git_branch = git_branch\n+        self.git_commit = git_commit\n+\n+\n+class coreclr_scenarios_args:\n+    def __init__(self, is_pr, arch, os, jit_name, opt_level, benchview_commit_name, bv_upload_sas_token, upload_to_benchview):\n+        self.is_pr = is_pr\n+        self.arch = arch\n+        self.os = os\n+        self.jit_name = jit_name\n+        self.opt_level = opt_level\n+        self.benchview_commit_name = benchview_commit_name\n+        self.bv_upload_sas_token = bv_upload_sas_token\n+        self.upload_to_benchview = upload_to_benchview\n+\n+def coreclr_scenarios(args, props):\n+    architecture = args.arch\n+    testEnv = ''\n+    configuration = 'Release'\n+    runType = 'private' if args.is_pr else 'rolling'\n+    benchViewName = 'CoreCLR-Scenarios private ' + args.benchview_commit_name if args.is_pr else 'CoreCLR-Scenarios rolling ' + props.git_branch_without_origin + ' ' + props.git_commit\n+    uploadString = '-uploadToBenchview' if args.upload_to_benchview else ''\n+    \n+    run_command('powershell -NoProfile wget https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -OutFile \\\"' + props.workspace + '\\\\nuget.exe\\\"', os.environ)\n+    if os.path.isdir(props.workspace + '\\\\Microsoft.BenchView.JSONFormat'):\n+        shutil.rmtree(props.workspace + '\\\\Microsoft.BenchView.JSONFormat')","path":"tests/scripts/ci_perf.py","position":104,"original_position":104,"commit_id":"44f0f2c2752e0e65c7bbeb12c6cea6111d6c0d90","original_commit_id":"44f0f2c2752e0e65c7bbeb12c6cea6111d6c0d90","user":{"login":"noahfalk","id":6243776,"avatar_url":"https://avatars0.githubusercontent.com/u/6243776?v=4","gravatar_id":"","url":"https://api.github.com/users/noahfalk","html_url":"https://github.com/noahfalk","followers_url":"https://api.github.com/users/noahfalk/followers","following_url":"https://api.github.com/users/noahfalk/following{/other_user}","gists_url":"https://api.github.com/users/noahfalk/gists{/gist_id}","starred_url":"https://api.github.com/users/noahfalk/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/noahfalk/subscriptions","organizations_url":"https://api.github.com/users/noahfalk/orgs","repos_url":"https://api.github.com/users/noahfalk/repos","events_url":"https://api.github.com/users/noahfalk/events{/privacy}","received_events_url":"https://api.github.com/users/noahfalk/received_events","type":"User","site_admin":false},"body":"I agree that all of these suggestions would be improvements to the algorithm, but my priority in this script was to repro issues that might occur in CI, and to do that I tried to stay faithful to the source material.  All of the issues you call out are present in perf.groovy. I'd be glad to see perf.groovy improved as you suggest, but that is beyond the scope of work I'm willing to undertake.\r\n\r\nIf you want to do things you are suggesting I'd suggest someone else could either:\r\na) update perf.groovy to call this script directly and then improve this script as desired.\r\nb) update perf.groovy with these improvements, then update this script to match\r\n","created_at":"2018-02-16T23:00:35Z","updated_at":"2018-02-16T23:00:35Z","html_url":"https://github.com/dotnet/coreclr/pull/16414#discussion_r168891843","pull_request_url":"https://api.github.com/repos/dotnet/coreclr/pulls/16414","author_association":"MEMBER","_links":{"self":{"href":"https://api.github.com/repos/dotnet/coreclr/pulls/comments/168891843"},"html":{"href":"https://github.com/dotnet/coreclr/pull/16414#discussion_r168891843"},"pull_request":{"href":"https://api.github.com/repos/dotnet/coreclr/pulls/16414"}},"in_reply_to_id":168809944},"pull_request":{"url":"https://api.github.com/repos/dotnet/coreclr/pulls/16414","id":169530227,"html_url":"https://github.com/dotnet/coreclr/pull/16414","diff_url":"https://github.com/dotnet/coreclr/pull/16414.diff","patch_url":"https://github.com/dotnet/coreclr/pull/16414.patch","issue_url":"https://api.github.com/repos/dotnet/coreclr/issues/16414","number":16414,"state":"open","locked":false,"title":"Add CI perf repro script","user":{"login":"noahfalk","id":6243776,"avatar_url":"https://avatars0.githubusercontent.com/u/6243776?v=4","gravatar_id":"","url":"https://api.github.com/users/noahfalk","html_url":"https://github.com/noahfalk","followers_url":"https://api.github.com/users/noahfalk/followers","following_url":"https://api.github.com/users/noahfalk/following{/other_user}","gists_url":"https://api.github.com/users/noahfalk/gists{/gist_id}","starred_url":"https://api.github.com/users/noahfalk/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/noahfalk/subscriptions","organizations_url":"https://api.github.com/users/noahfalk/orgs","repos_url":"https://api.github.com/users/noahfalk/repos","events_url":"https://api.github.com/users/noahfalk/events{/privacy}","received_events_url":"https://api.github.com/users/noahfalk/received_events","type":"User","site_admin":false},"body":"There wasn't any good way to reproduce what happens in CI for perf jobs because groovy can't be directly run. As a first step to solving this I created a python script that reproduces the perf leg I needed.","created_at":"2018-02-16T01:28:31Z","updated_at":"2018-02-16T23:00:35Z","closed_at":null,"merged_at":null,"merge_commit_sha":"7cc18fbda2a5df95df3277e4c893da47af05ac3e","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/dotnet/coreclr/pulls/16414/commits","review_comments_url":"https://api.github.com/repos/dotnet/coreclr/pulls/16414/comments","review_comment_url":"https://api.github.com/repos/dotnet/coreclr/pulls/comments{/number}","comments_url":"https://api.github.com/repos/dotnet/coreclr/issues/16414/comments","statuses_url":"https://api.github.com/repos/dotnet/coreclr/statuses/44f0f2c2752e0e65c7bbeb12c6cea6111d6c0d90","head":{"label":"noahfalk:simulate_perf_ci","ref":"simulate_perf_ci","sha":"44f0f2c2752e0e65c7bbeb12c6cea6111d6c0d90","user":{"login":"noahfalk","id":6243776,"avatar_url":"https://avatars0.githubusercontent.com/u/6243776?v=4","gravatar_id":"","url":"https://api.github.com/users/noahfalk","html_url":"https://github.com/noahfalk","followers_url":"https://api.github.com/users/noahfalk/followers","following_url":"https://api.github.com/users/noahfalk/following{/other_user}","gists_url":"https://api.github.com/users/noahfalk/gists{/gist_id}","starred_url":"https://api.github.com/users/noahfalk/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/noahfalk/subscriptions","organizations_url":"https://api.github.com/users/noahfalk/orgs","repos_url":"https://api.github.com/users/noahfalk/repos","events_url":"https://api.github.com/users/noahfalk/events{/privacy}","received_events_url":"https://api.github.com/users/noahfalk/received_events","type":"User","site_admin":false},"repo":{"id":42578269,"name":"coreclr","full_name":"noahfalk/coreclr","owner":{"login":"noahfalk","id":6243776,"avatar_url":"https://avatars0.githubusercontent.com/u/6243776?v=4","gravatar_id":"","url":"https://api.github.com/users/noahfalk","html_url":"https://github.com/noahfalk","followers_url":"https://api.github.com/users/noahfalk/followers","following_url":"https://api.github.com/users/noahfalk/following{/other_user}","gists_url":"https://api.github.com/users/noahfalk/gists{/gist_id}","starred_url":"https://api.github.com/users/noahfalk/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/noahfalk/subscriptions","organizations_url":"https://api.github.com/users/noahfalk/orgs","repos_url":"https://api.github.com/users/noahfalk/repos","events_url":"https://api.github.com/users/noahfalk/events{/privacy}","received_events_url":"https://api.github.com/users/noahfalk/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/noahfalk/coreclr","description":"This repo contains the .NET Core runtime, called CoreCLR, and the base library, called mscorlib. It includes the garbage collector, JIT compiler, base .NET data types and many low-level classes.","fork":true,"url":"https://api.github.com/repos/noahfalk/coreclr","forks_url":"https://api.github.com/repos/noahfalk/coreclr/forks","keys_url":"https://api.github.com/repos/noahfalk/coreclr/keys{/key_id}","collaborators_url":"https://api.github.com/repos/noahfalk/coreclr/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/noahfalk/coreclr/teams","hooks_url":"https://api.github.com/repos/noahfalk/coreclr/hooks","issue_events_url":"https://api.github.com/repos/noahfalk/coreclr/issues/events{/number}","events_url":"https://api.github.com/repos/noahfalk/coreclr/events","assignees_url":"https://api.github.com/repos/noahfalk/coreclr/assignees{/user}","branches_url":"https://api.github.com/repos/noahfalk/coreclr/branches{/branch}","tags_url":"https://api.github.com/repos/noahfalk/coreclr/tags","blobs_url":"https://api.github.com/repos/noahfalk/coreclr/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/noahfalk/coreclr/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/noahfalk/coreclr/git/refs{/sha}","trees_url":"https://api.github.com/repos/noahfalk/coreclr/git/trees{/sha}","statuses_url":"https://api.github.com/repos/noahfalk/coreclr/statuses/{sha}","languages_url":"https://api.github.com/repos/noahfalk/coreclr/languages","stargazers_url":"https://api.github.com/repos/noahfalk/coreclr/stargazers","contributors_url":"https://api.github.com/repos/noahfalk/coreclr/contributors","subscribers_url":"https://api.github.com/repos/noahfalk/coreclr/subscribers","subscription_url":"https://api.github.com/repos/noahfalk/coreclr/subscription","commits_url":"https://api.github.com/repos/noahfalk/coreclr/commits{/sha}","git_commits_url":"https://api.github.com/repos/noahfalk/coreclr/git/commits{/sha}","comments_url":"https://api.github.com/repos/noahfalk/coreclr/comments{/number}","issue_comment_url":"https://api.github.com/repos/noahfalk/coreclr/issues/comments{/number}","contents_url":"https://api.github.com/repos/noahfalk/coreclr/contents/{+path}","compare_url":"https://api.github.com/repos/noahfalk/coreclr/compare/{base}...{head}","merges_url":"https://api.github.com/repos/noahfalk/coreclr/merges","archive_url":"https://api.github.com/repos/noahfalk/coreclr/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/noahfalk/coreclr/downloads","issues_url":"https://api.github.com/repos/noahfalk/coreclr/issues{/number}","pulls_url":"https://api.github.com/repos/noahfalk/coreclr/pulls{/number}","milestones_url":"https://api.github.com/repos/noahfalk/coreclr/milestones{/number}","notifications_url":"https://api.github.com/repos/noahfalk/coreclr/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/noahfalk/coreclr/labels{/name}","releases_url":"https://api.github.com/repos/noahfalk/coreclr/releases{/id}","deployments_url":"https://api.github.com/repos/noahfalk/coreclr/deployments","created_at":"2015-09-16T09:38:49Z","updated_at":"2016-02-28T06:04:15Z","pushed_at":"2018-02-16T01:26:39Z","git_url":"git://github.com/noahfalk/coreclr.git","ssh_url":"git@github.com:noahfalk/coreclr.git","clone_url":"https://github.com/noahfalk/coreclr.git","svn_url":"https://github.com/noahfalk/coreclr","homepage":"http://dotnet.github.io/core/","size":206067,"stargazers_count":0,"watchers_count":0,"language":"C#","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":0,"license":{"key":"mit","name":"MIT License","spdx_id":"MIT","url":"https://api.github.com/licenses/mit"},"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"dotnet:master","ref":"master","sha":"dab4790f86d857c9a124d474e48fe49b96abcd5c","user":{"login":"dotnet","id":9141961,"avatar_url":"https://avatars0.githubusercontent.com/u/9141961?v=4","gravatar_id":"","url":"https://api.github.com/users/dotnet","html_url":"https://github.com/dotnet","followers_url":"https://api.github.com/users/dotnet/followers","following_url":"https://api.github.com/users/dotnet/following{/other_user}","gists_url":"https://api.github.com/users/dotnet/gists{/gist_id}","starred_url":"https://api.github.com/users/dotnet/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dotnet/subscriptions","organizations_url":"https://api.github.com/users/dotnet/orgs","repos_url":"https://api.github.com/users/dotnet/repos","events_url":"https://api.github.com/users/dotnet/events{/privacy}","received_events_url":"https://api.github.com/users/dotnet/received_events","type":"Organization","site_admin":false},"repo":{"id":30092893,"name":"coreclr","full_name":"dotnet/coreclr","owner":{"login":"dotnet","id":9141961,"avatar_url":"https://avatars0.githubusercontent.com/u/9141961?v=4","gravatar_id":"","url":"https://api.github.com/users/dotnet","html_url":"https://github.com/dotnet","followers_url":"https://api.github.com/users/dotnet/followers","following_url":"https://api.github.com/users/dotnet/following{/other_user}","gists_url":"https://api.github.com/users/dotnet/gists{/gist_id}","starred_url":"https://api.github.com/users/dotnet/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dotnet/subscriptions","organizations_url":"https://api.github.com/users/dotnet/orgs","repos_url":"https://api.github.com/users/dotnet/repos","events_url":"https://api.github.com/users/dotnet/events{/privacy}","received_events_url":"https://api.github.com/users/dotnet/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/dotnet/coreclr","description":"This repo contains the .NET Core runtime, called CoreCLR, and the base library, called System.Private.Corelib (or mscorlib). It includes the garbage collector, JIT compiler, base .NET data types and many low-level classes. We welcome contributions.","fork":false,"url":"https://api.github.com/repos/dotnet/coreclr","forks_url":"https://api.github.com/repos/dotnet/coreclr/forks","keys_url":"https://api.github.com/repos/dotnet/coreclr/keys{/key_id}","collaborators_url":"https://api.github.com/repos/dotnet/coreclr/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/dotnet/coreclr/teams","hooks_url":"https://api.github.com/repos/dotnet/coreclr/hooks","issue_events_url":"https://api.github.com/repos/dotnet/coreclr/issues/events{/number}","events_url":"https://api.github.com/repos/dotnet/coreclr/events","assignees_url":"https://api.github.com/repos/dotnet/coreclr/assignees{/user}","branches_url":"https://api.github.com/repos/dotnet/coreclr/branches{/branch}","tags_url":"https://api.github.com/repos/dotnet/coreclr/tags","blobs_url":"https://api.github.com/repos/dotnet/coreclr/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/dotnet/coreclr/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/dotnet/coreclr/git/refs{/sha}","trees_url":"https://api.github.com/repos/dotnet/coreclr/git/trees{/sha}","statuses_url":"https://api.github.com/repos/dotnet/coreclr/statuses/{sha}","languages_url":"https://api.github.com/repos/dotnet/coreclr/languages","stargazers_url":"https://api.github.com/repos/dotnet/coreclr/stargazers","contributors_url":"https://api.github.com/repos/dotnet/coreclr/contributors","subscribers_url":"https://api.github.com/repos/dotnet/coreclr/subscribers","subscription_url":"https://api.github.com/repos/dotnet/coreclr/subscription","commits_url":"https://api.github.com/repos/dotnet/coreclr/commits{/sha}","git_commits_url":"https://api.github.com/repos/dotnet/coreclr/git/commits{/sha}","comments_url":"https://api.github.com/repos/dotnet/coreclr/comments{/number}","issue_comment_url":"https://api.github.com/repos/dotnet/coreclr/issues/comments{/number}","contents_url":"https://api.github.com/repos/dotnet/coreclr/contents/{+path}","compare_url":"https://api.github.com/repos/dotnet/coreclr/compare/{base}...{head}","merges_url":"https://api.github.com/repos/dotnet/coreclr/merges","archive_url":"https://api.github.com/repos/dotnet/coreclr/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/dotnet/coreclr/downloads","issues_url":"https://api.github.com/repos/dotnet/coreclr/issues{/number}","pulls_url":"https://api.github.com/repos/dotnet/coreclr/pulls{/number}","milestones_url":"https://api.github.com/repos/dotnet/coreclr/milestones{/number}","notifications_url":"https://api.github.com/repos/dotnet/coreclr/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/dotnet/coreclr/labels{/name}","releases_url":"https://api.github.com/repos/dotnet/coreclr/releases{/id}","deployments_url":"https://api.github.com/repos/dotnet/coreclr/deployments","created_at":"2015-01-30T22:10:02Z","updated_at":"2018-02-16T21:34:33Z","pushed_at":"2018-02-16T22:48:08Z","git_url":"git://github.com/dotnet/coreclr.git","ssh_url":"git@github.com:dotnet/coreclr.git","clone_url":"https://github.com/dotnet/coreclr.git","svn_url":"https://github.com/dotnet/coreclr","homepage":"https://github.com/dotnet/core","size":207669,"stargazers_count":8943,"watchers_count":8943,"language":"C#","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":2251,"mirror_url":null,"archived":false,"open_issues_count":1510,"license":null,"forks":2251,"open_issues":1510,"watchers":8943,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/dotnet/coreclr/pulls/16414"},"html":{"href":"https://github.com/dotnet/coreclr/pull/16414"},"issue":{"href":"https://api.github.com/repos/dotnet/coreclr/issues/16414"},"comments":{"href":"https://api.github.com/repos/dotnet/coreclr/issues/16414/comments"},"review_comments":{"href":"https://api.github.com/repos/dotnet/coreclr/pulls/16414/comments"},"review_comment":{"href":"https://api.github.com/repos/dotnet/coreclr/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/dotnet/coreclr/pulls/16414/commits"},"statuses":{"href":"https://api.github.com/repos/dotnet/coreclr/statuses/44f0f2c2752e0e65c7bbeb12c6cea6111d6c0d90"}},"author_association":"MEMBER"}}
{"actor":{"display_login":"noahfalk"}}