{"action":"created","comment":{"url":"https://api.github.com/repos/grpc/grpc/pulls/comments/57973953","id":57973953,"diff_hunk":"@@ -0,0 +1,353 @@\n+#!/usr/bin/env python2.7\n+# Copyright 2016, Google Inc.\n+# All rights reserved.\n+#\n+# Redistribution and use in source and binary forms, with or without\n+# modification, are permitted provided that the following conditions are\n+# met:\n+#\n+#     * Redistributions of source code must retain the above copyright\n+# notice, this list of conditions and the following disclaimer.\n+#     * Redistributions in binary form must reproduce the above\n+# copyright notice, this list of conditions and the following disclaimer\n+# in the documentation and/or other materials provided with the\n+# distribution.\n+#     * Neither the name of Google Inc. nor the names of its\n+# contributors may be used to endorse or promote products derived from\n+# this software without specific prior written permission.\n+#\n+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS\n+# \"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT\n+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR\n+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT\n+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\n+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\n+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\n+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\n+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n+\n+\"\"\"Run performance tests locally or remotely.\"\"\"\n+\n+import argparse\n+import jobset\n+import multiprocessing\n+import os\n+import subprocess\n+import sys\n+import tempfile\n+import time\n+import uuid\n+\n+\n+_ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..'))\n+os.chdir(_ROOT)\n+\n+\n+_REMOTE_HOST_USERNAME = 'jenkins'\n+\n+\n+class CXXLanguage:\n+\n+  def __init__(self):\n+    self.safename = 'cxx'\n+\n+  def scenarios(self):\n+    # TODO(jtattermusch): add more scenarios\n+    return {\n+            # Scenario 1: generic async streaming ping-pong (contentionless latency)\n+            'cpp_async_generic_streaming_ping_pong': [\n+                '--rpc_type=STREAMING',\n+                '--client_type=ASYNC_CLIENT',\n+                '--server_type=ASYNC_GENERIC_SERVER',\n+                '--outstanding_rpcs_per_channel=1',\n+                '--client_channels=1',\n+                '--bbuf_req_size=0',\n+                '--bbuf_resp_size=0',\n+                '--async_client_threads=1',\n+                '--async_server_threads=1',\n+                '--secure_test=true',\n+                '--num_servers=1',\n+                '--num_clients=1',\n+                '--server_core_limit=0',\n+                '--client_core_limit=0'],\n+            # Scenario 5: Sync unary ping-pong with protobufs\n+            'cpp_sync_unary_ping_pong_protobuf': [\n+                '--rpc_type=UNARY',\n+                '--client_type=SYNC_CLIENT',\n+                '--server_type=SYNC_SERVER',\n+                '--outstanding_rpcs_per_channel=1',\n+                '--client_channels=1',\n+                '--simple_req_size=0',\n+                '--simple_resp_size=0',\n+                '--secure_test=true',\n+                '--num_servers=1',\n+                '--num_clients=1',\n+                '--server_core_limit=0',\n+                '--client_core_limit=0']}\n+\n+  def __str__(self):\n+    return 'c++'\n+\n+\n+class CSharpLanguage:\n+\n+  def __init__(self):\n+    self.safename = str(self)\n+\n+  def __str__(self):\n+    return 'csharp'\n+\n+\n+class NodeLanguage:\n+\n+  def __init__(self):\n+    pass\n+    self.safename = str(self)\n+\n+  def __str__(self):\n+    return 'node'\n+\n+\n+_LANGUAGES = {\n+    'c++' : CXXLanguage(),\n+    'csharp' : CSharpLanguage(),\n+    'node' : NodeLanguage(),\n+}\n+\n+\n+class QpsWorkerJob:\n+  \"\"\"Encapsulates a qps worker server job.\"\"\"\n+\n+  def __init__(self, spec, host_and_port):\n+    self._spec = spec\n+    self.host_and_port = host_and_port\n+    self._job = jobset.Job(spec, bin_hash=None, newline_on_success=True, travis=True, add_env={})\n+\n+  def is_running(self):\n+    \"\"\"Polls a job and returns True if given job is still running.\"\"\"\n+    return self._job.state(jobset.NoCache()) == jobset._RUNNING\n+\n+  def kill(self):\n+    return self._job.kill()\n+\n+\n+def create_qpsworker_job(language, port=10000, remote_host=None):\n+  # TODO: support more languages\n+  cmd = 'bins/opt/qps_worker --driver_port=%s' % port","path":"tools/run_tests/run_performance_tests.py","position":138,"original_position":138,"commit_id":"b27584486474cf4cf6d38641c6446e7114529c83","original_commit_id":"b27584486474cf4cf6d38641c6446e7114529c83","user":{"login":"vjpai","id":8864912,"avatar_url":"https://avatars.githubusercontent.com/u/8864912?v=3","gravatar_id":"","url":"https://api.github.com/users/vjpai","html_url":"https://github.com/vjpai","followers_url":"https://api.github.com/users/vjpai/followers","following_url":"https://api.github.com/users/vjpai/following{/other_user}","gists_url":"https://api.github.com/users/vjpai/gists{/gist_id}","starred_url":"https://api.github.com/users/vjpai/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/vjpai/subscriptions","organizations_url":"https://api.github.com/users/vjpai/orgs","repos_url":"https://api.github.com/users/vjpai/repos","events_url":"https://api.github.com/users/vjpai/events{/privacy}","received_events_url":"https://api.github.com/users/vjpai/received_events","type":"User","site_admin":false},"body":"Will more languages be supported by adding something to their language definition classes above?","created_at":"2016-03-30T22:20:24Z","updated_at":"2016-03-30T22:20:24Z","html_url":"https://github.com/grpc/grpc/pull/5984#discussion_r57973953","pull_request_url":"https://api.github.com/repos/grpc/grpc/pulls/5984","_links":{"self":{"href":"https://api.github.com/repos/grpc/grpc/pulls/comments/57973953"},"html":{"href":"https://github.com/grpc/grpc/pull/5984#discussion_r57973953"},"pull_request":{"href":"https://api.github.com/repos/grpc/grpc/pulls/5984"}}},"pull_request":{"url":"https://api.github.com/repos/grpc/grpc/pulls/5984","id":64401415,"html_url":"https://github.com/grpc/grpc/pull/5984","diff_url":"https://github.com/grpc/grpc/pull/5984.diff","patch_url":"https://github.com/grpc/grpc/pull/5984.patch","issue_url":"https://api.github.com/repos/grpc/grpc/issues/5984","number":5984,"state":"open","locked":false,"title":"Add python script for running performance tests","user":{"login":"jtattermusch","id":9939684,"avatar_url":"https://avatars.githubusercontent.com/u/9939684?v=3","gravatar_id":"","url":"https://api.github.com/users/jtattermusch","html_url":"https://github.com/jtattermusch","followers_url":"https://api.github.com/users/jtattermusch/followers","following_url":"https://api.github.com/users/jtattermusch/following{/other_user}","gists_url":"https://api.github.com/users/jtattermusch/gists{/gist_id}","starred_url":"https://api.github.com/users/jtattermusch/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jtattermusch/subscriptions","organizations_url":"https://api.github.com/users/jtattermusch/orgs","repos_url":"https://api.github.com/users/jtattermusch/repos","events_url":"https://api.github.com/users/jtattermusch/events{/privacy}","received_events_url":"https://api.github.com/users/jtattermusch/received_events","type":"User","site_admin":false},"body":"Add tools/run_tests/run_performance_tests.py script that will soon be the way to run performance tests.\r\n\r\nFeatures supported:\r\n-- running driver / workers either locally or remotely (including pushing the current sources and building remotely on a slave machine)\r\n-- support for multi-machine setup\r\n-- support for multiple languages (not working yet, only added some building blocks so far).\r\n-- less verbose output and build parallelism by reusing jobset.py\r\n\r\nI also created 4 new GCE machines:\r\ngrpc-performance-driver   (4 core, run_performance_tests.py will be spawned by jenkins here, qps_driver will run here and workers will be distributed between server1 and client1-3)\r\ngrpc-performance-server1  (32core)\r\ngrpc-performance-client1  (16core)\r\ngrpc-performance-client2  (16core)\r\ngrpc-performance-client3  (16core)\r\n\r\nWhat needs to be done:\r\n-- fully populate the C++ scenarios to reach parity with run_performance.sh\r\n-- add C# and node scenarios","created_at":"2016-03-29T01:28:46Z","updated_at":"2016-03-30T22:20:24Z","closed_at":null,"merged_at":null,"merge_commit_sha":"9490f9f5779e3db9e00e8915237cb5fdb8ca839f","assignee":{"login":"vjpai","id":8864912,"avatar_url":"https://avatars.githubusercontent.com/u/8864912?v=3","gravatar_id":"","url":"https://api.github.com/users/vjpai","html_url":"https://github.com/vjpai","followers_url":"https://api.github.com/users/vjpai/followers","following_url":"https://api.github.com/users/vjpai/following{/other_user}","gists_url":"https://api.github.com/users/vjpai/gists{/gist_id}","starred_url":"https://api.github.com/users/vjpai/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/vjpai/subscriptions","organizations_url":"https://api.github.com/users/vjpai/orgs","repos_url":"https://api.github.com/users/vjpai/repos","events_url":"https://api.github.com/users/vjpai/events{/privacy}","received_events_url":"https://api.github.com/users/vjpai/received_events","type":"User","site_admin":false},"milestone":null,"commits_url":"https://api.github.com/repos/grpc/grpc/pulls/5984/commits","review_comments_url":"https://api.github.com/repos/grpc/grpc/pulls/5984/comments","review_comment_url":"https://api.github.com/repos/grpc/grpc/pulls/comments{/number}","comments_url":"https://api.github.com/repos/grpc/grpc/issues/5984/comments","statuses_url":"https://api.github.com/repos/grpc/grpc/statuses/b27584486474cf4cf6d38641c6446e7114529c83","head":{"label":"jtattermusch:perf_tests_script","ref":"perf_tests_script","sha":"b27584486474cf4cf6d38641c6446e7114529c83","user":{"login":"jtattermusch","id":9939684,"avatar_url":"https://avatars.githubusercontent.com/u/9939684?v=3","gravatar_id":"","url":"https://api.github.com/users/jtattermusch","html_url":"https://github.com/jtattermusch","followers_url":"https://api.github.com/users/jtattermusch/followers","following_url":"https://api.github.com/users/jtattermusch/following{/other_user}","gists_url":"https://api.github.com/users/jtattermusch/gists{/gist_id}","starred_url":"https://api.github.com/users/jtattermusch/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jtattermusch/subscriptions","organizations_url":"https://api.github.com/users/jtattermusch/orgs","repos_url":"https://api.github.com/users/jtattermusch/repos","events_url":"https://api.github.com/users/jtattermusch/events{/privacy}","received_events_url":"https://api.github.com/users/jtattermusch/received_events","type":"User","site_admin":false},"repo":{"id":31382470,"name":"grpc","full_name":"jtattermusch/grpc","owner":{"login":"jtattermusch","id":9939684,"avatar_url":"https://avatars.githubusercontent.com/u/9939684?v=3","gravatar_id":"","url":"https://api.github.com/users/jtattermusch","html_url":"https://github.com/jtattermusch","followers_url":"https://api.github.com/users/jtattermusch/followers","following_url":"https://api.github.com/users/jtattermusch/following{/other_user}","gists_url":"https://api.github.com/users/jtattermusch/gists{/gist_id}","starred_url":"https://api.github.com/users/jtattermusch/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jtattermusch/subscriptions","organizations_url":"https://api.github.com/users/jtattermusch/orgs","repos_url":"https://api.github.com/users/jtattermusch/repos","events_url":"https://api.github.com/users/jtattermusch/events{/privacy}","received_events_url":"https://api.github.com/users/jtattermusch/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/jtattermusch/grpc","description":"The C based gRPC (C++, Node.js, Python, Ruby, Objective-C, PHP, C#)","fork":true,"url":"https://api.github.com/repos/jtattermusch/grpc","forks_url":"https://api.github.com/repos/jtattermusch/grpc/forks","keys_url":"https://api.github.com/repos/jtattermusch/grpc/keys{/key_id}","collaborators_url":"https://api.github.com/repos/jtattermusch/grpc/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/jtattermusch/grpc/teams","hooks_url":"https://api.github.com/repos/jtattermusch/grpc/hooks","issue_events_url":"https://api.github.com/repos/jtattermusch/grpc/issues/events{/number}","events_url":"https://api.github.com/repos/jtattermusch/grpc/events","assignees_url":"https://api.github.com/repos/jtattermusch/grpc/assignees{/user}","branches_url":"https://api.github.com/repos/jtattermusch/grpc/branches{/branch}","tags_url":"https://api.github.com/repos/jtattermusch/grpc/tags","blobs_url":"https://api.github.com/repos/jtattermusch/grpc/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/jtattermusch/grpc/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/jtattermusch/grpc/git/refs{/sha}","trees_url":"https://api.github.com/repos/jtattermusch/grpc/git/trees{/sha}","statuses_url":"https://api.github.com/repos/jtattermusch/grpc/statuses/{sha}","languages_url":"https://api.github.com/repos/jtattermusch/grpc/languages","stargazers_url":"https://api.github.com/repos/jtattermusch/grpc/stargazers","contributors_url":"https://api.github.com/repos/jtattermusch/grpc/contributors","subscribers_url":"https://api.github.com/repos/jtattermusch/grpc/subscribers","subscription_url":"https://api.github.com/repos/jtattermusch/grpc/subscription","commits_url":"https://api.github.com/repos/jtattermusch/grpc/commits{/sha}","git_commits_url":"https://api.github.com/repos/jtattermusch/grpc/git/commits{/sha}","comments_url":"https://api.github.com/repos/jtattermusch/grpc/comments{/number}","issue_comment_url":"https://api.github.com/repos/jtattermusch/grpc/issues/comments{/number}","contents_url":"https://api.github.com/repos/jtattermusch/grpc/contents/{+path}","compare_url":"https://api.github.com/repos/jtattermusch/grpc/compare/{base}...{head}","merges_url":"https://api.github.com/repos/jtattermusch/grpc/merges","archive_url":"https://api.github.com/repos/jtattermusch/grpc/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/jtattermusch/grpc/downloads","issues_url":"https://api.github.com/repos/jtattermusch/grpc/issues{/number}","pulls_url":"https://api.github.com/repos/jtattermusch/grpc/pulls{/number}","milestones_url":"https://api.github.com/repos/jtattermusch/grpc/milestones{/number}","notifications_url":"https://api.github.com/repos/jtattermusch/grpc/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/jtattermusch/grpc/labels{/name}","releases_url":"https://api.github.com/repos/jtattermusch/grpc/releases{/id}","deployments_url":"https://api.github.com/repos/jtattermusch/grpc/deployments","created_at":"2015-02-26T18:39:52Z","updated_at":"2015-05-08T17:05:57Z","pushed_at":"2016-03-30T20:55:26Z","git_url":"git://github.com/jtattermusch/grpc.git","ssh_url":"git@github.com:jtattermusch/grpc.git","clone_url":"https://github.com/jtattermusch/grpc.git","svn_url":"https://github.com/jtattermusch/grpc","homepage":"","size":65331,"stargazers_count":0,"watchers_count":0,"language":"C","has_issues":false,"has_downloads":false,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"grpc:master","ref":"master","sha":"d29d515ec24101076842a3955fe611ae50c14e96","user":{"login":"grpc","id":7802525,"avatar_url":"https://avatars.githubusercontent.com/u/7802525?v=3","gravatar_id":"","url":"https://api.github.com/users/grpc","html_url":"https://github.com/grpc","followers_url":"https://api.github.com/users/grpc/followers","following_url":"https://api.github.com/users/grpc/following{/other_user}","gists_url":"https://api.github.com/users/grpc/gists{/gist_id}","starred_url":"https://api.github.com/users/grpc/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/grpc/subscriptions","organizations_url":"https://api.github.com/users/grpc/orgs","repos_url":"https://api.github.com/users/grpc/repos","events_url":"https://api.github.com/users/grpc/events{/privacy}","received_events_url":"https://api.github.com/users/grpc/received_events","type":"Organization","site_admin":false},"repo":{"id":27729880,"name":"grpc","full_name":"grpc/grpc","owner":{"login":"grpc","id":7802525,"avatar_url":"https://avatars.githubusercontent.com/u/7802525?v=3","gravatar_id":"","url":"https://api.github.com/users/grpc","html_url":"https://github.com/grpc","followers_url":"https://api.github.com/users/grpc/followers","following_url":"https://api.github.com/users/grpc/following{/other_user}","gists_url":"https://api.github.com/users/grpc/gists{/gist_id}","starred_url":"https://api.github.com/users/grpc/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/grpc/subscriptions","organizations_url":"https://api.github.com/users/grpc/orgs","repos_url":"https://api.github.com/users/grpc/repos","events_url":"https://api.github.com/users/grpc/events{/privacy}","received_events_url":"https://api.github.com/users/grpc/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/grpc/grpc","description":"The C based gRPC (C++, Node.js, Python, Ruby, Objective-C, PHP, C#)","fork":false,"url":"https://api.github.com/repos/grpc/grpc","forks_url":"https://api.github.com/repos/grpc/grpc/forks","keys_url":"https://api.github.com/repos/grpc/grpc/keys{/key_id}","collaborators_url":"https://api.github.com/repos/grpc/grpc/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/grpc/grpc/teams","hooks_url":"https://api.github.com/repos/grpc/grpc/hooks","issue_events_url":"https://api.github.com/repos/grpc/grpc/issues/events{/number}","events_url":"https://api.github.com/repos/grpc/grpc/events","assignees_url":"https://api.github.com/repos/grpc/grpc/assignees{/user}","branches_url":"https://api.github.com/repos/grpc/grpc/branches{/branch}","tags_url":"https://api.github.com/repos/grpc/grpc/tags","blobs_url":"https://api.github.com/repos/grpc/grpc/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/grpc/grpc/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/grpc/grpc/git/refs{/sha}","trees_url":"https://api.github.com/repos/grpc/grpc/git/trees{/sha}","statuses_url":"https://api.github.com/repos/grpc/grpc/statuses/{sha}","languages_url":"https://api.github.com/repos/grpc/grpc/languages","stargazers_url":"https://api.github.com/repos/grpc/grpc/stargazers","contributors_url":"https://api.github.com/repos/grpc/grpc/contributors","subscribers_url":"https://api.github.com/repos/grpc/grpc/subscribers","subscription_url":"https://api.github.com/repos/grpc/grpc/subscription","commits_url":"https://api.github.com/repos/grpc/grpc/commits{/sha}","git_commits_url":"https://api.github.com/repos/grpc/grpc/git/commits{/sha}","comments_url":"https://api.github.com/repos/grpc/grpc/comments{/number}","issue_comment_url":"https://api.github.com/repos/grpc/grpc/issues/comments{/number}","contents_url":"https://api.github.com/repos/grpc/grpc/contents/{+path}","compare_url":"https://api.github.com/repos/grpc/grpc/compare/{base}...{head}","merges_url":"https://api.github.com/repos/grpc/grpc/merges","archive_url":"https://api.github.com/repos/grpc/grpc/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/grpc/grpc/downloads","issues_url":"https://api.github.com/repos/grpc/grpc/issues{/number}","pulls_url":"https://api.github.com/repos/grpc/grpc/pulls{/number}","milestones_url":"https://api.github.com/repos/grpc/grpc/milestones{/number}","notifications_url":"https://api.github.com/repos/grpc/grpc/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/grpc/grpc/labels{/name}","releases_url":"https://api.github.com/repos/grpc/grpc/releases{/id}","deployments_url":"https://api.github.com/repos/grpc/grpc/deployments","created_at":"2014-12-08T18:58:53Z","updated_at":"2016-03-30T13:33:18Z","pushed_at":"2016-03-30T22:19:07Z","git_url":"git://github.com/grpc/grpc.git","ssh_url":"git@github.com:grpc/grpc.git","clone_url":"https://github.com/grpc/grpc.git","svn_url":"https://github.com/grpc/grpc","homepage":"","size":179605,"stargazers_count":4569,"watchers_count":4569,"language":"C","has_issues":true,"has_downloads":false,"has_wiki":true,"has_pages":true,"forks_count":818,"mirror_url":null,"open_issues_count":752,"forks":818,"open_issues":752,"watchers":4569,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/grpc/grpc/pulls/5984"},"html":{"href":"https://github.com/grpc/grpc/pull/5984"},"issue":{"href":"https://api.github.com/repos/grpc/grpc/issues/5984"},"comments":{"href":"https://api.github.com/repos/grpc/grpc/issues/5984/comments"},"review_comments":{"href":"https://api.github.com/repos/grpc/grpc/pulls/5984/comments"},"review_comment":{"href":"https://api.github.com/repos/grpc/grpc/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/grpc/grpc/pulls/5984/commits"},"statuses":{"href":"https://api.github.com/repos/grpc/grpc/statuses/b27584486474cf4cf6d38641c6446e7114529c83"}}}}