{"action":"created","comment":{"url":"https://api.github.com/repos/tensorflow/models/pulls/comments/177593008","pull_request_review_id":107486683,"id":177593008,"diff_hunk":"@@ -0,0 +1,141 @@\n+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.\n+#\n+# Licensed under the Apache License, Version 2.0 (the \"License\");\n+# you may not use this file except in compliance with the License.\n+# You may obtain a copy of the License at\n+#\n+#     http://www.apache.org/licenses/LICENSE-2.0\n+#\n+# Unless required by applicable law or agreed to in writing, software\n+# distributed under the License is distributed on an \"AS IS\" BASIS,\n+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+# See the License for the specific language governing permissions and\n+# limitations under the License.\n+# ==============================================================================\n+\n+\"\"\"Library to upload benchmark generated by BenchmarkLogger to remote repo.\n+\n+This library require google cloud bigquery lib as dependency, which can be\n+installed with:\n+  > pip install --upgrade google-cloud-bigquery\n+\"\"\"\n+\n+from __future__ import absolute_import\n+from __future__ import division\n+from __future__ import print_function\n+\n+import argparse\n+import json\n+import os\n+import sys\n+import uuid\n+\n+from google.cloud import bigquery\n+\n+import tensorflow as tf # pylint: disable=g-bad-import-order\n+\n+from official.utils.logging import logger\n+\n+\n+FLAGS = None\n+\n+\n+class BigQueryUploader(object):\n+  \"\"\"Upload the benchmark and metric info to BigQuery.\"\"\"\n+\n+  def __init__(self, logging_dir, gcp_project=None, credentials=None):\n+    \"\"\"Initialized BigQueryUploader with proper setting.\n+\n+    Args:\n+      logging_dir: string, logging directory that contains the benchmark log.\n+      gcp_project: string, the name of the GCP project that the log will be\n+        uploaded to. The default project name will be detected from local\n+        environment if no value is provide.\n+      credentials: google.auth.credentials. The credential to access the\n+        BigQuery service. The default service account credential will be\n+        detected from local environment if no value is provided. Please use\n+        google.oauth2.service_account.Credentials to load credential from local\n+        file for the case that the test is run out side of GCP.\n+    \"\"\"\n+    self._logging_dir = logging_dir\n+    self._bq_client = bigquery.Client(\n+        project=gcp_project, credentials=credentials)\n+\n+  def upload_benchmark_run(self, dataset_name, table_name, run_id):\n+    \"\"\"Upload benchmark run information to Bigquery.\"\"\"\n+    expected_file = os.path.join(\n+        self._logging_dir, logger.BENCHMARK_RUN_LOG_FILE_NAME)\n+    with tf.gfile.GFile(expected_file) as f:\n+      benchmark_json = json.load(f)\n+      benchmark_json[\"model_id\"] = run_id\n+      table_ref = self._bq_client.dataset(dataset_name).table(table_name)\n+      errors = self._bq_client.insert_rows_json(table_ref, [benchmark_json])\n+      if errors:\n+        tf.logging.error(\n+            \"Failed to upload benchmark info to bigquery: {}\".format(errors))\n+\n+  def upload_metric(self, dataset_name, table_name, run_id):\n+    \"\"\"Upload metric information to Bigquery.\"\"\"\n+    expected_file = os.path.join(\n+        self._logging_dir, logger.METRIC_LOG_FILE_NAME)\n+    with tf.gfile.GFile(expected_file) as f:\n+      lines = f.readlines()\n+      metrics = []\n+      for l in lines:\n+        if not l.strip(): continue\n+        metric = json.loads(l)\n+        metric[\"run_id\"] = run_id\n+        metrics.append(metric)\n+      table_ref = self._bq_client.dataset(dataset_name).table(table_name)\n+      errors = self._bq_client.insert_rows_json(table_ref, metrics)\n+      if errors:","path":"official/utils/logging/benchmark_uploader.py","position":91,"original_position":91,"commit_id":"9106239d04773edcb96b3fb1cc3fd71daaa39b31","original_commit_id":"9106239d04773edcb96b3fb1cc3fd71daaa39b31","user":{"login":"robieta","id":13089297,"avatar_url":"https://avatars0.githubusercontent.com/u/13089297?v=4","gravatar_id":"","url":"https://api.github.com/users/robieta","html_url":"https://github.com/robieta","followers_url":"https://api.github.com/users/robieta/followers","following_url":"https://api.github.com/users/robieta/following{/other_user}","gists_url":"https://api.github.com/users/robieta/gists{/gist_id}","starred_url":"https://api.github.com/users/robieta/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/robieta/subscriptions","organizations_url":"https://api.github.com/users/robieta/orgs","repos_url":"https://api.github.com/users/robieta/repos","events_url":"https://api.github.com/users/robieta/events{/privacy}","received_events_url":"https://api.github.com/users/robieta/received_events","type":"User","site_admin":false},"body":"Should we have retry logic here? (Or does the BigTable API already support it.) Once we're containerized we should expect these files will disappear after the test is done, so it would be a shame to lose a run due to a DB connection hiccup.","created_at":"2018-03-27T22:36:18Z","updated_at":"2018-03-27T23:03:19Z","html_url":"https://github.com/tensorflow/models/pull/3776#discussion_r177593008","pull_request_url":"https://api.github.com/repos/tensorflow/models/pulls/3776","author_association":"CONTRIBUTOR","_links":{"self":{"href":"https://api.github.com/repos/tensorflow/models/pulls/comments/177593008"},"html":{"href":"https://github.com/tensorflow/models/pull/3776#discussion_r177593008"},"pull_request":{"href":"https://api.github.com/repos/tensorflow/models/pulls/3776"}}},"pull_request":{"url":"https://api.github.com/repos/tensorflow/models/pulls/3776","id":177837326,"html_url":"https://github.com/tensorflow/models/pull/3776","diff_url":"https://github.com/tensorflow/models/pull/3776.diff","patch_url":"https://github.com/tensorflow/models/pull/3776.patch","issue_url":"https://api.github.com/repos/tensorflow/models/issues/3776","number":3776,"state":"open","locked":false,"title":"Add benchmark upload util to Bigquery.","user":{"login":"qlzh727","id":5118881,"avatar_url":"https://avatars3.githubusercontent.com/u/5118881?v=4","gravatar_id":"","url":"https://api.github.com/users/qlzh727","html_url":"https://github.com/qlzh727","followers_url":"https://api.github.com/users/qlzh727/followers","following_url":"https://api.github.com/users/qlzh727/following{/other_user}","gists_url":"https://api.github.com/users/qlzh727/gists{/gist_id}","starred_url":"https://api.github.com/users/qlzh727/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/qlzh727/subscriptions","organizations_url":"https://api.github.com/users/qlzh727/orgs","repos_url":"https://api.github.com/users/qlzh727/repos","events_url":"https://api.github.com/users/qlzh727/events{/privacy}","received_events_url":"https://api.github.com/users/qlzh727/received_events","type":"User","site_admin":false},"body":"Also update the benchmark logger and bigquery schema for the\r\nerrors found during the integration test.","created_at":"2018-03-27T19:11:44Z","updated_at":"2018-03-27T23:03:19Z","closed_at":null,"merged_at":null,"merge_commit_sha":"6e30133a814bd2d436b85d4ae4bd7c59cc08e91d","assignee":null,"assignees":[],"requested_reviewers":[{"login":"karmel","id":667809,"avatar_url":"https://avatars1.githubusercontent.com/u/667809?v=4","gravatar_id":"","url":"https://api.github.com/users/karmel","html_url":"https://github.com/karmel","followers_url":"https://api.github.com/users/karmel/followers","following_url":"https://api.github.com/users/karmel/following{/other_user}","gists_url":"https://api.github.com/users/karmel/gists{/gist_id}","starred_url":"https://api.github.com/users/karmel/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/karmel/subscriptions","organizations_url":"https://api.github.com/users/karmel/orgs","repos_url":"https://api.github.com/users/karmel/repos","events_url":"https://api.github.com/users/karmel/events{/privacy}","received_events_url":"https://api.github.com/users/karmel/received_events","type":"User","site_admin":false},{"login":"nealwu","id":726075,"avatar_url":"https://avatars3.githubusercontent.com/u/726075?v=4","gravatar_id":"","url":"https://api.github.com/users/nealwu","html_url":"https://github.com/nealwu","followers_url":"https://api.github.com/users/nealwu/followers","following_url":"https://api.github.com/users/nealwu/following{/other_user}","gists_url":"https://api.github.com/users/nealwu/gists{/gist_id}","starred_url":"https://api.github.com/users/nealwu/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/nealwu/subscriptions","organizations_url":"https://api.github.com/users/nealwu/orgs","repos_url":"https://api.github.com/users/nealwu/repos","events_url":"https://api.github.com/users/nealwu/events{/privacy}","received_events_url":"https://api.github.com/users/nealwu/received_events","type":"User","site_admin":false},{"login":"k-w-w","id":31663267,"avatar_url":"https://avatars2.githubusercontent.com/u/31663267?v=4","gravatar_id":"","url":"https://api.github.com/users/k-w-w","html_url":"https://github.com/k-w-w","followers_url":"https://api.github.com/users/k-w-w/followers","following_url":"https://api.github.com/users/k-w-w/following{/other_user}","gists_url":"https://api.github.com/users/k-w-w/gists{/gist_id}","starred_url":"https://api.github.com/users/k-w-w/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/k-w-w/subscriptions","organizations_url":"https://api.github.com/users/k-w-w/orgs","repos_url":"https://api.github.com/users/k-w-w/repos","events_url":"https://api.github.com/users/k-w-w/events{/privacy}","received_events_url":"https://api.github.com/users/k-w-w/received_events","type":"User","site_admin":false},{"login":"yhliang2018","id":36285763,"avatar_url":"https://avatars1.githubusercontent.com/u/36285763?v=4","gravatar_id":"","url":"https://api.github.com/users/yhliang2018","html_url":"https://github.com/yhliang2018","followers_url":"https://api.github.com/users/yhliang2018/followers","following_url":"https://api.github.com/users/yhliang2018/following{/other_user}","gists_url":"https://api.github.com/users/yhliang2018/gists{/gist_id}","starred_url":"https://api.github.com/users/yhliang2018/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/yhliang2018/subscriptions","organizations_url":"https://api.github.com/users/yhliang2018/orgs","repos_url":"https://api.github.com/users/yhliang2018/repos","events_url":"https://api.github.com/users/yhliang2018/events{/privacy}","received_events_url":"https://api.github.com/users/yhliang2018/received_events","type":"User","site_admin":false}],"requested_teams":[],"labels":[{"id":337864321,"url":"https://api.github.com/repos/tensorflow/models/labels/cla:%20yes","name":"cla: yes","color":"009800","default":false}],"milestone":null,"commits_url":"https://api.github.com/repos/tensorflow/models/pulls/3776/commits","review_comments_url":"https://api.github.com/repos/tensorflow/models/pulls/3776/comments","review_comment_url":"https://api.github.com/repos/tensorflow/models/pulls/comments{/number}","comments_url":"https://api.github.com/repos/tensorflow/models/issues/3776/comments","statuses_url":"https://api.github.com/repos/tensorflow/models/statuses/9106239d04773edcb96b3fb1cc3fd71daaa39b31","head":{"label":"qlzh727:metric-upload","ref":"metric-upload","sha":"9106239d04773edcb96b3fb1cc3fd71daaa39b31","user":{"login":"qlzh727","id":5118881,"avatar_url":"https://avatars3.githubusercontent.com/u/5118881?v=4","gravatar_id":"","url":"https://api.github.com/users/qlzh727","html_url":"https://github.com/qlzh727","followers_url":"https://api.github.com/users/qlzh727/followers","following_url":"https://api.github.com/users/qlzh727/following{/other_user}","gists_url":"https://api.github.com/users/qlzh727/gists{/gist_id}","starred_url":"https://api.github.com/users/qlzh727/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/qlzh727/subscriptions","organizations_url":"https://api.github.com/users/qlzh727/orgs","repos_url":"https://api.github.com/users/qlzh727/repos","events_url":"https://api.github.com/users/qlzh727/events{/privacy}","received_events_url":"https://api.github.com/users/qlzh727/received_events","type":"User","site_admin":false},"repo":{"id":123482918,"name":"models","full_name":"qlzh727/models","owner":{"login":"qlzh727","id":5118881,"avatar_url":"https://avatars3.githubusercontent.com/u/5118881?v=4","gravatar_id":"","url":"https://api.github.com/users/qlzh727","html_url":"https://github.com/qlzh727","followers_url":"https://api.github.com/users/qlzh727/followers","following_url":"https://api.github.com/users/qlzh727/following{/other_user}","gists_url":"https://api.github.com/users/qlzh727/gists{/gist_id}","starred_url":"https://api.github.com/users/qlzh727/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/qlzh727/subscriptions","organizations_url":"https://api.github.com/users/qlzh727/orgs","repos_url":"https://api.github.com/users/qlzh727/repos","events_url":"https://api.github.com/users/qlzh727/events{/privacy}","received_events_url":"https://api.github.com/users/qlzh727/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/qlzh727/models","description":"Models and examples built with TensorFlow","fork":true,"url":"https://api.github.com/repos/qlzh727/models","forks_url":"https://api.github.com/repos/qlzh727/models/forks","keys_url":"https://api.github.com/repos/qlzh727/models/keys{/key_id}","collaborators_url":"https://api.github.com/repos/qlzh727/models/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/qlzh727/models/teams","hooks_url":"https://api.github.com/repos/qlzh727/models/hooks","issue_events_url":"https://api.github.com/repos/qlzh727/models/issues/events{/number}","events_url":"https://api.github.com/repos/qlzh727/models/events","assignees_url":"https://api.github.com/repos/qlzh727/models/assignees{/user}","branches_url":"https://api.github.com/repos/qlzh727/models/branches{/branch}","tags_url":"https://api.github.com/repos/qlzh727/models/tags","blobs_url":"https://api.github.com/repos/qlzh727/models/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/qlzh727/models/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/qlzh727/models/git/refs{/sha}","trees_url":"https://api.github.com/repos/qlzh727/models/git/trees{/sha}","statuses_url":"https://api.github.com/repos/qlzh727/models/statuses/{sha}","languages_url":"https://api.github.com/repos/qlzh727/models/languages","stargazers_url":"https://api.github.com/repos/qlzh727/models/stargazers","contributors_url":"https://api.github.com/repos/qlzh727/models/contributors","subscribers_url":"https://api.github.com/repos/qlzh727/models/subscribers","subscription_url":"https://api.github.com/repos/qlzh727/models/subscription","commits_url":"https://api.github.com/repos/qlzh727/models/commits{/sha}","git_commits_url":"https://api.github.com/repos/qlzh727/models/git/commits{/sha}","comments_url":"https://api.github.com/repos/qlzh727/models/comments{/number}","issue_comment_url":"https://api.github.com/repos/qlzh727/models/issues/comments{/number}","contents_url":"https://api.github.com/repos/qlzh727/models/contents/{+path}","compare_url":"https://api.github.com/repos/qlzh727/models/compare/{base}...{head}","merges_url":"https://api.github.com/repos/qlzh727/models/merges","archive_url":"https://api.github.com/repos/qlzh727/models/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/qlzh727/models/downloads","issues_url":"https://api.github.com/repos/qlzh727/models/issues{/number}","pulls_url":"https://api.github.com/repos/qlzh727/models/pulls{/number}","milestones_url":"https://api.github.com/repos/qlzh727/models/milestones{/number}","notifications_url":"https://api.github.com/repos/qlzh727/models/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/qlzh727/models/labels{/name}","releases_url":"https://api.github.com/repos/qlzh727/models/releases{/id}","deployments_url":"https://api.github.com/repos/qlzh727/models/deployments","created_at":"2018-03-01T19:31:59Z","updated_at":"2018-03-27T21:46:13Z","pushed_at":"2018-03-27T22:32:30Z","git_url":"git://github.com/qlzh727/models.git","ssh_url":"git@github.com:qlzh727/models.git","clone_url":"https://github.com/qlzh727/models.git","svn_url":"https://github.com/qlzh727/models","homepage":"","size":432699,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":0,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0"},"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"tensorflow:master","ref":"master","sha":"86cb0aa38d49b8b8435515561f3f834f24163841","user":{"login":"tensorflow","id":15658638,"avatar_url":"https://avatars1.githubusercontent.com/u/15658638?v=4","gravatar_id":"","url":"https://api.github.com/users/tensorflow","html_url":"https://github.com/tensorflow","followers_url":"https://api.github.com/users/tensorflow/followers","following_url":"https://api.github.com/users/tensorflow/following{/other_user}","gists_url":"https://api.github.com/users/tensorflow/gists{/gist_id}","starred_url":"https://api.github.com/users/tensorflow/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tensorflow/subscriptions","organizations_url":"https://api.github.com/users/tensorflow/orgs","repos_url":"https://api.github.com/users/tensorflow/repos","events_url":"https://api.github.com/users/tensorflow/events{/privacy}","received_events_url":"https://api.github.com/users/tensorflow/received_events","type":"Organization","site_admin":false},"repo":{"id":51117837,"name":"models","full_name":"tensorflow/models","owner":{"login":"tensorflow","id":15658638,"avatar_url":"https://avatars1.githubusercontent.com/u/15658638?v=4","gravatar_id":"","url":"https://api.github.com/users/tensorflow","html_url":"https://github.com/tensorflow","followers_url":"https://api.github.com/users/tensorflow/followers","following_url":"https://api.github.com/users/tensorflow/following{/other_user}","gists_url":"https://api.github.com/users/tensorflow/gists{/gist_id}","starred_url":"https://api.github.com/users/tensorflow/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tensorflow/subscriptions","organizations_url":"https://api.github.com/users/tensorflow/orgs","repos_url":"https://api.github.com/users/tensorflow/repos","events_url":"https://api.github.com/users/tensorflow/events{/privacy}","received_events_url":"https://api.github.com/users/tensorflow/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/tensorflow/models","description":"Models and examples built with TensorFlow","fork":false,"url":"https://api.github.com/repos/tensorflow/models","forks_url":"https://api.github.com/repos/tensorflow/models/forks","keys_url":"https://api.github.com/repos/tensorflow/models/keys{/key_id}","collaborators_url":"https://api.github.com/repos/tensorflow/models/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/tensorflow/models/teams","hooks_url":"https://api.github.com/repos/tensorflow/models/hooks","issue_events_url":"https://api.github.com/repos/tensorflow/models/issues/events{/number}","events_url":"https://api.github.com/repos/tensorflow/models/events","assignees_url":"https://api.github.com/repos/tensorflow/models/assignees{/user}","branches_url":"https://api.github.com/repos/tensorflow/models/branches{/branch}","tags_url":"https://api.github.com/repos/tensorflow/models/tags","blobs_url":"https://api.github.com/repos/tensorflow/models/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/tensorflow/models/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/tensorflow/models/git/refs{/sha}","trees_url":"https://api.github.com/repos/tensorflow/models/git/trees{/sha}","statuses_url":"https://api.github.com/repos/tensorflow/models/statuses/{sha}","languages_url":"https://api.github.com/repos/tensorflow/models/languages","stargazers_url":"https://api.github.com/repos/tensorflow/models/stargazers","contributors_url":"https://api.github.com/repos/tensorflow/models/contributors","subscribers_url":"https://api.github.com/repos/tensorflow/models/subscribers","subscription_url":"https://api.github.com/repos/tensorflow/models/subscription","commits_url":"https://api.github.com/repos/tensorflow/models/commits{/sha}","git_commits_url":"https://api.github.com/repos/tensorflow/models/git/commits{/sha}","comments_url":"https://api.github.com/repos/tensorflow/models/comments{/number}","issue_comment_url":"https://api.github.com/repos/tensorflow/models/issues/comments{/number}","contents_url":"https://api.github.com/repos/tensorflow/models/contents/{+path}","compare_url":"https://api.github.com/repos/tensorflow/models/compare/{base}...{head}","merges_url":"https://api.github.com/repos/tensorflow/models/merges","archive_url":"https://api.github.com/repos/tensorflow/models/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/tensorflow/models/downloads","issues_url":"https://api.github.com/repos/tensorflow/models/issues{/number}","pulls_url":"https://api.github.com/repos/tensorflow/models/pulls{/number}","milestones_url":"https://api.github.com/repos/tensorflow/models/milestones{/number}","notifications_url":"https://api.github.com/repos/tensorflow/models/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/tensorflow/models/labels{/name}","releases_url":"https://api.github.com/repos/tensorflow/models/releases{/id}","deployments_url":"https://api.github.com/repos/tensorflow/models/deployments","created_at":"2016-02-05T01:15:20Z","updated_at":"2018-03-27T22:32:57Z","pushed_at":"2018-03-27T22:32:28Z","git_url":"git://github.com/tensorflow/models.git","ssh_url":"git@github.com:tensorflow/models.git","clone_url":"https://github.com/tensorflow/models.git","svn_url":"https://github.com/tensorflow/models","homepage":"","size":432712,"stargazers_count":31613,"watchers_count":31613,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":17577,"mirror_url":null,"archived":false,"open_issues_count":744,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0"},"forks":17577,"open_issues":744,"watchers":31613,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/tensorflow/models/pulls/3776"},"html":{"href":"https://github.com/tensorflow/models/pull/3776"},"issue":{"href":"https://api.github.com/repos/tensorflow/models/issues/3776"},"comments":{"href":"https://api.github.com/repos/tensorflow/models/issues/3776/comments"},"review_comments":{"href":"https://api.github.com/repos/tensorflow/models/pulls/3776/comments"},"review_comment":{"href":"https://api.github.com/repos/tensorflow/models/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/tensorflow/models/pulls/3776/commits"},"statuses":{"href":"https://api.github.com/repos/tensorflow/models/statuses/9106239d04773edcb96b3fb1cc3fd71daaa39b31"}},"author_association":"MEMBER"}}
{"actor":{"display_login":"robieta"}}