{"action":"created","comment":{"url":"https://api.github.com/repos/googlegenomics/pipelines-api-examples/pulls/comments/73604932","id":73604932,"diff_hunk":"@@ -110,12 +114,28 @@ def submit(self, wdl, workflow_inputs, workflow_options, sleep_time=15):\n \n     # Job is running.\n     cromwell_id = job['id']\n-    logging.info(\"Cromwell job id: %s\", cromwell_id)\n+    logging.info(\"Job submitted to Cromwell. job id: %s\", cromwell_id)\n \n-    # Poll for completion.\n+    # Poll Cromwell for job completion.\n+    attempt = 0\n     while True:\n       time.sleep(sleep_time)\n-      status_json = self.fetch(wf_id=cromwell_id, method='status')\n+\n+      # Cromwell occassionally fails to respond to the status request.\n+      # Only give up after 3 consecutive failed requests.\n+      try:\n+        status_json = self.fetch(wf_id=cromwell_id, method='status')\n+        attempt = 0\n+      except requests.exceptions.ConnectionError as e:\n+        attempt += 1\n+        logging.info(\"Error polling Cromwell job status (attempt %d): %s\",\n+          attempt, e)\n+\n+        if attempt >= 3:","path":"wdl_runner/cromwell_launcher/cromwell_driver.py","position":47,"original_position":47,"commit_id":"7020a30d818f2f23c4a92e13ab830ecfeade3075","original_commit_id":"7020a30d818f2f23c4a92e13ab830ecfeade3075","user":{"login":"pgrosu","id":6555937,"avatar_url":"https://avatars.githubusercontent.com/u/6555937?v=3","gravatar_id":"","url":"https://api.github.com/users/pgrosu","html_url":"https://github.com/pgrosu","followers_url":"https://api.github.com/users/pgrosu/followers","following_url":"https://api.github.com/users/pgrosu/following{/other_user}","gists_url":"https://api.github.com/users/pgrosu/gists{/gist_id}","starred_url":"https://api.github.com/users/pgrosu/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/pgrosu/subscriptions","organizations_url":"https://api.github.com/users/pgrosu/orgs","repos_url":"https://api.github.com/users/pgrosu/repos","events_url":"https://api.github.com/users/pgrosu/events{/privacy}","received_events_url":"https://api.github.com/users/pgrosu/received_events","type":"User","site_admin":false},"body":"And it probably also should be `attempt == max_num_retries` since it checks it after every failed attempt, so it will never be more than 3 failed attempts before it exits.","created_at":"2016-08-04T21:20:10Z","updated_at":"2016-08-04T21:20:10Z","html_url":"https://github.com/googlegenomics/pipelines-api-examples/pull/34#discussion_r73604932","pull_request_url":"https://api.github.com/repos/googlegenomics/pipelines-api-examples/pulls/34","_links":{"self":{"href":"https://api.github.com/repos/googlegenomics/pipelines-api-examples/pulls/comments/73604932"},"html":{"href":"https://github.com/googlegenomics/pipelines-api-examples/pull/34#discussion_r73604932"},"pull_request":{"href":"https://api.github.com/repos/googlegenomics/pipelines-api-examples/pulls/34"}}},"pull_request":{"url":"https://api.github.com/repos/googlegenomics/pipelines-api-examples/pulls/34","id":80124486,"html_url":"https://github.com/googlegenomics/pipelines-api-examples/pull/34","diff_url":"https://github.com/googlegenomics/pipelines-api-examples/pull/34.diff","patch_url":"https://github.com/googlegenomics/pipelines-api-examples/pull/34.patch","issue_url":"https://api.github.com/repos/googlegenomics/pipelines-api-examples/issues/34","number":34,"state":"open","locked":false,"title":"Add retries when polling Cromwell for job status.","user":{"login":"mbookman","id":5818294,"avatar_url":"https://avatars.githubusercontent.com/u/5818294?v=3","gravatar_id":"","url":"https://api.github.com/users/mbookman","html_url":"https://github.com/mbookman","followers_url":"https://api.github.com/users/mbookman/followers","following_url":"https://api.github.com/users/mbookman/following{/other_user}","gists_url":"https://api.github.com/users/mbookman/gists{/gist_id}","starred_url":"https://api.github.com/users/mbookman/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mbookman/subscriptions","organizations_url":"https://api.github.com/users/mbookman/orgs","repos_url":"https://api.github.com/users/mbookman/repos","events_url":"https://api.github.com/users/mbookman/events{/privacy}","received_events_url":"https://api.github.com/users/mbookman/received_events","type":"User","site_admin":false},"body":"Giving up after one broken connection made long-running workflows fragile.\r\n\r\nImproves logging output (format now includes timestamps and module name).\r\nAdds a short wait before the first attempt to submit a job to Cromwell,\r\nas it *always* failed and emitted a message before retrying.\r\n\r\nRemove the use of the \"-f\" flag in \"docker tag\" in the README instructions.\r\nThe \"-f\" flag has been removed in docker v1.12.0.","created_at":"2016-08-04T18:14:19Z","updated_at":"2016-08-04T21:20:10Z","closed_at":null,"merged_at":null,"merge_commit_sha":"3d14e7c6ff9b2c7bc50a513c2711f9da95072d18","assignee":{"login":"deflaux","id":1395500,"avatar_url":"https://avatars.githubusercontent.com/u/1395500?v=3","gravatar_id":"","url":"https://api.github.com/users/deflaux","html_url":"https://github.com/deflaux","followers_url":"https://api.github.com/users/deflaux/followers","following_url":"https://api.github.com/users/deflaux/following{/other_user}","gists_url":"https://api.github.com/users/deflaux/gists{/gist_id}","starred_url":"https://api.github.com/users/deflaux/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/deflaux/subscriptions","organizations_url":"https://api.github.com/users/deflaux/orgs","repos_url":"https://api.github.com/users/deflaux/repos","events_url":"https://api.github.com/users/deflaux/events{/privacy}","received_events_url":"https://api.github.com/users/deflaux/received_events","type":"User","site_admin":false},"assignees":[{"login":"deflaux","id":1395500,"avatar_url":"https://avatars.githubusercontent.com/u/1395500?v=3","gravatar_id":"","url":"https://api.github.com/users/deflaux","html_url":"https://github.com/deflaux","followers_url":"https://api.github.com/users/deflaux/followers","following_url":"https://api.github.com/users/deflaux/following{/other_user}","gists_url":"https://api.github.com/users/deflaux/gists{/gist_id}","starred_url":"https://api.github.com/users/deflaux/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/deflaux/subscriptions","organizations_url":"https://api.github.com/users/deflaux/orgs","repos_url":"https://api.github.com/users/deflaux/repos","events_url":"https://api.github.com/users/deflaux/events{/privacy}","received_events_url":"https://api.github.com/users/deflaux/received_events","type":"User","site_admin":false}],"milestone":null,"commits_url":"https://api.github.com/repos/googlegenomics/pipelines-api-examples/pulls/34/commits","review_comments_url":"https://api.github.com/repos/googlegenomics/pipelines-api-examples/pulls/34/comments","review_comment_url":"https://api.github.com/repos/googlegenomics/pipelines-api-examples/pulls/comments{/number}","comments_url":"https://api.github.com/repos/googlegenomics/pipelines-api-examples/issues/34/comments","statuses_url":"https://api.github.com/repos/googlegenomics/pipelines-api-examples/statuses/7020a30d818f2f23c4a92e13ab830ecfeade3075","head":{"label":"mbookman:wdl_runner","ref":"wdl_runner","sha":"7020a30d818f2f23c4a92e13ab830ecfeade3075","user":{"login":"mbookman","id":5818294,"avatar_url":"https://avatars.githubusercontent.com/u/5818294?v=3","gravatar_id":"","url":"https://api.github.com/users/mbookman","html_url":"https://github.com/mbookman","followers_url":"https://api.github.com/users/mbookman/followers","following_url":"https://api.github.com/users/mbookman/following{/other_user}","gists_url":"https://api.github.com/users/mbookman/gists{/gist_id}","starred_url":"https://api.github.com/users/mbookman/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mbookman/subscriptions","organizations_url":"https://api.github.com/users/mbookman/orgs","repos_url":"https://api.github.com/users/mbookman/repos","events_url":"https://api.github.com/users/mbookman/events{/privacy}","received_events_url":"https://api.github.com/users/mbookman/received_events","type":"User","site_admin":false},"repo":{"id":52829203,"name":"pipelines-api-examples","full_name":"mbookman/pipelines-api-examples","owner":{"login":"mbookman","id":5818294,"avatar_url":"https://avatars.githubusercontent.com/u/5818294?v=3","gravatar_id":"","url":"https://api.github.com/users/mbookman","html_url":"https://github.com/mbookman","followers_url":"https://api.github.com/users/mbookman/followers","following_url":"https://api.github.com/users/mbookman/following{/other_user}","gists_url":"https://api.github.com/users/mbookman/gists{/gist_id}","starred_url":"https://api.github.com/users/mbookman/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mbookman/subscriptions","organizations_url":"https://api.github.com/users/mbookman/orgs","repos_url":"https://api.github.com/users/mbookman/repos","events_url":"https://api.github.com/users/mbookman/events{/privacy}","received_events_url":"https://api.github.com/users/mbookman/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/mbookman/pipelines-api-examples","description":"Examples for the Google Genomics Pipelines API.","fork":true,"url":"https://api.github.com/repos/mbookman/pipelines-api-examples","forks_url":"https://api.github.com/repos/mbookman/pipelines-api-examples/forks","keys_url":"https://api.github.com/repos/mbookman/pipelines-api-examples/keys{/key_id}","collaborators_url":"https://api.github.com/repos/mbookman/pipelines-api-examples/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/mbookman/pipelines-api-examples/teams","hooks_url":"https://api.github.com/repos/mbookman/pipelines-api-examples/hooks","issue_events_url":"https://api.github.com/repos/mbookman/pipelines-api-examples/issues/events{/number}","events_url":"https://api.github.com/repos/mbookman/pipelines-api-examples/events","assignees_url":"https://api.github.com/repos/mbookman/pipelines-api-examples/assignees{/user}","branches_url":"https://api.github.com/repos/mbookman/pipelines-api-examples/branches{/branch}","tags_url":"https://api.github.com/repos/mbookman/pipelines-api-examples/tags","blobs_url":"https://api.github.com/repos/mbookman/pipelines-api-examples/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/mbookman/pipelines-api-examples/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/mbookman/pipelines-api-examples/git/refs{/sha}","trees_url":"https://api.github.com/repos/mbookman/pipelines-api-examples/git/trees{/sha}","statuses_url":"https://api.github.com/repos/mbookman/pipelines-api-examples/statuses/{sha}","languages_url":"https://api.github.com/repos/mbookman/pipelines-api-examples/languages","stargazers_url":"https://api.github.com/repos/mbookman/pipelines-api-examples/stargazers","contributors_url":"https://api.github.com/repos/mbookman/pipelines-api-examples/contributors","subscribers_url":"https://api.github.com/repos/mbookman/pipelines-api-examples/subscribers","subscription_url":"https://api.github.com/repos/mbookman/pipelines-api-examples/subscription","commits_url":"https://api.github.com/repos/mbookman/pipelines-api-examples/commits{/sha}","git_commits_url":"https://api.github.com/repos/mbookman/pipelines-api-examples/git/commits{/sha}","comments_url":"https://api.github.com/repos/mbookman/pipelines-api-examples/comments{/number}","issue_comment_url":"https://api.github.com/repos/mbookman/pipelines-api-examples/issues/comments{/number}","contents_url":"https://api.github.com/repos/mbookman/pipelines-api-examples/contents/{+path}","compare_url":"https://api.github.com/repos/mbookman/pipelines-api-examples/compare/{base}...{head}","merges_url":"https://api.github.com/repos/mbookman/pipelines-api-examples/merges","archive_url":"https://api.github.com/repos/mbookman/pipelines-api-examples/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/mbookman/pipelines-api-examples/downloads","issues_url":"https://api.github.com/repos/mbookman/pipelines-api-examples/issues{/number}","pulls_url":"https://api.github.com/repos/mbookman/pipelines-api-examples/pulls{/number}","milestones_url":"https://api.github.com/repos/mbookman/pipelines-api-examples/milestones{/number}","notifications_url":"https://api.github.com/repos/mbookman/pipelines-api-examples/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/mbookman/pipelines-api-examples/labels{/name}","releases_url":"https://api.github.com/repos/mbookman/pipelines-api-examples/releases{/id}","deployments_url":"https://api.github.com/repos/mbookman/pipelines-api-examples/deployments","created_at":"2016-02-29T22:21:21Z","updated_at":"2016-03-01T01:33:21Z","pushed_at":"2016-08-04T18:12:25Z","git_url":"git://github.com/mbookman/pipelines-api-examples.git","ssh_url":"git@github.com:mbookman/pipelines-api-examples.git","clone_url":"https://github.com/mbookman/pipelines-api-examples.git","svn_url":"https://github.com/mbookman/pipelines-api-examples","homepage":"","size":111,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"googlegenomics:master","ref":"master","sha":"9ab83acb69e9cea27c04e2a34a19753f4c2be3ec","user":{"login":"googlegenomics","id":7434120,"avatar_url":"https://avatars.githubusercontent.com/u/7434120?v=3","gravatar_id":"","url":"https://api.github.com/users/googlegenomics","html_url":"https://github.com/googlegenomics","followers_url":"https://api.github.com/users/googlegenomics/followers","following_url":"https://api.github.com/users/googlegenomics/following{/other_user}","gists_url":"https://api.github.com/users/googlegenomics/gists{/gist_id}","starred_url":"https://api.github.com/users/googlegenomics/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/googlegenomics/subscriptions","organizations_url":"https://api.github.com/users/googlegenomics/orgs","repos_url":"https://api.github.com/users/googlegenomics/repos","events_url":"https://api.github.com/users/googlegenomics/events{/privacy}","received_events_url":"https://api.github.com/users/googlegenomics/received_events","type":"Organization","site_admin":false},"repo":{"id":50465020,"name":"pipelines-api-examples","full_name":"googlegenomics/pipelines-api-examples","owner":{"login":"googlegenomics","id":7434120,"avatar_url":"https://avatars.githubusercontent.com/u/7434120?v=3","gravatar_id":"","url":"https://api.github.com/users/googlegenomics","html_url":"https://github.com/googlegenomics","followers_url":"https://api.github.com/users/googlegenomics/followers","following_url":"https://api.github.com/users/googlegenomics/following{/other_user}","gists_url":"https://api.github.com/users/googlegenomics/gists{/gist_id}","starred_url":"https://api.github.com/users/googlegenomics/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/googlegenomics/subscriptions","organizations_url":"https://api.github.com/users/googlegenomics/orgs","repos_url":"https://api.github.com/users/googlegenomics/repos","events_url":"https://api.github.com/users/googlegenomics/events{/privacy}","received_events_url":"https://api.github.com/users/googlegenomics/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/googlegenomics/pipelines-api-examples","description":"Examples for the Google Genomics Pipelines API.","fork":false,"url":"https://api.github.com/repos/googlegenomics/pipelines-api-examples","forks_url":"https://api.github.com/repos/googlegenomics/pipelines-api-examples/forks","keys_url":"https://api.github.com/repos/googlegenomics/pipelines-api-examples/keys{/key_id}","collaborators_url":"https://api.github.com/repos/googlegenomics/pipelines-api-examples/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/googlegenomics/pipelines-api-examples/teams","hooks_url":"https://api.github.com/repos/googlegenomics/pipelines-api-examples/hooks","issue_events_url":"https://api.github.com/repos/googlegenomics/pipelines-api-examples/issues/events{/number}","events_url":"https://api.github.com/repos/googlegenomics/pipelines-api-examples/events","assignees_url":"https://api.github.com/repos/googlegenomics/pipelines-api-examples/assignees{/user}","branches_url":"https://api.github.com/repos/googlegenomics/pipelines-api-examples/branches{/branch}","tags_url":"https://api.github.com/repos/googlegenomics/pipelines-api-examples/tags","blobs_url":"https://api.github.com/repos/googlegenomics/pipelines-api-examples/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/googlegenomics/pipelines-api-examples/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/googlegenomics/pipelines-api-examples/git/refs{/sha}","trees_url":"https://api.github.com/repos/googlegenomics/pipelines-api-examples/git/trees{/sha}","statuses_url":"https://api.github.com/repos/googlegenomics/pipelines-api-examples/statuses/{sha}","languages_url":"https://api.github.com/repos/googlegenomics/pipelines-api-examples/languages","stargazers_url":"https://api.github.com/repos/googlegenomics/pipelines-api-examples/stargazers","contributors_url":"https://api.github.com/repos/googlegenomics/pipelines-api-examples/contributors","subscribers_url":"https://api.github.com/repos/googlegenomics/pipelines-api-examples/subscribers","subscription_url":"https://api.github.com/repos/googlegenomics/pipelines-api-examples/subscription","commits_url":"https://api.github.com/repos/googlegenomics/pipelines-api-examples/commits{/sha}","git_commits_url":"https://api.github.com/repos/googlegenomics/pipelines-api-examples/git/commits{/sha}","comments_url":"https://api.github.com/repos/googlegenomics/pipelines-api-examples/comments{/number}","issue_comment_url":"https://api.github.com/repos/googlegenomics/pipelines-api-examples/issues/comments{/number}","contents_url":"https://api.github.com/repos/googlegenomics/pipelines-api-examples/contents/{+path}","compare_url":"https://api.github.com/repos/googlegenomics/pipelines-api-examples/compare/{base}...{head}","merges_url":"https://api.github.com/repos/googlegenomics/pipelines-api-examples/merges","archive_url":"https://api.github.com/repos/googlegenomics/pipelines-api-examples/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/googlegenomics/pipelines-api-examples/downloads","issues_url":"https://api.github.com/repos/googlegenomics/pipelines-api-examples/issues{/number}","pulls_url":"https://api.github.com/repos/googlegenomics/pipelines-api-examples/pulls{/number}","milestones_url":"https://api.github.com/repos/googlegenomics/pipelines-api-examples/milestones{/number}","notifications_url":"https://api.github.com/repos/googlegenomics/pipelines-api-examples/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/googlegenomics/pipelines-api-examples/labels{/name}","releases_url":"https://api.github.com/repos/googlegenomics/pipelines-api-examples/releases{/id}","deployments_url":"https://api.github.com/repos/googlegenomics/pipelines-api-examples/deployments","created_at":"2016-01-26T22:48:30Z","updated_at":"2016-07-25T00:25:11Z","pushed_at":"2016-08-04T18:14:19Z","git_url":"git://github.com/googlegenomics/pipelines-api-examples.git","ssh_url":"git@github.com:googlegenomics/pipelines-api-examples.git","clone_url":"https://github.com/googlegenomics/pipelines-api-examples.git","svn_url":"https://github.com/googlegenomics/pipelines-api-examples","homepage":"","size":115,"stargazers_count":15,"watchers_count":15,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":8,"mirror_url":null,"open_issues_count":9,"forks":8,"open_issues":9,"watchers":15,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/googlegenomics/pipelines-api-examples/pulls/34"},"html":{"href":"https://github.com/googlegenomics/pipelines-api-examples/pull/34"},"issue":{"href":"https://api.github.com/repos/googlegenomics/pipelines-api-examples/issues/34"},"comments":{"href":"https://api.github.com/repos/googlegenomics/pipelines-api-examples/issues/34/comments"},"review_comments":{"href":"https://api.github.com/repos/googlegenomics/pipelines-api-examples/pulls/34/comments"},"review_comment":{"href":"https://api.github.com/repos/googlegenomics/pipelines-api-examples/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/googlegenomics/pipelines-api-examples/pulls/34/commits"},"statuses":{"href":"https://api.github.com/repos/googlegenomics/pipelines-api-examples/statuses/7020a30d818f2f23c4a92e13ab830ecfeade3075"}}}}
{"actor":{"display_login":"pgrosu"}}