{"action":"created","comment":{"url":"https://api.github.com/repos/ESMCI/cime/pulls/comments/185560719","pull_request_review_id":116975144,"id":185560719,"diff_hunk":"@@ -106,29 +115,36 @@ def _run_model_impl(case, lid, skip_pnl=False, da_cycle=0):\n         stat = run_and_log_case_status(run_func, \"model execution\", caseroot=case.get_value(\"CASEROOT\"))\n         model_logfile = os.path.join(rundir, model + \".log.\" + lid)\n         # Determine if failure was due to a failed node, if so, try to restart\n-        if node_fail_re:\n-            node_fail_regex = re.compile(node_fail_re)\n+        if retry_run_re or node_fail_re:\n             model_logfile = os.path.join(rundir, model + \".log.\" + lid)\n             if os.path.exists(model_logfile):\n-                num_fails = len(node_fail_regex.findall(open(model_logfile, 'r').read()))\n-                if num_fails > 0 and case.spare_nodes >= num_fails:\n+                num_node_fails=0\n+                num_retry_fails=0\n+                if node_fail_re:\n+                    num_node_fails = len(node_fail_regex.findall(open(model_logfile, 'r').read()))\n+                if retry_run_re:\n+                    num_retry_fails = len(retry_run_regex.findall(open(model_logfile, 'r').read()))\n+                logger.debug (\"RETRY: num_retry_fails {} spare_nodes {} retry_count {}\".\n+                              format(num_retry_fails, case.spare_nodes, retry_count))\n+                if num_node_fails > 0 and case.spare_nodes >= num_node_fails:\n                         # We failed due to node failure!\n                     logger.warning(\"Detected model run failed due to node failure, restarting\")\n-\n+                    case.spare_nodes -= num_node_fails\n+                    loop = True\n+                elif num_retry_fails > 0 and retry_count >= num_retry_fails:\n+                    logger.warning(\"Detected model run failed, restarting\")\n+                    retry_count -= num_retry_fails\n+                    loop = True\n+                if loop:\n                     # Archive the last consistent set of restart files and restore them\n-                    case.case_st_archive(no_resubmit=True)\n-                    case.restore_from_archive()\n-\n+                    if case.get_value(\"DOUT_S\"):","path":"scripts/lib/CIME/case/case_run.py","position":57,"original_position":57,"commit_id":"b293d59849f0211f37429576103361c54caa7cc6","original_commit_id":"b293d59849f0211f37429576103361c54caa7cc6","user":{"login":"jedwards4b","id":6126104,"avatar_url":"https://avatars3.githubusercontent.com/u/6126104?v=4","gravatar_id":"","url":"https://api.github.com/users/jedwards4b","html_url":"https://github.com/jedwards4b","followers_url":"https://api.github.com/users/jedwards4b/followers","following_url":"https://api.github.com/users/jedwards4b/following{/other_user}","gists_url":"https://api.github.com/users/jedwards4b/gists{/gist_id}","starred_url":"https://api.github.com/users/jedwards4b/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jedwards4b/subscriptions","organizations_url":"https://api.github.com/users/jedwards4b/orgs","repos_url":"https://api.github.com/users/jedwards4b/repos","events_url":"https://api.github.com/users/jedwards4b/events{/privacy}","received_events_url":"https://api.github.com/users/jedwards4b/received_events","type":"User","site_admin":false},"body":"If DOUT_S is off the restart files remain in the run directory - in most cases DOUT_S is true.   In cases where DOUT_S is False the archive directory was never created and so you would get an error trying to write there. ","created_at":"2018-05-02T16:37:22Z","updated_at":"2018-05-02T16:37:22Z","html_url":"https://github.com/ESMCI/cime/pull/2526#discussion_r185560719","pull_request_url":"https://api.github.com/repos/ESMCI/cime/pulls/2526","author_association":"CONTRIBUTOR","_links":{"self":{"href":"https://api.github.com/repos/ESMCI/cime/pulls/comments/185560719"},"html":{"href":"https://github.com/ESMCI/cime/pull/2526#discussion_r185560719"},"pull_request":{"href":"https://api.github.com/repos/ESMCI/cime/pulls/2526"}},"in_reply_to_id":185554521},"pull_request":{"url":"https://api.github.com/repos/ESMCI/cime/pulls/2526","id":185279451,"html_url":"https://github.com/ESMCI/cime/pull/2526","diff_url":"https://github.com/ESMCI/cime/pull/2526.diff","patch_url":"https://github.com/ESMCI/cime/pull/2526.patch","issue_url":"https://api.github.com/repos/ESMCI/cime/issues/2526","number":2526,"state":"open","locked":false,"title":"rename NODE_FAIL RETRY_MPIRUN","user":{"login":"jedwards4b","id":6126104,"avatar_url":"https://avatars3.githubusercontent.com/u/6126104?v=4","gravatar_id":"","url":"https://api.github.com/users/jedwards4b","html_url":"https://github.com/jedwards4b","followers_url":"https://api.github.com/users/jedwards4b/followers","following_url":"https://api.github.com/users/jedwards4b/following{/other_user}","gists_url":"https://api.github.com/users/jedwards4b/gists{/gist_id}","starred_url":"https://api.github.com/users/jedwards4b/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jedwards4b/subscriptions","organizations_url":"https://api.github.com/users/jedwards4b/orgs","repos_url":"https://api.github.com/users/jedwards4b/repos","events_url":"https://api.github.com/users/jedwards4b/events{/privacy}","received_events_url":"https://api.github.com/users/jedwards4b/received_events","type":"User","site_admin":false},"body":"Renaming some variables to clarify functionality NODE_FAIL was too narrow in scope since there\r\nare other reasons you might want to reattempt the mpirun command\r\n\r\nTest suite: scripts_regression_tests, hand testing of retry code on cheyenne (in progress)\r\nTest baseline: \r\nTest namelist changes: \r\nTest status: bit for bit\r\nFixes \r\n\r\nUser interface changes?: \r\n\r\nUpdate gh-pages html (Y/N)?:\r\n\r\nCode review: \r\n","created_at":"2018-05-01T21:20:44Z","updated_at":"2018-05-02T16:37:22Z","closed_at":null,"merged_at":null,"merge_commit_sha":"457c03df2568e57e6e4f248c62819ab9c852f2a4","assignee":{"login":"jgfouca","id":7292036,"avatar_url":"https://avatars1.githubusercontent.com/u/7292036?v=4","gravatar_id":"","url":"https://api.github.com/users/jgfouca","html_url":"https://github.com/jgfouca","followers_url":"https://api.github.com/users/jgfouca/followers","following_url":"https://api.github.com/users/jgfouca/following{/other_user}","gists_url":"https://api.github.com/users/jgfouca/gists{/gist_id}","starred_url":"https://api.github.com/users/jgfouca/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jgfouca/subscriptions","organizations_url":"https://api.github.com/users/jgfouca/orgs","repos_url":"https://api.github.com/users/jgfouca/repos","events_url":"https://api.github.com/users/jgfouca/events{/privacy}","received_events_url":"https://api.github.com/users/jgfouca/received_events","type":"User","site_admin":false},"assignees":[{"login":"jgfouca","id":7292036,"avatar_url":"https://avatars1.githubusercontent.com/u/7292036?v=4","gravatar_id":"","url":"https://api.github.com/users/jgfouca","html_url":"https://github.com/jgfouca","followers_url":"https://api.github.com/users/jgfouca/followers","following_url":"https://api.github.com/users/jgfouca/following{/other_user}","gists_url":"https://api.github.com/users/jgfouca/gists{/gist_id}","starred_url":"https://api.github.com/users/jgfouca/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jgfouca/subscriptions","organizations_url":"https://api.github.com/users/jgfouca/orgs","repos_url":"https://api.github.com/users/jgfouca/repos","events_url":"https://api.github.com/users/jgfouca/events{/privacy}","received_events_url":"https://api.github.com/users/jgfouca/received_events","type":"User","site_admin":false}],"requested_reviewers":[],"requested_teams":[],"labels":[{"id":341453307,"url":"https://api.github.com/repos/ESMCI/cime/labels/in%20progress","name":"in progress","color":"ededed","default":false}],"milestone":null,"commits_url":"https://api.github.com/repos/ESMCI/cime/pulls/2526/commits","review_comments_url":"https://api.github.com/repos/ESMCI/cime/pulls/2526/comments","review_comment_url":"https://api.github.com/repos/ESMCI/cime/pulls/comments{/number}","comments_url":"https://api.github.com/repos/ESMCI/cime/issues/2526/comments","statuses_url":"https://api.github.com/repos/ESMCI/cime/statuses/b293d59849f0211f37429576103361c54caa7cc6","head":{"label":"jedwards4b:rework_and_clarify_mpirun_retry","ref":"rework_and_clarify_mpirun_retry","sha":"b293d59849f0211f37429576103361c54caa7cc6","user":{"login":"jedwards4b","id":6126104,"avatar_url":"https://avatars3.githubusercontent.com/u/6126104?v=4","gravatar_id":"","url":"https://api.github.com/users/jedwards4b","html_url":"https://github.com/jedwards4b","followers_url":"https://api.github.com/users/jedwards4b/followers","following_url":"https://api.github.com/users/jedwards4b/following{/other_user}","gists_url":"https://api.github.com/users/jedwards4b/gists{/gist_id}","starred_url":"https://api.github.com/users/jedwards4b/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jedwards4b/subscriptions","organizations_url":"https://api.github.com/users/jedwards4b/orgs","repos_url":"https://api.github.com/users/jedwards4b/repos","events_url":"https://api.github.com/users/jedwards4b/events{/privacy}","received_events_url":"https://api.github.com/users/jedwards4b/received_events","type":"User","site_admin":false},"repo":{"id":42525196,"name":"cime","full_name":"jedwards4b/cime","owner":{"login":"jedwards4b","id":6126104,"avatar_url":"https://avatars3.githubusercontent.com/u/6126104?v=4","gravatar_id":"","url":"https://api.github.com/users/jedwards4b","html_url":"https://github.com/jedwards4b","followers_url":"https://api.github.com/users/jedwards4b/followers","following_url":"https://api.github.com/users/jedwards4b/following{/other_user}","gists_url":"https://api.github.com/users/jedwards4b/gists{/gist_id}","starred_url":"https://api.github.com/users/jedwards4b/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jedwards4b/subscriptions","organizations_url":"https://api.github.com/users/jedwards4b/orgs","repos_url":"https://api.github.com/users/jedwards4b/repos","events_url":"https://api.github.com/users/jedwards4b/events{/privacy}","received_events_url":"https://api.github.com/users/jedwards4b/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/jedwards4b/cime","description":"Common Infrastructure for Modeling the Environment","fork":true,"url":"https://api.github.com/repos/jedwards4b/cime","forks_url":"https://api.github.com/repos/jedwards4b/cime/forks","keys_url":"https://api.github.com/repos/jedwards4b/cime/keys{/key_id}","collaborators_url":"https://api.github.com/repos/jedwards4b/cime/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/jedwards4b/cime/teams","hooks_url":"https://api.github.com/repos/jedwards4b/cime/hooks","issue_events_url":"https://api.github.com/repos/jedwards4b/cime/issues/events{/number}","events_url":"https://api.github.com/repos/jedwards4b/cime/events","assignees_url":"https://api.github.com/repos/jedwards4b/cime/assignees{/user}","branches_url":"https://api.github.com/repos/jedwards4b/cime/branches{/branch}","tags_url":"https://api.github.com/repos/jedwards4b/cime/tags","blobs_url":"https://api.github.com/repos/jedwards4b/cime/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/jedwards4b/cime/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/jedwards4b/cime/git/refs{/sha}","trees_url":"https://api.github.com/repos/jedwards4b/cime/git/trees{/sha}","statuses_url":"https://api.github.com/repos/jedwards4b/cime/statuses/{sha}","languages_url":"https://api.github.com/repos/jedwards4b/cime/languages","stargazers_url":"https://api.github.com/repos/jedwards4b/cime/stargazers","contributors_url":"https://api.github.com/repos/jedwards4b/cime/contributors","subscribers_url":"https://api.github.com/repos/jedwards4b/cime/subscribers","subscription_url":"https://api.github.com/repos/jedwards4b/cime/subscription","commits_url":"https://api.github.com/repos/jedwards4b/cime/commits{/sha}","git_commits_url":"https://api.github.com/repos/jedwards4b/cime/git/commits{/sha}","comments_url":"https://api.github.com/repos/jedwards4b/cime/comments{/number}","issue_comment_url":"https://api.github.com/repos/jedwards4b/cime/issues/comments{/number}","contents_url":"https://api.github.com/repos/jedwards4b/cime/contents/{+path}","compare_url":"https://api.github.com/repos/jedwards4b/cime/compare/{base}...{head}","merges_url":"https://api.github.com/repos/jedwards4b/cime/merges","archive_url":"https://api.github.com/repos/jedwards4b/cime/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/jedwards4b/cime/downloads","issues_url":"https://api.github.com/repos/jedwards4b/cime/issues{/number}","pulls_url":"https://api.github.com/repos/jedwards4b/cime/pulls{/number}","milestones_url":"https://api.github.com/repos/jedwards4b/cime/milestones{/number}","notifications_url":"https://api.github.com/repos/jedwards4b/cime/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/jedwards4b/cime/labels{/name}","releases_url":"https://api.github.com/repos/jedwards4b/cime/releases{/id}","deployments_url":"https://api.github.com/repos/jedwards4b/cime/deployments","created_at":"2015-09-15T14:38:34Z","updated_at":"2016-01-21T01:38:18Z","pushed_at":"2018-05-02T14:09:37Z","git_url":"git://github.com/jedwards4b/cime.git","ssh_url":"git@github.com:jedwards4b/cime.git","clone_url":"https://github.com/jedwards4b/cime.git","svn_url":"https://github.com/jedwards4b/cime","homepage":null,"size":73807,"stargazers_count":0,"watchers_count":0,"language":"FORTRAN","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":0,"license":{"key":"other","name":"Other","spdx_id":null,"url":null},"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"ESMCI:master","ref":"master","sha":"974839b9585467d310cd9212cd52553e536f1731","user":{"login":"ESMCI","id":10676238,"avatar_url":"https://avatars0.githubusercontent.com/u/10676238?v=4","gravatar_id":"","url":"https://api.github.com/users/ESMCI","html_url":"https://github.com/ESMCI","followers_url":"https://api.github.com/users/ESMCI/followers","following_url":"https://api.github.com/users/ESMCI/following{/other_user}","gists_url":"https://api.github.com/users/ESMCI/gists{/gist_id}","starred_url":"https://api.github.com/users/ESMCI/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ESMCI/subscriptions","organizations_url":"https://api.github.com/users/ESMCI/orgs","repos_url":"https://api.github.com/users/ESMCI/repos","events_url":"https://api.github.com/users/ESMCI/events{/privacy}","received_events_url":"https://api.github.com/users/ESMCI/received_events","type":"Organization","site_admin":false},"repo":{"id":31605662,"name":"cime","full_name":"ESMCI/cime","owner":{"login":"ESMCI","id":10676238,"avatar_url":"https://avatars0.githubusercontent.com/u/10676238?v=4","gravatar_id":"","url":"https://api.github.com/users/ESMCI","html_url":"https://github.com/ESMCI","followers_url":"https://api.github.com/users/ESMCI/followers","following_url":"https://api.github.com/users/ESMCI/following{/other_user}","gists_url":"https://api.github.com/users/ESMCI/gists{/gist_id}","starred_url":"https://api.github.com/users/ESMCI/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ESMCI/subscriptions","organizations_url":"https://api.github.com/users/ESMCI/orgs","repos_url":"https://api.github.com/users/ESMCI/repos","events_url":"https://api.github.com/users/ESMCI/events{/privacy}","received_events_url":"https://api.github.com/users/ESMCI/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/ESMCI/cime","description":"Common Infrastructure for Modeling the Earth","fork":false,"url":"https://api.github.com/repos/ESMCI/cime","forks_url":"https://api.github.com/repos/ESMCI/cime/forks","keys_url":"https://api.github.com/repos/ESMCI/cime/keys{/key_id}","collaborators_url":"https://api.github.com/repos/ESMCI/cime/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/ESMCI/cime/teams","hooks_url":"https://api.github.com/repos/ESMCI/cime/hooks","issue_events_url":"https://api.github.com/repos/ESMCI/cime/issues/events{/number}","events_url":"https://api.github.com/repos/ESMCI/cime/events","assignees_url":"https://api.github.com/repos/ESMCI/cime/assignees{/user}","branches_url":"https://api.github.com/repos/ESMCI/cime/branches{/branch}","tags_url":"https://api.github.com/repos/ESMCI/cime/tags","blobs_url":"https://api.github.com/repos/ESMCI/cime/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/ESMCI/cime/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/ESMCI/cime/git/refs{/sha}","trees_url":"https://api.github.com/repos/ESMCI/cime/git/trees{/sha}","statuses_url":"https://api.github.com/repos/ESMCI/cime/statuses/{sha}","languages_url":"https://api.github.com/repos/ESMCI/cime/languages","stargazers_url":"https://api.github.com/repos/ESMCI/cime/stargazers","contributors_url":"https://api.github.com/repos/ESMCI/cime/contributors","subscribers_url":"https://api.github.com/repos/ESMCI/cime/subscribers","subscription_url":"https://api.github.com/repos/ESMCI/cime/subscription","commits_url":"https://api.github.com/repos/ESMCI/cime/commits{/sha}","git_commits_url":"https://api.github.com/repos/ESMCI/cime/git/commits{/sha}","comments_url":"https://api.github.com/repos/ESMCI/cime/comments{/number}","issue_comment_url":"https://api.github.com/repos/ESMCI/cime/issues/comments{/number}","contents_url":"https://api.github.com/repos/ESMCI/cime/contents/{+path}","compare_url":"https://api.github.com/repos/ESMCI/cime/compare/{base}...{head}","merges_url":"https://api.github.com/repos/ESMCI/cime/merges","archive_url":"https://api.github.com/repos/ESMCI/cime/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/ESMCI/cime/downloads","issues_url":"https://api.github.com/repos/ESMCI/cime/issues{/number}","pulls_url":"https://api.github.com/repos/ESMCI/cime/pulls{/number}","milestones_url":"https://api.github.com/repos/ESMCI/cime/milestones{/number}","notifications_url":"https://api.github.com/repos/ESMCI/cime/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/ESMCI/cime/labels{/name}","releases_url":"https://api.github.com/repos/ESMCI/cime/releases{/id}","deployments_url":"https://api.github.com/repos/ESMCI/cime/deployments","created_at":"2015-03-03T15:33:00Z","updated_at":"2018-05-02T16:34:33Z","pushed_at":"2018-05-02T16:35:11Z","git_url":"git://github.com/ESMCI/cime.git","ssh_url":"git@github.com:ESMCI/cime.git","clone_url":"https://github.com/ESMCI/cime.git","svn_url":"https://github.com/ESMCI/cime","homepage":"","size":76524,"stargazers_count":18,"watchers_count":18,"language":"Fortran","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":true,"forks_count":50,"mirror_url":null,"archived":false,"open_issues_count":158,"license":{"key":"other","name":"Other","spdx_id":null,"url":null},"forks":50,"open_issues":158,"watchers":18,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/ESMCI/cime/pulls/2526"},"html":{"href":"https://github.com/ESMCI/cime/pull/2526"},"issue":{"href":"https://api.github.com/repos/ESMCI/cime/issues/2526"},"comments":{"href":"https://api.github.com/repos/ESMCI/cime/issues/2526/comments"},"review_comments":{"href":"https://api.github.com/repos/ESMCI/cime/pulls/2526/comments"},"review_comment":{"href":"https://api.github.com/repos/ESMCI/cime/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/ESMCI/cime/pulls/2526/commits"},"statuses":{"href":"https://api.github.com/repos/ESMCI/cime/statuses/b293d59849f0211f37429576103361c54caa7cc6"}},"author_association":"CONTRIBUTOR"}}
{"actor":{"display_login":"jedwards4b"}}