{"action":"created","comment":{"url":"https://api.github.com/repos/openvstorage/framework/pulls/comments/62308140","id":62308140,"diff_hunk":"@@ -277,13 +282,50 @@ def gather_scrub_work():\n                                                                               vdisk_guids=local_vdisks_to_scrub)\n             except Exception as ex:\n                 logger.error('Gather Scrub - Storage Router {0:<15} - Scrubbing failed with error:\\n - {1}'.format(local_storage_router.ip, ex))\n-        all_results = result_set.join(propagate=False)  # Propagate False makes sure all jobs are waited for even when 1 or more jobs fail\n+\n+        all_results, failed_nodes = manage_running_tasks(result_set, timesleep=60)  # Check every 60 seconds if tasks are still running\n+\n+        for index, result in enumerate(all_results):\n+            if isinstance(result, list):\n+                processed_guids.extend(result)\n+            else:\n+                logger.error('Gather Scrub - Storage Router {0:<15} - Scrubbing failed with error:\\n - {1}'.format(storage_router_list[index].ip, result))\n+\n+        result_set = {}\n+        for failed_node in failed_nodes:\n+            logger.warning('Scrubbing failed on node {0}. Will reschedule on another node.'.format(failed_node))\n+            vdisk_guids_to_scrub = scrub_map[failed_node]\n+            rescheduled_work = False\n+            for storage_router, scrub_location in scrub_locations.items():\n+                if storage_router.ip != failed_node:","path":"ovs/lib/scheduledtask.py","position":77,"original_position":77,"commit_id":"575dbfbecbaf438bd7a8f037b4315a8598f2615b","original_commit_id":"575dbfbecbaf438bd7a8f037b4315a8598f2615b","user":{"login":"kvanhijf","id":13535590,"avatar_url":"https://avatars.githubusercontent.com/u/13535590?v=3","gravatar_id":"","url":"https://api.github.com/users/kvanhijf","html_url":"https://github.com/kvanhijf","followers_url":"https://api.github.com/users/kvanhijf/followers","following_url":"https://api.github.com/users/kvanhijf/following{/other_user}","gists_url":"https://api.github.com/users/kvanhijf/gists{/gist_id}","starred_url":"https://api.github.com/users/kvanhijf/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/kvanhijf/subscriptions","organizations_url":"https://api.github.com/users/kvanhijf/orgs","repos_url":"https://api.github.com/users/kvanhijf/repos","events_url":"https://api.github.com/users/kvanhijf/events{/privacy}","received_events_url":"https://api.github.com/users/kvanhijf/received_events","type":"User","site_admin":false},"body":"If you have multiple failed nodes, this code can potentially start scrubbing on a different failed node, than the 1 being checked currently","created_at":"2016-05-06T09:38:31Z","updated_at":"2016-05-06T09:38:31Z","html_url":"https://github.com/openvstorage/framework/pull/525#discussion_r62308140","pull_request_url":"https://api.github.com/repos/openvstorage/framework/pulls/525","_links":{"self":{"href":"https://api.github.com/repos/openvstorage/framework/pulls/comments/62308140"},"html":{"href":"https://github.com/openvstorage/framework/pull/525#discussion_r62308140"},"pull_request":{"href":"https://api.github.com/repos/openvstorage/framework/pulls/525"}}},"pull_request":{"url":"https://api.github.com/repos/openvstorage/framework/pulls/525","id":68700121,"html_url":"https://github.com/openvstorage/framework/pull/525","diff_url":"https://github.com/openvstorage/framework/pull/525.diff","patch_url":"https://github.com/openvstorage/framework/pull/525.patch","issue_url":"https://api.github.com/repos/openvstorage/framework/issues/525","number":525,"state":"open","locked":false,"title":"OVS-4342 - Gather scrubwork is not properly exited after run","user":{"login":"eduardBM","id":4077789,"avatar_url":"https://avatars.githubusercontent.com/u/4077789?v=3","gravatar_id":"","url":"https://api.github.com/users/eduardBM","html_url":"https://github.com/eduardBM","followers_url":"https://api.github.com/users/eduardBM/followers","following_url":"https://api.github.com/users/eduardBM/following{/other_user}","gists_url":"https://api.github.com/users/eduardBM/gists{/gist_id}","starred_url":"https://api.github.com/users/eduardBM/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/eduardBM/subscriptions","organizations_url":"https://api.github.com/users/eduardBM/orgs","repos_url":"https://api.github.com/users/eduardBM/repos","events_url":"https://api.github.com/users/eduardBM/events{/privacy}","received_events_url":"https://api.github.com/users/eduardBM/received_events","type":"User","site_admin":false},"body":"- make sure remote tasks are able to run (workers are running)\r\n- timeout in case workers go down while executing task (1 hr)","created_at":"2016-05-03T12:49:14Z","updated_at":"2016-05-06T09:38:31Z","closed_at":null,"merged_at":null,"merge_commit_sha":"ca9d75d984b4c1b23acf24fcce844090a92cf980","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/openvstorage/framework/pulls/525/commits","review_comments_url":"https://api.github.com/repos/openvstorage/framework/pulls/525/comments","review_comment_url":"https://api.github.com/repos/openvstorage/framework/pulls/comments{/number}","comments_url":"https://api.github.com/repos/openvstorage/framework/issues/525/comments","statuses_url":"https://api.github.com/repos/openvstorage/framework/statuses/575dbfbecbaf438bd7a8f037b4315a8598f2615b","head":{"label":"openvstorage:OVS-4342_scrubwork_timeout","ref":"OVS-4342_scrubwork_timeout","sha":"575dbfbecbaf438bd7a8f037b4315a8598f2615b","user":{"login":"openvstorage","id":12763393,"avatar_url":"https://avatars.githubusercontent.com/u/12763393?v=3","gravatar_id":"","url":"https://api.github.com/users/openvstorage","html_url":"https://github.com/openvstorage","followers_url":"https://api.github.com/users/openvstorage/followers","following_url":"https://api.github.com/users/openvstorage/following{/other_user}","gists_url":"https://api.github.com/users/openvstorage/gists{/gist_id}","starred_url":"https://api.github.com/users/openvstorage/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/openvstorage/subscriptions","organizations_url":"https://api.github.com/users/openvstorage/orgs","repos_url":"https://api.github.com/users/openvstorage/repos","events_url":"https://api.github.com/users/openvstorage/events{/privacy}","received_events_url":"https://api.github.com/users/openvstorage/received_events","type":"Organization","site_admin":false},"repo":{"id":39504796,"name":"framework","full_name":"openvstorage/framework","owner":{"login":"openvstorage","id":12763393,"avatar_url":"https://avatars.githubusercontent.com/u/12763393?v=3","gravatar_id":"","url":"https://api.github.com/users/openvstorage","html_url":"https://github.com/openvstorage","followers_url":"https://api.github.com/users/openvstorage/followers","following_url":"https://api.github.com/users/openvstorage/following{/other_user}","gists_url":"https://api.github.com/users/openvstorage/gists{/gist_id}","starred_url":"https://api.github.com/users/openvstorage/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/openvstorage/subscriptions","organizations_url":"https://api.github.com/users/openvstorage/orgs","repos_url":"https://api.github.com/users/openvstorage/repos","events_url":"https://api.github.com/users/openvstorage/events{/privacy}","received_events_url":"https://api.github.com/users/openvstorage/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/openvstorage/framework","description":"The Framework is a set of components and tools which brings the user an interface (GUI / API) to setup, extend and manage an Open vStorage platform.","fork":false,"url":"https://api.github.com/repos/openvstorage/framework","forks_url":"https://api.github.com/repos/openvstorage/framework/forks","keys_url":"https://api.github.com/repos/openvstorage/framework/keys{/key_id}","collaborators_url":"https://api.github.com/repos/openvstorage/framework/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/openvstorage/framework/teams","hooks_url":"https://api.github.com/repos/openvstorage/framework/hooks","issue_events_url":"https://api.github.com/repos/openvstorage/framework/issues/events{/number}","events_url":"https://api.github.com/repos/openvstorage/framework/events","assignees_url":"https://api.github.com/repos/openvstorage/framework/assignees{/user}","branches_url":"https://api.github.com/repos/openvstorage/framework/branches{/branch}","tags_url":"https://api.github.com/repos/openvstorage/framework/tags","blobs_url":"https://api.github.com/repos/openvstorage/framework/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/openvstorage/framework/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/openvstorage/framework/git/refs{/sha}","trees_url":"https://api.github.com/repos/openvstorage/framework/git/trees{/sha}","statuses_url":"https://api.github.com/repos/openvstorage/framework/statuses/{sha}","languages_url":"https://api.github.com/repos/openvstorage/framework/languages","stargazers_url":"https://api.github.com/repos/openvstorage/framework/stargazers","contributors_url":"https://api.github.com/repos/openvstorage/framework/contributors","subscribers_url":"https://api.github.com/repos/openvstorage/framework/subscribers","subscription_url":"https://api.github.com/repos/openvstorage/framework/subscription","commits_url":"https://api.github.com/repos/openvstorage/framework/commits{/sha}","git_commits_url":"https://api.github.com/repos/openvstorage/framework/git/commits{/sha}","comments_url":"https://api.github.com/repos/openvstorage/framework/comments{/number}","issue_comment_url":"https://api.github.com/repos/openvstorage/framework/issues/comments{/number}","contents_url":"https://api.github.com/repos/openvstorage/framework/contents/{+path}","compare_url":"https://api.github.com/repos/openvstorage/framework/compare/{base}...{head}","merges_url":"https://api.github.com/repos/openvstorage/framework/merges","archive_url":"https://api.github.com/repos/openvstorage/framework/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/openvstorage/framework/downloads","issues_url":"https://api.github.com/repos/openvstorage/framework/issues{/number}","pulls_url":"https://api.github.com/repos/openvstorage/framework/pulls{/number}","milestones_url":"https://api.github.com/repos/openvstorage/framework/milestones{/number}","notifications_url":"https://api.github.com/repos/openvstorage/framework/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/openvstorage/framework/labels{/name}","releases_url":"https://api.github.com/repos/openvstorage/framework/releases{/id}","deployments_url":"https://api.github.com/repos/openvstorage/framework/deployments","created_at":"2015-07-22T12:29:15Z","updated_at":"2016-04-14T12:14:10Z","pushed_at":"2016-05-06T07:50:25Z","git_url":"git://github.com/openvstorage/framework.git","ssh_url":"git@github.com:openvstorage/framework.git","clone_url":"https://github.com/openvstorage/framework.git","svn_url":"https://github.com/openvstorage/framework","homepage":"","size":10330,"stargazers_count":15,"watchers_count":15,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":15,"mirror_url":null,"open_issues_count":22,"forks":15,"open_issues":22,"watchers":15,"default_branch":"master"}},"base":{"label":"openvstorage:master","ref":"master","sha":"42447162076e8a8dde28cea8515128ccb75120be","user":{"login":"openvstorage","id":12763393,"avatar_url":"https://avatars.githubusercontent.com/u/12763393?v=3","gravatar_id":"","url":"https://api.github.com/users/openvstorage","html_url":"https://github.com/openvstorage","followers_url":"https://api.github.com/users/openvstorage/followers","following_url":"https://api.github.com/users/openvstorage/following{/other_user}","gists_url":"https://api.github.com/users/openvstorage/gists{/gist_id}","starred_url":"https://api.github.com/users/openvstorage/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/openvstorage/subscriptions","organizations_url":"https://api.github.com/users/openvstorage/orgs","repos_url":"https://api.github.com/users/openvstorage/repos","events_url":"https://api.github.com/users/openvstorage/events{/privacy}","received_events_url":"https://api.github.com/users/openvstorage/received_events","type":"Organization","site_admin":false},"repo":{"id":39504796,"name":"framework","full_name":"openvstorage/framework","owner":{"login":"openvstorage","id":12763393,"avatar_url":"https://avatars.githubusercontent.com/u/12763393?v=3","gravatar_id":"","url":"https://api.github.com/users/openvstorage","html_url":"https://github.com/openvstorage","followers_url":"https://api.github.com/users/openvstorage/followers","following_url":"https://api.github.com/users/openvstorage/following{/other_user}","gists_url":"https://api.github.com/users/openvstorage/gists{/gist_id}","starred_url":"https://api.github.com/users/openvstorage/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/openvstorage/subscriptions","organizations_url":"https://api.github.com/users/openvstorage/orgs","repos_url":"https://api.github.com/users/openvstorage/repos","events_url":"https://api.github.com/users/openvstorage/events{/privacy}","received_events_url":"https://api.github.com/users/openvstorage/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/openvstorage/framework","description":"The Framework is a set of components and tools which brings the user an interface (GUI / API) to setup, extend and manage an Open vStorage platform.","fork":false,"url":"https://api.github.com/repos/openvstorage/framework","forks_url":"https://api.github.com/repos/openvstorage/framework/forks","keys_url":"https://api.github.com/repos/openvstorage/framework/keys{/key_id}","collaborators_url":"https://api.github.com/repos/openvstorage/framework/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/openvstorage/framework/teams","hooks_url":"https://api.github.com/repos/openvstorage/framework/hooks","issue_events_url":"https://api.github.com/repos/openvstorage/framework/issues/events{/number}","events_url":"https://api.github.com/repos/openvstorage/framework/events","assignees_url":"https://api.github.com/repos/openvstorage/framework/assignees{/user}","branches_url":"https://api.github.com/repos/openvstorage/framework/branches{/branch}","tags_url":"https://api.github.com/repos/openvstorage/framework/tags","blobs_url":"https://api.github.com/repos/openvstorage/framework/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/openvstorage/framework/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/openvstorage/framework/git/refs{/sha}","trees_url":"https://api.github.com/repos/openvstorage/framework/git/trees{/sha}","statuses_url":"https://api.github.com/repos/openvstorage/framework/statuses/{sha}","languages_url":"https://api.github.com/repos/openvstorage/framework/languages","stargazers_url":"https://api.github.com/repos/openvstorage/framework/stargazers","contributors_url":"https://api.github.com/repos/openvstorage/framework/contributors","subscribers_url":"https://api.github.com/repos/openvstorage/framework/subscribers","subscription_url":"https://api.github.com/repos/openvstorage/framework/subscription","commits_url":"https://api.github.com/repos/openvstorage/framework/commits{/sha}","git_commits_url":"https://api.github.com/repos/openvstorage/framework/git/commits{/sha}","comments_url":"https://api.github.com/repos/openvstorage/framework/comments{/number}","issue_comment_url":"https://api.github.com/repos/openvstorage/framework/issues/comments{/number}","contents_url":"https://api.github.com/repos/openvstorage/framework/contents/{+path}","compare_url":"https://api.github.com/repos/openvstorage/framework/compare/{base}...{head}","merges_url":"https://api.github.com/repos/openvstorage/framework/merges","archive_url":"https://api.github.com/repos/openvstorage/framework/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/openvstorage/framework/downloads","issues_url":"https://api.github.com/repos/openvstorage/framework/issues{/number}","pulls_url":"https://api.github.com/repos/openvstorage/framework/pulls{/number}","milestones_url":"https://api.github.com/repos/openvstorage/framework/milestones{/number}","notifications_url":"https://api.github.com/repos/openvstorage/framework/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/openvstorage/framework/labels{/name}","releases_url":"https://api.github.com/repos/openvstorage/framework/releases{/id}","deployments_url":"https://api.github.com/repos/openvstorage/framework/deployments","created_at":"2015-07-22T12:29:15Z","updated_at":"2016-04-14T12:14:10Z","pushed_at":"2016-05-06T07:50:25Z","git_url":"git://github.com/openvstorage/framework.git","ssh_url":"git@github.com:openvstorage/framework.git","clone_url":"https://github.com/openvstorage/framework.git","svn_url":"https://github.com/openvstorage/framework","homepage":"","size":10330,"stargazers_count":15,"watchers_count":15,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":15,"mirror_url":null,"open_issues_count":22,"forks":15,"open_issues":22,"watchers":15,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/openvstorage/framework/pulls/525"},"html":{"href":"https://github.com/openvstorage/framework/pull/525"},"issue":{"href":"https://api.github.com/repos/openvstorage/framework/issues/525"},"comments":{"href":"https://api.github.com/repos/openvstorage/framework/issues/525/comments"},"review_comments":{"href":"https://api.github.com/repos/openvstorage/framework/pulls/525/comments"},"review_comment":{"href":"https://api.github.com/repos/openvstorage/framework/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/openvstorage/framework/pulls/525/commits"},"statuses":{"href":"https://api.github.com/repos/openvstorage/framework/statuses/575dbfbecbaf438bd7a8f037b4315a8598f2615b"}}}}