{"action":"created","comment":{"url":"https://api.github.com/repos/SUSE/DeepSea/pulls/comments/177744689","pull_request_review_id":107670805,"id":177744689,"diff_hunk":"@@ -382,28 +392,104 @@ def wait(self):\n         i = 0\n         last_pgs = 0\n         while i < self.settings['timeout']/self.settings['delay']:\n+            rc, msg = self.osd_safe_to_destroy()\n+            if rc == 0:\n+                log.info(\"osd.{} is safe to destroy\".format(self.osd_id))\n+                return \"\"\n             entry = self.osd_df()\n             if 'pgs' in entry:\n                 if entry['pgs'] == 0:\n-                    log.info(\"osd.{} has no PGs\".format(self.osd_id))\n-                    return \"\"\n+                    log.warning(\"osd.{} has {} PGs remaining but {}\".\n+                                format(self.osd_id, entry['pgs'], msg))\n                 else:\n                     log.warning(\"osd.{} has {} PGs remaining\".format(self.osd_id, entry['pgs']))\n                     if last_pgs != entry['pgs']:\n                         # Making progress, reset countdown\n                         i = 0\n                         last_pgs = entry['pgs']\n             else:\n-                msg = \"osd.{} does not exist\".format(self.osd_id)\n+                msg = \"osd.{} does not exist {}\".format(self.osd_id, msg)\n                 log.warning(msg)\n-                return msg\n             i += 1\n             time.sleep(self.settings['delay'])\n \n         log.debug(\"Timeout expired\")\n         raise RuntimeError(\"Timeout expired\")\n \n \n+class CephPGs(object):\n+    \"\"\"\n+    Query PG states and pause until all are active+clean\n+    \"\"\"\n+\n+    def __init__(self, **kwargs):\n+        \"\"\"\n+        Initialize settings, connect to Ceph cluster\n+        \"\"\"\n+        self.settings = {\n+            'conf': \"/etc/ceph/ceph.conf\",\n+            'timeout': 120,\n+            'keyring': '/etc/ceph/ceph.client.admin.keyring',\n+            'client': 'client.admin',\n+            'delay': 12\n+        }\n+        self.settings.update(kwargs)\n+        log.debug(\"settings: {}\".format(pprint.pformat(self.settings)))\n+        self.cluster = rados.Rados(conffile=self.settings['conf'],\n+                                   conf=dict(keyring=self.settings['keyring']),\n+                                   name=self.settings['client'])\n+        try:\n+            self.cluster.connect()\n+        except Exception as error:\n+            raise RuntimeError(\"connection error: {}\".format(error))\n+\n+    def quiescent(self):\n+        \"\"\"\n+        Wait until PGs are active+clean or timeout is reached.  Default is a\n+        2 minute sliding window.\n+        \"\"\"\n+        i = 0\n+        last = []\n+        while i < self.settings['timeout']/self.settings['delay']:","path":"srv/salt/_modules/osd.py","position":90,"original_position":90,"commit_id":"c608b5698f3950ddf99ddbe0a97e849e967cc3fb","original_commit_id":"c608b5698f3950ddf99ddbe0a97e849e967cc3fb","user":{"login":"jan--f","id":458501,"avatar_url":"https://avatars0.githubusercontent.com/u/458501?v=4","gravatar_id":"","url":"https://api.github.com/users/jan--f","html_url":"https://github.com/jan--f","followers_url":"https://api.github.com/users/jan--f/followers","following_url":"https://api.github.com/users/jan--f/following{/other_user}","gists_url":"https://api.github.com/users/jan--f/gists{/gist_id}","starred_url":"https://api.github.com/users/jan--f/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jan--f/subscriptions","organizations_url":"https://api.github.com/users/jan--f/orgs","repos_url":"https://api.github.com/users/jan--f/repos","events_url":"https://api.github.com/users/jan--f/events{/privacy}","received_events_url":"https://api.github.com/users/jan--f/received_events","type":"User","site_admin":false},"body":"Only because we use it in a division here and its a config, maybe check that delay is not 0? Pretty sure some of our usual suspects will try to set delay to 0 at some point...to make things go faster.","created_at":"2018-03-28T13:15:39Z","updated_at":"2018-03-28T13:25:44Z","html_url":"https://github.com/SUSE/DeepSea/pull/1026#discussion_r177744689","pull_request_url":"https://api.github.com/repos/SUSE/DeepSea/pulls/1026","author_association":"CONTRIBUTOR","_links":{"self":{"href":"https://api.github.com/repos/SUSE/DeepSea/pulls/comments/177744689"},"html":{"href":"https://github.com/SUSE/DeepSea/pull/1026#discussion_r177744689"},"pull_request":{"href":"https://api.github.com/repos/SUSE/DeepSea/pulls/1026"}}},"pull_request":{"url":"https://api.github.com/repos/SUSE/DeepSea/pulls/1026","id":176576938,"html_url":"https://github.com/SUSE/DeepSea/pull/1026","diff_url":"https://github.com/SUSE/DeepSea/pull/1026.diff","patch_url":"https://github.com/SUSE/DeepSea/pull/1026.patch","issue_url":"https://api.github.com/repos/SUSE/DeepSea/issues/1026","number":1026,"state":"open","locked":false,"title":"Add active+clean wait for migrations","user":{"login":"swiftgist","id":4697887,"avatar_url":"https://avatars2.githubusercontent.com/u/4697887?v=4","gravatar_id":"","url":"https://api.github.com/users/swiftgist","html_url":"https://github.com/swiftgist","followers_url":"https://api.github.com/users/swiftgist/followers","following_url":"https://api.github.com/users/swiftgist/following{/other_user}","gists_url":"https://api.github.com/users/swiftgist/gists{/gist_id}","starred_url":"https://api.github.com/users/swiftgist/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/swiftgist/subscriptions","organizations_url":"https://api.github.com/users/swiftgist/orgs","repos_url":"https://api.github.com/users/swiftgist/repos","events_url":"https://api.github.com/users/swiftgist/events{/privacy}","received_events_url":"https://api.github.com/users/swiftgist/received_events","type":"User","site_admin":false},"body":"Before removing an OSD, ensure that all PGs are active+clean.\r\nIncludes unit and functional tests.\r\n\r\nSigned-off-by: Eric Jackson <ejackson@suse.com>\r\n\r\n","created_at":"2018-03-21T18:52:37Z","updated_at":"2018-03-28T13:25:44Z","closed_at":null,"merged_at":null,"merge_commit_sha":"54abfcff0e70f3bf6ce84bc6a98cbc0b8a561f93","assignee":null,"assignees":[],"requested_reviewers":[{"login":"tserong","id":754700,"avatar_url":"https://avatars1.githubusercontent.com/u/754700?v=4","gravatar_id":"","url":"https://api.github.com/users/tserong","html_url":"https://github.com/tserong","followers_url":"https://api.github.com/users/tserong/followers","following_url":"https://api.github.com/users/tserong/following{/other_user}","gists_url":"https://api.github.com/users/tserong/gists{/gist_id}","starred_url":"https://api.github.com/users/tserong/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tserong/subscriptions","organizations_url":"https://api.github.com/users/tserong/orgs","repos_url":"https://api.github.com/users/tserong/repos","events_url":"https://api.github.com/users/tserong/events{/privacy}","received_events_url":"https://api.github.com/users/tserong/received_events","type":"User","site_admin":false}],"requested_teams":[],"labels":[{"id":730559433,"url":"https://api.github.com/repos/SUSE/DeepSea/labels/needs%20backport","name":"needs backport","color":"9fed93","default":false}],"milestone":null,"commits_url":"https://api.github.com/repos/SUSE/DeepSea/pulls/1026/commits","review_comments_url":"https://api.github.com/repos/SUSE/DeepSea/pulls/1026/comments","review_comment_url":"https://api.github.com/repos/SUSE/DeepSea/pulls/comments{/number}","comments_url":"https://api.github.com/repos/SUSE/DeepSea/issues/1026/comments","statuses_url":"https://api.github.com/repos/SUSE/DeepSea/statuses/c608b5698f3950ddf99ddbe0a97e849e967cc3fb","head":{"label":"SUSE:wip-pg-wait","ref":"wip-pg-wait","sha":"c608b5698f3950ddf99ddbe0a97e849e967cc3fb","user":{"login":"SUSE","id":1067733,"avatar_url":"https://avatars0.githubusercontent.com/u/1067733?v=4","gravatar_id":"","url":"https://api.github.com/users/SUSE","html_url":"https://github.com/SUSE","followers_url":"https://api.github.com/users/SUSE/followers","following_url":"https://api.github.com/users/SUSE/following{/other_user}","gists_url":"https://api.github.com/users/SUSE/gists{/gist_id}","starred_url":"https://api.github.com/users/SUSE/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/SUSE/subscriptions","organizations_url":"https://api.github.com/users/SUSE/orgs","repos_url":"https://api.github.com/users/SUSE/repos","events_url":"https://api.github.com/users/SUSE/events{/privacy}","received_events_url":"https://api.github.com/users/SUSE/received_events","type":"Organization","site_admin":false},"repo":{"id":66941942,"name":"DeepSea","full_name":"SUSE/DeepSea","owner":{"login":"SUSE","id":1067733,"avatar_url":"https://avatars0.githubusercontent.com/u/1067733?v=4","gravatar_id":"","url":"https://api.github.com/users/SUSE","html_url":"https://github.com/SUSE","followers_url":"https://api.github.com/users/SUSE/followers","following_url":"https://api.github.com/users/SUSE/following{/other_user}","gists_url":"https://api.github.com/users/SUSE/gists{/gist_id}","starred_url":"https://api.github.com/users/SUSE/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/SUSE/subscriptions","organizations_url":"https://api.github.com/users/SUSE/orgs","repos_url":"https://api.github.com/users/SUSE/repos","events_url":"https://api.github.com/users/SUSE/events{/privacy}","received_events_url":"https://api.github.com/users/SUSE/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/SUSE/DeepSea","description":null,"fork":false,"url":"https://api.github.com/repos/SUSE/DeepSea","forks_url":"https://api.github.com/repos/SUSE/DeepSea/forks","keys_url":"https://api.github.com/repos/SUSE/DeepSea/keys{/key_id}","collaborators_url":"https://api.github.com/repos/SUSE/DeepSea/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/SUSE/DeepSea/teams","hooks_url":"https://api.github.com/repos/SUSE/DeepSea/hooks","issue_events_url":"https://api.github.com/repos/SUSE/DeepSea/issues/events{/number}","events_url":"https://api.github.com/repos/SUSE/DeepSea/events","assignees_url":"https://api.github.com/repos/SUSE/DeepSea/assignees{/user}","branches_url":"https://api.github.com/repos/SUSE/DeepSea/branches{/branch}","tags_url":"https://api.github.com/repos/SUSE/DeepSea/tags","blobs_url":"https://api.github.com/repos/SUSE/DeepSea/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/SUSE/DeepSea/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/SUSE/DeepSea/git/refs{/sha}","trees_url":"https://api.github.com/repos/SUSE/DeepSea/git/trees{/sha}","statuses_url":"https://api.github.com/repos/SUSE/DeepSea/statuses/{sha}","languages_url":"https://api.github.com/repos/SUSE/DeepSea/languages","stargazers_url":"https://api.github.com/repos/SUSE/DeepSea/stargazers","contributors_url":"https://api.github.com/repos/SUSE/DeepSea/contributors","subscribers_url":"https://api.github.com/repos/SUSE/DeepSea/subscribers","subscription_url":"https://api.github.com/repos/SUSE/DeepSea/subscription","commits_url":"https://api.github.com/repos/SUSE/DeepSea/commits{/sha}","git_commits_url":"https://api.github.com/repos/SUSE/DeepSea/git/commits{/sha}","comments_url":"https://api.github.com/repos/SUSE/DeepSea/comments{/number}","issue_comment_url":"https://api.github.com/repos/SUSE/DeepSea/issues/comments{/number}","contents_url":"https://api.github.com/repos/SUSE/DeepSea/contents/{+path}","compare_url":"https://api.github.com/repos/SUSE/DeepSea/compare/{base}...{head}","merges_url":"https://api.github.com/repos/SUSE/DeepSea/merges","archive_url":"https://api.github.com/repos/SUSE/DeepSea/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/SUSE/DeepSea/downloads","issues_url":"https://api.github.com/repos/SUSE/DeepSea/issues{/number}","pulls_url":"https://api.github.com/repos/SUSE/DeepSea/pulls{/number}","milestones_url":"https://api.github.com/repos/SUSE/DeepSea/milestones{/number}","notifications_url":"https://api.github.com/repos/SUSE/DeepSea/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/SUSE/DeepSea/labels{/name}","releases_url":"https://api.github.com/repos/SUSE/DeepSea/releases{/id}","deployments_url":"https://api.github.com/repos/SUSE/DeepSea/deployments","created_at":"2016-08-30T13:15:19Z","updated_at":"2018-03-26T14:13:44Z","pushed_at":"2018-03-28T12:57:53Z","git_url":"git://github.com/SUSE/DeepSea.git","ssh_url":"git@github.com:SUSE/DeepSea.git","clone_url":"https://github.com/SUSE/DeepSea.git","svn_url":"https://github.com/SUSE/DeepSea","homepage":null,"size":3799,"stargazers_count":55,"watchers_count":55,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":38,"mirror_url":null,"archived":false,"open_issues_count":187,"license":{"key":"gpl-3.0","name":"GNU General Public License v3.0","spdx_id":"GPL-3.0","url":"https://api.github.com/licenses/gpl-3.0"},"forks":38,"open_issues":187,"watchers":55,"default_branch":"master"}},"base":{"label":"SUSE:master","ref":"master","sha":"94242a57908beb1caccf9592fa0da8ba5bae68a1","user":{"login":"SUSE","id":1067733,"avatar_url":"https://avatars0.githubusercontent.com/u/1067733?v=4","gravatar_id":"","url":"https://api.github.com/users/SUSE","html_url":"https://github.com/SUSE","followers_url":"https://api.github.com/users/SUSE/followers","following_url":"https://api.github.com/users/SUSE/following{/other_user}","gists_url":"https://api.github.com/users/SUSE/gists{/gist_id}","starred_url":"https://api.github.com/users/SUSE/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/SUSE/subscriptions","organizations_url":"https://api.github.com/users/SUSE/orgs","repos_url":"https://api.github.com/users/SUSE/repos","events_url":"https://api.github.com/users/SUSE/events{/privacy}","received_events_url":"https://api.github.com/users/SUSE/received_events","type":"Organization","site_admin":false},"repo":{"id":66941942,"name":"DeepSea","full_name":"SUSE/DeepSea","owner":{"login":"SUSE","id":1067733,"avatar_url":"https://avatars0.githubusercontent.com/u/1067733?v=4","gravatar_id":"","url":"https://api.github.com/users/SUSE","html_url":"https://github.com/SUSE","followers_url":"https://api.github.com/users/SUSE/followers","following_url":"https://api.github.com/users/SUSE/following{/other_user}","gists_url":"https://api.github.com/users/SUSE/gists{/gist_id}","starred_url":"https://api.github.com/users/SUSE/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/SUSE/subscriptions","organizations_url":"https://api.github.com/users/SUSE/orgs","repos_url":"https://api.github.com/users/SUSE/repos","events_url":"https://api.github.com/users/SUSE/events{/privacy}","received_events_url":"https://api.github.com/users/SUSE/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/SUSE/DeepSea","description":null,"fork":false,"url":"https://api.github.com/repos/SUSE/DeepSea","forks_url":"https://api.github.com/repos/SUSE/DeepSea/forks","keys_url":"https://api.github.com/repos/SUSE/DeepSea/keys{/key_id}","collaborators_url":"https://api.github.com/repos/SUSE/DeepSea/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/SUSE/DeepSea/teams","hooks_url":"https://api.github.com/repos/SUSE/DeepSea/hooks","issue_events_url":"https://api.github.com/repos/SUSE/DeepSea/issues/events{/number}","events_url":"https://api.github.com/repos/SUSE/DeepSea/events","assignees_url":"https://api.github.com/repos/SUSE/DeepSea/assignees{/user}","branches_url":"https://api.github.com/repos/SUSE/DeepSea/branches{/branch}","tags_url":"https://api.github.com/repos/SUSE/DeepSea/tags","blobs_url":"https://api.github.com/repos/SUSE/DeepSea/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/SUSE/DeepSea/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/SUSE/DeepSea/git/refs{/sha}","trees_url":"https://api.github.com/repos/SUSE/DeepSea/git/trees{/sha}","statuses_url":"https://api.github.com/repos/SUSE/DeepSea/statuses/{sha}","languages_url":"https://api.github.com/repos/SUSE/DeepSea/languages","stargazers_url":"https://api.github.com/repos/SUSE/DeepSea/stargazers","contributors_url":"https://api.github.com/repos/SUSE/DeepSea/contributors","subscribers_url":"https://api.github.com/repos/SUSE/DeepSea/subscribers","subscription_url":"https://api.github.com/repos/SUSE/DeepSea/subscription","commits_url":"https://api.github.com/repos/SUSE/DeepSea/commits{/sha}","git_commits_url":"https://api.github.com/repos/SUSE/DeepSea/git/commits{/sha}","comments_url":"https://api.github.com/repos/SUSE/DeepSea/comments{/number}","issue_comment_url":"https://api.github.com/repos/SUSE/DeepSea/issues/comments{/number}","contents_url":"https://api.github.com/repos/SUSE/DeepSea/contents/{+path}","compare_url":"https://api.github.com/repos/SUSE/DeepSea/compare/{base}...{head}","merges_url":"https://api.github.com/repos/SUSE/DeepSea/merges","archive_url":"https://api.github.com/repos/SUSE/DeepSea/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/SUSE/DeepSea/downloads","issues_url":"https://api.github.com/repos/SUSE/DeepSea/issues{/number}","pulls_url":"https://api.github.com/repos/SUSE/DeepSea/pulls{/number}","milestones_url":"https://api.github.com/repos/SUSE/DeepSea/milestones{/number}","notifications_url":"https://api.github.com/repos/SUSE/DeepSea/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/SUSE/DeepSea/labels{/name}","releases_url":"https://api.github.com/repos/SUSE/DeepSea/releases{/id}","deployments_url":"https://api.github.com/repos/SUSE/DeepSea/deployments","created_at":"2016-08-30T13:15:19Z","updated_at":"2018-03-26T14:13:44Z","pushed_at":"2018-03-28T12:57:53Z","git_url":"git://github.com/SUSE/DeepSea.git","ssh_url":"git@github.com:SUSE/DeepSea.git","clone_url":"https://github.com/SUSE/DeepSea.git","svn_url":"https://github.com/SUSE/DeepSea","homepage":null,"size":3799,"stargazers_count":55,"watchers_count":55,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":38,"mirror_url":null,"archived":false,"open_issues_count":187,"license":{"key":"gpl-3.0","name":"GNU General Public License v3.0","spdx_id":"GPL-3.0","url":"https://api.github.com/licenses/gpl-3.0"},"forks":38,"open_issues":187,"watchers":55,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/SUSE/DeepSea/pulls/1026"},"html":{"href":"https://github.com/SUSE/DeepSea/pull/1026"},"issue":{"href":"https://api.github.com/repos/SUSE/DeepSea/issues/1026"},"comments":{"href":"https://api.github.com/repos/SUSE/DeepSea/issues/1026/comments"},"review_comments":{"href":"https://api.github.com/repos/SUSE/DeepSea/pulls/1026/comments"},"review_comment":{"href":"https://api.github.com/repos/SUSE/DeepSea/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/SUSE/DeepSea/pulls/1026/commits"},"statuses":{"href":"https://api.github.com/repos/SUSE/DeepSea/statuses/c608b5698f3950ddf99ddbe0a97e849e967cc3fb"}},"author_association":"CONTRIBUTOR"}}
{"actor":{"display_login":"jan--f"}}