{"action":"created","comment":{"url":"https://api.github.com/repos/CanonicalLtd/DiagCephCluster/pulls/comments/63812008","id":63812008,"diff_hunk":"@@ -33,17 +37,80 @@ def _get_opt_parser(self):\n         parser.add_option('-p', '--pass', dest='password', default=None)\n         return parser\n \n-    def _get_connection(self):\n+    def _get_connection(self, hostname, username, password):\n         client = paramiko.SSHClient()\n         client.set_missing_host_key_policy(paramiko.AutoAddPolicy())\n-        client.connect(hostname=self.options.host, username=self.options.user,\n-                       password=self.options.password)\n+        client.connect(hostname=hostname, username=username,\n+                       password=password)\n         return client\n \n-    def _execute_command(self, command):\n-        (stdin, stdout, stderr) = self.connection.exec_command(command)\n+    def _execute_command(self, connection, command):\n+        (stdin, stdout, stderr) = connection.exec_command(command)\n         return (stdout, stderr)\n \n+    def troubleshoot_mon(self):\n+        command = 'sudo ceph health'\n+        (output, err) = self._execute_command(self.connection, command)\n+\n+        try:\n+            self._get_eof(output, command)\n+            cluster_status = output.read().split(' ')[0].strip()\n+\n+        except TimeoutError as err:\n+            # cluster is screwed\n+            # Best guess some mon machines are down\n+            # just printing error for now\n+            # TODO\n+            print err\n+            return\n+\n+        if cluster_status in self.GOOD_HEALTH:\n+            print cluster_status\n+            print 'All good up here, no mon issues :-)'\n+\n+        if cluster_status in self.BAD_HEALTH:\n+            print 'MON Status : ', cluster_status\n+            print '\\nProbable cause Ceph service not running in some machines'\n+            print 'Try & start service in the machines not in quorum',\n+            print '(yes/no) (default no)?',\n+            response = raw_input()\n+\n+            if response in ['no', '']:\n+                print 'not proceeding with updating machines, aborting'\n+                return\n+            if response not in ['yes']:\n+                print 'response not valid, aborting'\n+                return\n+\n+            command = 'sudo ceph mon_status'\n+            (output, err) = self._execute_command(self.connection, command)\n+\n+            mon_status = eval(output.read())\n+            quorum_list = mon_status['quorum']\n+            mon_list = mon_status['monmap']['mons']\n+\n+            if len(quorum_list) != len(mon_list):\n+                for mon in mon_list:\n+                    if mon['rank'] not in quorum_list:\n+                        print '\\n' + mon['name'] + ' not in quorum list,',\n+                        print 'restarting ceph services'\n+                        mon_addr = mon['addr'].split(':')[0]\n+\n+                        self._restart_ceph_services(mon_addr,\n+                                                    self.options.user,\n+                                                    self.options.password)\n+\n+    def _restart_ceph_services(self, addr, username, password):\n+        connection = self._get_connection(addr, username, password)\n+        self._execute_command(connection, 'sudo start ceph-all')","path":"run.py","position":107,"original_position":107,"commit_id":"59a5b986b3412dbddf69c81ff5df6ff2c8b9e297","original_commit_id":"59a5b986b3412dbddf69c81ff5df6ff2c8b9e297","user":{"login":"cholcombe973","id":3058195,"avatar_url":"https://avatars.githubusercontent.com/u/3058195?v=3","gravatar_id":"","url":"https://api.github.com/users/cholcombe973","html_url":"https://github.com/cholcombe973","followers_url":"https://api.github.com/users/cholcombe973/followers","following_url":"https://api.github.com/users/cholcombe973/following{/other_user}","gists_url":"https://api.github.com/users/cholcombe973/gists{/gist_id}","starred_url":"https://api.github.com/users/cholcombe973/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cholcombe973/subscriptions","organizations_url":"https://api.github.com/users/cholcombe973/orgs","repos_url":"https://api.github.com/users/cholcombe973/repos","events_url":"https://api.github.com/users/cholcombe973/events{/privacy}","received_events_url":"https://api.github.com/users/cholcombe973/received_events","type":"User","site_admin":false},"body":"It's easy to see what systemd ceph looks like. Just deploy the xenial ceph\nwith juju\nOn May 18, 2016 6:46 PM, \"Kumar Rishabh\" <notifications@github.com> wrote:\n\n> In run.py\n> <https://github.com/CanonicalLtd/DiagCephCluster/pull/1#discussion_r63811918>\n> :\n>\n> > +            mon_list = mon_status['monmap']['mons']\n> > +\n> > +            if len(quorum_list) != len(mon_list):\n> > +                for mon in mon_list:\n> > +                    if mon['rank'] not in quorum_list:\n> > +                        print '\\n' + mon['name'] + ' not in quorum list,',\n> > +                        print 'restarting ceph services'\n> > +                        mon_addr = mon['addr'].split(':')[0]\n> > +\n> > +                        self._restart_ceph_services(mon_addr,\n> > +                                                    self.options.user,\n> > +                                                    self.options.password)\n> > +\n> > +    def _restart_ceph_services(self, addr, username, password):\n> > +        connection = self._get_connection(addr, username, password)\n> > +        self._execute_command(connection, 'sudo start ceph-all')\n>\n> Yeah, almost all the systems I work on are sysvinit based and hence didnt\n> check. Will correct that.\n>\n> —\n> You are receiving this because you were mentioned.\n> Reply to this email directly or view it on GitHub\n> <https://github.com/CanonicalLtd/DiagCephCluster/pull/1/files/59a5b986b3412dbddf69c81ff5df6ff2c8b9e297#r63811918>\n>\n","created_at":"2016-05-19T01:47:33Z","updated_at":"2016-05-19T01:47:33Z","html_url":"https://github.com/CanonicalLtd/DiagCephCluster/pull/1#discussion_r63812008","pull_request_url":"https://api.github.com/repos/CanonicalLtd/DiagCephCluster/pulls/1","_links":{"self":{"href":"https://api.github.com/repos/CanonicalLtd/DiagCephCluster/pulls/comments/63812008"},"html":{"href":"https://github.com/CanonicalLtd/DiagCephCluster/pull/1#discussion_r63812008"},"pull_request":{"href":"https://api.github.com/repos/CanonicalLtd/DiagCephCluster/pulls/1"}}},"pull_request":{"url":"https://api.github.com/repos/CanonicalLtd/DiagCephCluster/pulls/1","id":70354645,"html_url":"https://github.com/CanonicalLtd/DiagCephCluster/pull/1","diff_url":"https://github.com/CanonicalLtd/DiagCephCluster/pull/1.diff","patch_url":"https://github.com/CanonicalLtd/DiagCephCluster/pull/1.patch","issue_url":"https://api.github.com/repos/CanonicalLtd/DiagCephCluster/issues/1","number":1,"state":"open","locked":false,"title":"Adding basic monitor checks","user":{"login":"hellboy2k8","id":1503683,"avatar_url":"https://avatars.githubusercontent.com/u/1503683?v=3","gravatar_id":"","url":"https://api.github.com/users/hellboy2k8","html_url":"https://github.com/hellboy2k8","followers_url":"https://api.github.com/users/hellboy2k8/followers","following_url":"https://api.github.com/users/hellboy2k8/following{/other_user}","gists_url":"https://api.github.com/users/hellboy2k8/gists{/gist_id}","starred_url":"https://api.github.com/users/hellboy2k8/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/hellboy2k8/subscriptions","organizations_url":"https://api.github.com/users/hellboy2k8/orgs","repos_url":"https://api.github.com/users/hellboy2k8/repos","events_url":"https://api.github.com/users/hellboy2k8/events{/privacy}","received_events_url":"https://api.github.com/users/hellboy2k8/received_events","type":"User","site_admin":false},"body":"This patch adds basic monitor checks which works as follows\r\n    if `ceph health status` does not return we need to restart machines\r\n        TODO\r\n    if `ceph health status` returns GOOD_HEALTH(say HEALTH_OK)\r\n        No monitor issues\r\n    if `ceph health status` return BAD_HEALTH(say HEALTH_WARN)\r\n        It mostly means the quorum is being establised some mons maybe\r\n        down\r\n        So if the user permits the script would log into each machine\r\n        not in quorum and restart ceph services","created_at":"2016-05-17T13:15:50Z","updated_at":"2016-05-19T01:47:34Z","closed_at":null,"merged_at":null,"merge_commit_sha":"08e820ff06dbeccc61717ac83397dd4447ba57b2","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/CanonicalLtd/DiagCephCluster/pulls/1/commits","review_comments_url":"https://api.github.com/repos/CanonicalLtd/DiagCephCluster/pulls/1/comments","review_comment_url":"https://api.github.com/repos/CanonicalLtd/DiagCephCluster/pulls/comments{/number}","comments_url":"https://api.github.com/repos/CanonicalLtd/DiagCephCluster/issues/1/comments","statuses_url":"https://api.github.com/repos/CanonicalLtd/DiagCephCluster/statuses/59a5b986b3412dbddf69c81ff5df6ff2c8b9e297","head":{"label":"hellboy2k8:master","ref":"master","sha":"59a5b986b3412dbddf69c81ff5df6ff2c8b9e297","user":{"login":"hellboy2k8","id":1503683,"avatar_url":"https://avatars.githubusercontent.com/u/1503683?v=3","gravatar_id":"","url":"https://api.github.com/users/hellboy2k8","html_url":"https://github.com/hellboy2k8","followers_url":"https://api.github.com/users/hellboy2k8/followers","following_url":"https://api.github.com/users/hellboy2k8/following{/other_user}","gists_url":"https://api.github.com/users/hellboy2k8/gists{/gist_id}","starred_url":"https://api.github.com/users/hellboy2k8/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/hellboy2k8/subscriptions","organizations_url":"https://api.github.com/users/hellboy2k8/orgs","repos_url":"https://api.github.com/users/hellboy2k8/repos","events_url":"https://api.github.com/users/hellboy2k8/events{/privacy}","received_events_url":"https://api.github.com/users/hellboy2k8/received_events","type":"User","site_admin":false},"repo":{"id":59023293,"name":"DiagCephCluster","full_name":"hellboy2k8/DiagCephCluster","owner":{"login":"hellboy2k8","id":1503683,"avatar_url":"https://avatars.githubusercontent.com/u/1503683?v=3","gravatar_id":"","url":"https://api.github.com/users/hellboy2k8","html_url":"https://github.com/hellboy2k8","followers_url":"https://api.github.com/users/hellboy2k8/followers","following_url":"https://api.github.com/users/hellboy2k8/following{/other_user}","gists_url":"https://api.github.com/users/hellboy2k8/gists{/gist_id}","starred_url":"https://api.github.com/users/hellboy2k8/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/hellboy2k8/subscriptions","organizations_url":"https://api.github.com/users/hellboy2k8/orgs","repos_url":"https://api.github.com/users/hellboy2k8/repos","events_url":"https://api.github.com/users/hellboy2k8/events{/privacy}","received_events_url":"https://api.github.com/users/hellboy2k8/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/hellboy2k8/DiagCephCluster","description":"Diagnose Ceph Cluster GSOC","fork":true,"url":"https://api.github.com/repos/hellboy2k8/DiagCephCluster","forks_url":"https://api.github.com/repos/hellboy2k8/DiagCephCluster/forks","keys_url":"https://api.github.com/repos/hellboy2k8/DiagCephCluster/keys{/key_id}","collaborators_url":"https://api.github.com/repos/hellboy2k8/DiagCephCluster/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/hellboy2k8/DiagCephCluster/teams","hooks_url":"https://api.github.com/repos/hellboy2k8/DiagCephCluster/hooks","issue_events_url":"https://api.github.com/repos/hellboy2k8/DiagCephCluster/issues/events{/number}","events_url":"https://api.github.com/repos/hellboy2k8/DiagCephCluster/events","assignees_url":"https://api.github.com/repos/hellboy2k8/DiagCephCluster/assignees{/user}","branches_url":"https://api.github.com/repos/hellboy2k8/DiagCephCluster/branches{/branch}","tags_url":"https://api.github.com/repos/hellboy2k8/DiagCephCluster/tags","blobs_url":"https://api.github.com/repos/hellboy2k8/DiagCephCluster/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/hellboy2k8/DiagCephCluster/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/hellboy2k8/DiagCephCluster/git/refs{/sha}","trees_url":"https://api.github.com/repos/hellboy2k8/DiagCephCluster/git/trees{/sha}","statuses_url":"https://api.github.com/repos/hellboy2k8/DiagCephCluster/statuses/{sha}","languages_url":"https://api.github.com/repos/hellboy2k8/DiagCephCluster/languages","stargazers_url":"https://api.github.com/repos/hellboy2k8/DiagCephCluster/stargazers","contributors_url":"https://api.github.com/repos/hellboy2k8/DiagCephCluster/contributors","subscribers_url":"https://api.github.com/repos/hellboy2k8/DiagCephCluster/subscribers","subscription_url":"https://api.github.com/repos/hellboy2k8/DiagCephCluster/subscription","commits_url":"https://api.github.com/repos/hellboy2k8/DiagCephCluster/commits{/sha}","git_commits_url":"https://api.github.com/repos/hellboy2k8/DiagCephCluster/git/commits{/sha}","comments_url":"https://api.github.com/repos/hellboy2k8/DiagCephCluster/comments{/number}","issue_comment_url":"https://api.github.com/repos/hellboy2k8/DiagCephCluster/issues/comments{/number}","contents_url":"https://api.github.com/repos/hellboy2k8/DiagCephCluster/contents/{+path}","compare_url":"https://api.github.com/repos/hellboy2k8/DiagCephCluster/compare/{base}...{head}","merges_url":"https://api.github.com/repos/hellboy2k8/DiagCephCluster/merges","archive_url":"https://api.github.com/repos/hellboy2k8/DiagCephCluster/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/hellboy2k8/DiagCephCluster/downloads","issues_url":"https://api.github.com/repos/hellboy2k8/DiagCephCluster/issues{/number}","pulls_url":"https://api.github.com/repos/hellboy2k8/DiagCephCluster/pulls{/number}","milestones_url":"https://api.github.com/repos/hellboy2k8/DiagCephCluster/milestones{/number}","notifications_url":"https://api.github.com/repos/hellboy2k8/DiagCephCluster/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/hellboy2k8/DiagCephCluster/labels{/name}","releases_url":"https://api.github.com/repos/hellboy2k8/DiagCephCluster/releases{/id}","deployments_url":"https://api.github.com/repos/hellboy2k8/DiagCephCluster/deployments","created_at":"2016-05-17T13:07:28Z","updated_at":"2016-05-17T13:07:29Z","pushed_at":"2016-05-17T13:15:02Z","git_url":"git://github.com/hellboy2k8/DiagCephCluster.git","ssh_url":"git@github.com:hellboy2k8/DiagCephCluster.git","clone_url":"https://github.com/hellboy2k8/DiagCephCluster.git","svn_url":"https://github.com/hellboy2k8/DiagCephCluster","homepage":null,"size":4,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"CanonicalLtd:master","ref":"master","sha":"a7ee794669c76a6011200e7d3e565522786af38d","user":{"login":"CanonicalLtd","id":7490288,"avatar_url":"https://avatars.githubusercontent.com/u/7490288?v=3","gravatar_id":"","url":"https://api.github.com/users/CanonicalLtd","html_url":"https://github.com/CanonicalLtd","followers_url":"https://api.github.com/users/CanonicalLtd/followers","following_url":"https://api.github.com/users/CanonicalLtd/following{/other_user}","gists_url":"https://api.github.com/users/CanonicalLtd/gists{/gist_id}","starred_url":"https://api.github.com/users/CanonicalLtd/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/CanonicalLtd/subscriptions","organizations_url":"https://api.github.com/users/CanonicalLtd/orgs","repos_url":"https://api.github.com/users/CanonicalLtd/repos","events_url":"https://api.github.com/users/CanonicalLtd/events{/privacy}","received_events_url":"https://api.github.com/users/CanonicalLtd/received_events","type":"Organization","site_admin":false},"repo":{"id":58473134,"name":"DiagCephCluster","full_name":"CanonicalLtd/DiagCephCluster","owner":{"login":"CanonicalLtd","id":7490288,"avatar_url":"https://avatars.githubusercontent.com/u/7490288?v=3","gravatar_id":"","url":"https://api.github.com/users/CanonicalLtd","html_url":"https://github.com/CanonicalLtd","followers_url":"https://api.github.com/users/CanonicalLtd/followers","following_url":"https://api.github.com/users/CanonicalLtd/following{/other_user}","gists_url":"https://api.github.com/users/CanonicalLtd/gists{/gist_id}","starred_url":"https://api.github.com/users/CanonicalLtd/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/CanonicalLtd/subscriptions","organizations_url":"https://api.github.com/users/CanonicalLtd/orgs","repos_url":"https://api.github.com/users/CanonicalLtd/repos","events_url":"https://api.github.com/users/CanonicalLtd/events{/privacy}","received_events_url":"https://api.github.com/users/CanonicalLtd/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/CanonicalLtd/DiagCephCluster","description":"Diagnose Ceph Cluster GSOC","fork":false,"url":"https://api.github.com/repos/CanonicalLtd/DiagCephCluster","forks_url":"https://api.github.com/repos/CanonicalLtd/DiagCephCluster/forks","keys_url":"https://api.github.com/repos/CanonicalLtd/DiagCephCluster/keys{/key_id}","collaborators_url":"https://api.github.com/repos/CanonicalLtd/DiagCephCluster/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/CanonicalLtd/DiagCephCluster/teams","hooks_url":"https://api.github.com/repos/CanonicalLtd/DiagCephCluster/hooks","issue_events_url":"https://api.github.com/repos/CanonicalLtd/DiagCephCluster/issues/events{/number}","events_url":"https://api.github.com/repos/CanonicalLtd/DiagCephCluster/events","assignees_url":"https://api.github.com/repos/CanonicalLtd/DiagCephCluster/assignees{/user}","branches_url":"https://api.github.com/repos/CanonicalLtd/DiagCephCluster/branches{/branch}","tags_url":"https://api.github.com/repos/CanonicalLtd/DiagCephCluster/tags","blobs_url":"https://api.github.com/repos/CanonicalLtd/DiagCephCluster/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/CanonicalLtd/DiagCephCluster/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/CanonicalLtd/DiagCephCluster/git/refs{/sha}","trees_url":"https://api.github.com/repos/CanonicalLtd/DiagCephCluster/git/trees{/sha}","statuses_url":"https://api.github.com/repos/CanonicalLtd/DiagCephCluster/statuses/{sha}","languages_url":"https://api.github.com/repos/CanonicalLtd/DiagCephCluster/languages","stargazers_url":"https://api.github.com/repos/CanonicalLtd/DiagCephCluster/stargazers","contributors_url":"https://api.github.com/repos/CanonicalLtd/DiagCephCluster/contributors","subscribers_url":"https://api.github.com/repos/CanonicalLtd/DiagCephCluster/subscribers","subscription_url":"https://api.github.com/repos/CanonicalLtd/DiagCephCluster/subscription","commits_url":"https://api.github.com/repos/CanonicalLtd/DiagCephCluster/commits{/sha}","git_commits_url":"https://api.github.com/repos/CanonicalLtd/DiagCephCluster/git/commits{/sha}","comments_url":"https://api.github.com/repos/CanonicalLtd/DiagCephCluster/comments{/number}","issue_comment_url":"https://api.github.com/repos/CanonicalLtd/DiagCephCluster/issues/comments{/number}","contents_url":"https://api.github.com/repos/CanonicalLtd/DiagCephCluster/contents/{+path}","compare_url":"https://api.github.com/repos/CanonicalLtd/DiagCephCluster/compare/{base}...{head}","merges_url":"https://api.github.com/repos/CanonicalLtd/DiagCephCluster/merges","archive_url":"https://api.github.com/repos/CanonicalLtd/DiagCephCluster/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/CanonicalLtd/DiagCephCluster/downloads","issues_url":"https://api.github.com/repos/CanonicalLtd/DiagCephCluster/issues{/number}","pulls_url":"https://api.github.com/repos/CanonicalLtd/DiagCephCluster/pulls{/number}","milestones_url":"https://api.github.com/repos/CanonicalLtd/DiagCephCluster/milestones{/number}","notifications_url":"https://api.github.com/repos/CanonicalLtd/DiagCephCluster/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/CanonicalLtd/DiagCephCluster/labels{/name}","releases_url":"https://api.github.com/repos/CanonicalLtd/DiagCephCluster/releases{/id}","deployments_url":"https://api.github.com/repos/CanonicalLtd/DiagCephCluster/deployments","created_at":"2016-05-10T15:32:17Z","updated_at":"2016-05-17T11:53:34Z","pushed_at":"2016-05-17T13:15:50Z","git_url":"git://github.com/CanonicalLtd/DiagCephCluster.git","ssh_url":"git@github.com:CanonicalLtd/DiagCephCluster.git","clone_url":"https://github.com/CanonicalLtd/DiagCephCluster.git","svn_url":"https://github.com/CanonicalLtd/DiagCephCluster","homepage":null,"size":1,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":1,"mirror_url":null,"open_issues_count":1,"forks":1,"open_issues":1,"watchers":0,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/CanonicalLtd/DiagCephCluster/pulls/1"},"html":{"href":"https://github.com/CanonicalLtd/DiagCephCluster/pull/1"},"issue":{"href":"https://api.github.com/repos/CanonicalLtd/DiagCephCluster/issues/1"},"comments":{"href":"https://api.github.com/repos/CanonicalLtd/DiagCephCluster/issues/1/comments"},"review_comments":{"href":"https://api.github.com/repos/CanonicalLtd/DiagCephCluster/pulls/1/comments"},"review_comment":{"href":"https://api.github.com/repos/CanonicalLtd/DiagCephCluster/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/CanonicalLtd/DiagCephCluster/pulls/1/commits"},"statuses":{"href":"https://api.github.com/repos/CanonicalLtd/DiagCephCluster/statuses/59a5b986b3412dbddf69c81ff5df6ff2c8b9e297"}}}}