{"action":"created","comment":{"url":"https://api.github.com/repos/NixOS/nixops/pulls/comments/165854123","pull_request_review_id":93854621,"id":165854123,"diff_hunk":"@@ -182,16 +183,40 @@ def reboot(self, hard=False):\n             reboot_command = \"systemctl reboot\"\n         self.run_command(reboot_command, check=False)\n         self.state = self.STARTING\n-        self.ssh.reset()\n+        if reset:\n+            self.ssh.reset()\n \n     def reboot_sync(self, hard=False):\n         \"\"\"Reboot this machine and wait until it's up again.\"\"\"\n-        self.reboot(hard=hard)\n-        self.log_start(\"waiting for the machine to finish rebooting...\")\n-        nixops.util.wait_for_tcp_port(self.get_ssh_name(), self.ssh_port, open=False, callback=lambda: self.log_continue(\".\"))\n-        self.log_continue(\"[down]\")\n-        nixops.util.wait_for_tcp_port(self.get_ssh_name(), self.ssh_port, callback=lambda: self.log_continue(\".\"))\n-        self.log_end(\"[up]\")\n+\n+        # To check when the machine has finished rebooting in a race-free\n+        # manner, we compare the output of `last reboot` before and after\n+        # the reboot. Once the output has changed, the reboot is done.\n+        def get_last_reboot_output():\n+            return self.run_command('last reboot --time-format iso | head -n1', capture_stdout=True).rstrip()\n+\n+        pre_reboot_last_reboot_output = get_last_reboot_output()\n+\n+        # We set reset=False so that we can continue running `last reboot`\n+        # remotely; when the reboot happens, our SSH connection will be reset\n+        # by the remote side instead.\n+        self.reboot(hard=hard, reset=False)\n+\n+        self.log_start(\"waiting for reboot to complete...\")\n+        while True:\n+            last_reboot_output = None\n+            try:\n+                last_reboot_output = get_last_reboot_output()\n+            except nixops.ssh_util.SSHCommandFailed:\n+                # We accept this because the machine might be down,\n+                # and show an 'x' as progress indicator in that case.\n+                self.log_continue(\"x\")\n+            if last_reboot_output is not None and last_reboot_output != pre_reboot_last_reboot_output:\n+                break","path":"nixops/backends/__init__.py","position":57,"original_position":57,"commit_id":"37bc5afde9531265b4e09efd5532b0097f4c94f3","original_commit_id":"37bc5afde9531265b4e09efd5532b0097f4c94f3","user":{"login":"domenkozar","id":126339,"avatar_url":"https://avatars2.githubusercontent.com/u/126339?v=4","gravatar_id":"","url":"https://api.github.com/users/domenkozar","html_url":"https://github.com/domenkozar","followers_url":"https://api.github.com/users/domenkozar/followers","following_url":"https://api.github.com/users/domenkozar/following{/other_user}","gists_url":"https://api.github.com/users/domenkozar/gists{/gist_id}","starred_url":"https://api.github.com/users/domenkozar/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/domenkozar/subscriptions","organizations_url":"https://api.github.com/users/domenkozar/orgs","repos_url":"https://api.github.com/users/domenkozar/repos","events_url":"https://api.github.com/users/domenkozar/events{/privacy}","received_events_url":"https://api.github.com/users/domenkozar/received_events","type":"User","site_admin":false},"body":"Shouldn't this call `ssh.reset()` eventually?","created_at":"2018-02-04T19:05:51Z","updated_at":"2018-02-04T19:05:51Z","html_url":"https://github.com/NixOS/nixops/pull/857#discussion_r165854123","pull_request_url":"https://api.github.com/repos/NixOS/nixops/pulls/857","author_association":"OWNER","_links":{"self":{"href":"https://api.github.com/repos/NixOS/nixops/pulls/comments/165854123"},"html":{"href":"https://github.com/NixOS/nixops/pull/857#discussion_r165854123"},"pull_request":{"href":"https://api.github.com/repos/NixOS/nixops/pulls/857"}}},"pull_request":{"url":"https://api.github.com/repos/NixOS/nixops/pulls/857","id":165531215,"html_url":"https://github.com/NixOS/nixops/pull/857","diff_url":"https://github.com/NixOS/nixops/pull/857.diff","patch_url":"https://github.com/NixOS/nixops/pull/857.patch","issue_url":"https://api.github.com/repos/NixOS/nixops/issues/857","number":857,"state":"open","locked":false,"title":"Don't use port open check to determine if reboot completed. Fixes #856.","user":{"login":"nh2","id":399535,"avatar_url":"https://avatars1.githubusercontent.com/u/399535?v=4","gravatar_id":"","url":"https://api.github.com/users/nh2","html_url":"https://github.com/nh2","followers_url":"https://api.github.com/users/nh2/followers","following_url":"https://api.github.com/users/nh2/following{/other_user}","gists_url":"https://api.github.com/users/nh2/gists{/gist_id}","starred_url":"https://api.github.com/users/nh2/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/nh2/subscriptions","organizations_url":"https://api.github.com/users/nh2/orgs","repos_url":"https://api.github.com/users/nh2/repos","events_url":"https://api.github.com/users/nh2/events{/privacy}","received_events_url":"https://api.github.com/users/nh2/received_events","type":"User","site_admin":false},"body":"See #856\r\n\r\nThe old approach, waiting for the machine to not having an open\r\nport, and then waiting for it to be open again, was insufficient,\r\nbecause of the race condition that the machine rebooted so quickly\r\nthat the port was immediately open again without nixops noticing\r\nthat it went down. I experienced this on a Hetzner cloud server.\r\n\r\nThe new approach checks the `last reboot` on the remote side\r\nto change, which is not racy.","created_at":"2018-01-27T18:56:28Z","updated_at":"2018-02-04T19:05:51Z","closed_at":null,"merged_at":null,"merge_commit_sha":"77eba3e1089387e2bcecf284b504183f2c6c30d4","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"milestone":null,"commits_url":"https://api.github.com/repos/NixOS/nixops/pulls/857/commits","review_comments_url":"https://api.github.com/repos/NixOS/nixops/pulls/857/comments","review_comment_url":"https://api.github.com/repos/NixOS/nixops/pulls/comments{/number}","comments_url":"https://api.github.com/repos/NixOS/nixops/issues/857/comments","statuses_url":"https://api.github.com/repos/NixOS/nixops/statuses/37bc5afde9531265b4e09efd5532b0097f4c94f3","head":{"label":"nh2:issue-856-too-fast-reboot-race-condition","ref":"issue-856-too-fast-reboot-race-condition","sha":"37bc5afde9531265b4e09efd5532b0097f4c94f3","user":{"login":"nh2","id":399535,"avatar_url":"https://avatars1.githubusercontent.com/u/399535?v=4","gravatar_id":"","url":"https://api.github.com/users/nh2","html_url":"https://github.com/nh2","followers_url":"https://api.github.com/users/nh2/followers","following_url":"https://api.github.com/users/nh2/following{/other_user}","gists_url":"https://api.github.com/users/nh2/gists{/gist_id}","starred_url":"https://api.github.com/users/nh2/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/nh2/subscriptions","organizations_url":"https://api.github.com/users/nh2/orgs","repos_url":"https://api.github.com/users/nh2/repos","events_url":"https://api.github.com/users/nh2/events{/privacy}","received_events_url":"https://api.github.com/users/nh2/received_events","type":"User","site_admin":false},"repo":{"id":81941851,"name":"nixops","full_name":"nh2/nixops","owner":{"login":"nh2","id":399535,"avatar_url":"https://avatars1.githubusercontent.com/u/399535?v=4","gravatar_id":"","url":"https://api.github.com/users/nh2","html_url":"https://github.com/nh2","followers_url":"https://api.github.com/users/nh2/followers","following_url":"https://api.github.com/users/nh2/following{/other_user}","gists_url":"https://api.github.com/users/nh2/gists{/gist_id}","starred_url":"https://api.github.com/users/nh2/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/nh2/subscriptions","organizations_url":"https://api.github.com/users/nh2/orgs","repos_url":"https://api.github.com/users/nh2/repos","events_url":"https://api.github.com/users/nh2/events{/privacy}","received_events_url":"https://api.github.com/users/nh2/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/nh2/nixops","description":"NixOps, the NixOS-based cloud deployment tool","fork":true,"url":"https://api.github.com/repos/nh2/nixops","forks_url":"https://api.github.com/repos/nh2/nixops/forks","keys_url":"https://api.github.com/repos/nh2/nixops/keys{/key_id}","collaborators_url":"https://api.github.com/repos/nh2/nixops/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/nh2/nixops/teams","hooks_url":"https://api.github.com/repos/nh2/nixops/hooks","issue_events_url":"https://api.github.com/repos/nh2/nixops/issues/events{/number}","events_url":"https://api.github.com/repos/nh2/nixops/events","assignees_url":"https://api.github.com/repos/nh2/nixops/assignees{/user}","branches_url":"https://api.github.com/repos/nh2/nixops/branches{/branch}","tags_url":"https://api.github.com/repos/nh2/nixops/tags","blobs_url":"https://api.github.com/repos/nh2/nixops/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/nh2/nixops/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/nh2/nixops/git/refs{/sha}","trees_url":"https://api.github.com/repos/nh2/nixops/git/trees{/sha}","statuses_url":"https://api.github.com/repos/nh2/nixops/statuses/{sha}","languages_url":"https://api.github.com/repos/nh2/nixops/languages","stargazers_url":"https://api.github.com/repos/nh2/nixops/stargazers","contributors_url":"https://api.github.com/repos/nh2/nixops/contributors","subscribers_url":"https://api.github.com/repos/nh2/nixops/subscribers","subscription_url":"https://api.github.com/repos/nh2/nixops/subscription","commits_url":"https://api.github.com/repos/nh2/nixops/commits{/sha}","git_commits_url":"https://api.github.com/repos/nh2/nixops/git/commits{/sha}","comments_url":"https://api.github.com/repos/nh2/nixops/comments{/number}","issue_comment_url":"https://api.github.com/repos/nh2/nixops/issues/comments{/number}","contents_url":"https://api.github.com/repos/nh2/nixops/contents/{+path}","compare_url":"https://api.github.com/repos/nh2/nixops/compare/{base}...{head}","merges_url":"https://api.github.com/repos/nh2/nixops/merges","archive_url":"https://api.github.com/repos/nh2/nixops/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/nh2/nixops/downloads","issues_url":"https://api.github.com/repos/nh2/nixops/issues{/number}","pulls_url":"https://api.github.com/repos/nh2/nixops/pulls{/number}","milestones_url":"https://api.github.com/repos/nh2/nixops/milestones{/number}","notifications_url":"https://api.github.com/repos/nh2/nixops/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/nh2/nixops/labels{/name}","releases_url":"https://api.github.com/repos/nh2/nixops/releases{/id}","deployments_url":"https://api.github.com/repos/nh2/nixops/deployments","created_at":"2017-02-14T12:11:23Z","updated_at":"2017-02-14T12:11:27Z","pushed_at":"2018-02-03T20:16:29Z","git_url":"git://github.com/nh2/nixops.git","ssh_url":"git@github.com:nh2/nixops.git","clone_url":"https://github.com/nh2/nixops.git","svn_url":"https://github.com/nh2/nixops","homepage":"http://nixos.org/","size":4556,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":0,"license":null,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"NixOS:master","ref":"master","sha":"76730c6e5e58300319ecdeb4e468396635ff0c59","user":{"login":"NixOS","id":487568,"avatar_url":"https://avatars3.githubusercontent.com/u/487568?v=4","gravatar_id":"","url":"https://api.github.com/users/NixOS","html_url":"https://github.com/NixOS","followers_url":"https://api.github.com/users/NixOS/followers","following_url":"https://api.github.com/users/NixOS/following{/other_user}","gists_url":"https://api.github.com/users/NixOS/gists{/gist_id}","starred_url":"https://api.github.com/users/NixOS/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/NixOS/subscriptions","organizations_url":"https://api.github.com/users/NixOS/orgs","repos_url":"https://api.github.com/users/NixOS/repos","events_url":"https://api.github.com/users/NixOS/events{/privacy}","received_events_url":"https://api.github.com/users/NixOS/received_events","type":"Organization","site_admin":false},"repo":{"id":2637203,"name":"nixops","full_name":"NixOS/nixops","owner":{"login":"NixOS","id":487568,"avatar_url":"https://avatars3.githubusercontent.com/u/487568?v=4","gravatar_id":"","url":"https://api.github.com/users/NixOS","html_url":"https://github.com/NixOS","followers_url":"https://api.github.com/users/NixOS/followers","following_url":"https://api.github.com/users/NixOS/following{/other_user}","gists_url":"https://api.github.com/users/NixOS/gists{/gist_id}","starred_url":"https://api.github.com/users/NixOS/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/NixOS/subscriptions","organizations_url":"https://api.github.com/users/NixOS/orgs","repos_url":"https://api.github.com/users/NixOS/repos","events_url":"https://api.github.com/users/NixOS/events{/privacy}","received_events_url":"https://api.github.com/users/NixOS/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/NixOS/nixops","description":"NixOps, the NixOS-based cloud deployment tool","fork":false,"url":"https://api.github.com/repos/NixOS/nixops","forks_url":"https://api.github.com/repos/NixOS/nixops/forks","keys_url":"https://api.github.com/repos/NixOS/nixops/keys{/key_id}","collaborators_url":"https://api.github.com/repos/NixOS/nixops/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/NixOS/nixops/teams","hooks_url":"https://api.github.com/repos/NixOS/nixops/hooks","issue_events_url":"https://api.github.com/repos/NixOS/nixops/issues/events{/number}","events_url":"https://api.github.com/repos/NixOS/nixops/events","assignees_url":"https://api.github.com/repos/NixOS/nixops/assignees{/user}","branches_url":"https://api.github.com/repos/NixOS/nixops/branches{/branch}","tags_url":"https://api.github.com/repos/NixOS/nixops/tags","blobs_url":"https://api.github.com/repos/NixOS/nixops/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/NixOS/nixops/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/NixOS/nixops/git/refs{/sha}","trees_url":"https://api.github.com/repos/NixOS/nixops/git/trees{/sha}","statuses_url":"https://api.github.com/repos/NixOS/nixops/statuses/{sha}","languages_url":"https://api.github.com/repos/NixOS/nixops/languages","stargazers_url":"https://api.github.com/repos/NixOS/nixops/stargazers","contributors_url":"https://api.github.com/repos/NixOS/nixops/contributors","subscribers_url":"https://api.github.com/repos/NixOS/nixops/subscribers","subscription_url":"https://api.github.com/repos/NixOS/nixops/subscription","commits_url":"https://api.github.com/repos/NixOS/nixops/commits{/sha}","git_commits_url":"https://api.github.com/repos/NixOS/nixops/git/commits{/sha}","comments_url":"https://api.github.com/repos/NixOS/nixops/comments{/number}","issue_comment_url":"https://api.github.com/repos/NixOS/nixops/issues/comments{/number}","contents_url":"https://api.github.com/repos/NixOS/nixops/contents/{+path}","compare_url":"https://api.github.com/repos/NixOS/nixops/compare/{base}...{head}","merges_url":"https://api.github.com/repos/NixOS/nixops/merges","archive_url":"https://api.github.com/repos/NixOS/nixops/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/NixOS/nixops/downloads","issues_url":"https://api.github.com/repos/NixOS/nixops/issues{/number}","pulls_url":"https://api.github.com/repos/NixOS/nixops/pulls{/number}","milestones_url":"https://api.github.com/repos/NixOS/nixops/milestones{/number}","notifications_url":"https://api.github.com/repos/NixOS/nixops/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/NixOS/nixops/labels{/name}","releases_url":"https://api.github.com/repos/NixOS/nixops/releases{/id}","deployments_url":"https://api.github.com/repos/NixOS/nixops/deployments","created_at":"2011-10-24T15:49:58Z","updated_at":"2018-01-26T00:13:14Z","pushed_at":"2018-02-04T19:04:07Z","git_url":"git://github.com/NixOS/nixops.git","ssh_url":"git@github.com:NixOS/nixops.git","clone_url":"https://github.com/NixOS/nixops.git","svn_url":"https://github.com/NixOS/nixops","homepage":"http://nixos.org/","size":5025,"stargazers_count":391,"watchers_count":391,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":185,"mirror_url":null,"archived":false,"open_issues_count":272,"license":{"key":"lgpl-3.0","name":"GNU Lesser General Public License v3.0","spdx_id":"LGPL-3.0","url":"https://api.github.com/licenses/lgpl-3.0"},"forks":185,"open_issues":272,"watchers":391,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/NixOS/nixops/pulls/857"},"html":{"href":"https://github.com/NixOS/nixops/pull/857"},"issue":{"href":"https://api.github.com/repos/NixOS/nixops/issues/857"},"comments":{"href":"https://api.github.com/repos/NixOS/nixops/issues/857/comments"},"review_comments":{"href":"https://api.github.com/repos/NixOS/nixops/pulls/857/comments"},"review_comment":{"href":"https://api.github.com/repos/NixOS/nixops/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/NixOS/nixops/pulls/857/commits"},"statuses":{"href":"https://api.github.com/repos/NixOS/nixops/statuses/37bc5afde9531265b4e09efd5532b0097f4c94f3"}},"author_association":"CONTRIBUTOR"}}
{"actor":{"display_login":"domenkozar"}}