{"action":"created","comment":{"url":"https://api.github.com/repos/nvie/rq/pulls/comments/71986812","id":71986812,"diff_hunk":"@@ -649,6 +654,34 @@ def test_working_worker_cold_shutdown(self):\n         self.assertIsNotNone(shutdown_requested_date)\n         self.assertEqual(type(shutdown_requested_date).__name__, 'datetime')\n \n+    @slow\n+    def test_work_horse_death_sets_job_failed(self):\n+        \"\"\"worker with an ongoing job whose work horse dies unexpectadly should\n+        set the job's status either to FINISHED or FAILED\n+        \"\"\"\n+        fooq = Queue('foo')\n+        failed_q = get_failed_queue()\n+        self.assertEqual(failed_q.count, 0)\n+        self.assertEqual(fooq.count, 0)\n+        w = Worker(fooq)\n+        sentinel_file = '/tmp/.rq_sentinel_work_horse_death'\n+        if os.path.exists(sentinel_file):\n+            os.remove(sentinel_file)\n+        fooq.enqueue(create_file_after_timeout, sentinel_file, 100)\n+        job, queue = w.dequeue_job_and_maintain_ttl(5)\n+        w.fork_work_horse(job, queue)\n+        p = Process(target=wait_and_kill_work_horse, args=(w._horse_pid, 0.5))\n+        p.start()\n+        w.monitor_work_horse(job)\n+        job_status = job.get_status()\n+        p.join(1)\n+        if os.path.exists(sentinel_file):","path":"tests/test_worker.py","position":46,"original_position":46,"commit_id":"9b774771dbef2c00eecde93bf6cd13820f616349","original_commit_id":"3362fe2ba5c3a1f18810472f29fbbb03ea479212","user":{"login":"selwin","id":188658,"avatar_url":"https://avatars.githubusercontent.com/u/188658?v=3","gravatar_id":"","url":"https://api.github.com/users/selwin","html_url":"https://github.com/selwin","followers_url":"https://api.github.com/users/selwin/followers","following_url":"https://api.github.com/users/selwin/following{/other_user}","gists_url":"https://api.github.com/users/selwin/gists{/gist_id}","starred_url":"https://api.github.com/users/selwin/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/selwin/subscriptions","organizations_url":"https://api.github.com/users/selwin/orgs","repos_url":"https://api.github.com/users/selwin/repos","events_url":"https://api.github.com/users/selwin/events{/privacy}","received_events_url":"https://api.github.com/users/selwin/received_events","type":"User","site_admin":false},"body":"Yeah, please remove the `else` block so we can more easily tell what the expected behavior is by reading the test case.","created_at":"2016-07-24T11:14:53Z","updated_at":"2016-07-24T11:14:53Z","html_url":"https://github.com/nvie/rq/pull/710#discussion_r71986812","pull_request_url":"https://api.github.com/repos/nvie/rq/pulls/710","_links":{"self":{"href":"https://api.github.com/repos/nvie/rq/pulls/comments/71986812"},"html":{"href":"https://github.com/nvie/rq/pull/710#discussion_r71986812"},"pull_request":{"href":"https://api.github.com/repos/nvie/rq/pulls/710"}}},"pull_request":{"url":"https://api.github.com/repos/nvie/rq/pulls/710","id":71449726,"html_url":"https://github.com/nvie/rq/pull/710","diff_url":"https://github.com/nvie/rq/pull/710.diff","patch_url":"https://github.com/nvie/rq/pull/710.patch","issue_url":"https://api.github.com/repos/nvie/rq/issues/710","number":710,"state":"open","locked":false,"title":"Fix issue 702","user":{"login":"spiliopoulos","id":8586010,"avatar_url":"https://avatars.githubusercontent.com/u/8586010?v=3","gravatar_id":"","url":"https://api.github.com/users/spiliopoulos","html_url":"https://github.com/spiliopoulos","followers_url":"https://api.github.com/users/spiliopoulos/followers","following_url":"https://api.github.com/users/spiliopoulos/following{/other_user}","gists_url":"https://api.github.com/users/spiliopoulos/gists{/gist_id}","starred_url":"https://api.github.com/users/spiliopoulos/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/spiliopoulos/subscriptions","organizations_url":"https://api.github.com/users/spiliopoulos/orgs","repos_url":"https://api.github.com/users/spiliopoulos/repos","events_url":"https://api.github.com/users/spiliopoulos/events{/privacy}","received_events_url":"https://api.github.com/users/spiliopoulos/received_events","type":"User","site_admin":false},"body":"In order to solve issue 702 we have to check whether a work-horse terminated unexpectedly (by inspecting the exit code of the work-horse process). If it exited unexpectedly we check if the job has either been marked as finished, failed or other valid states. If it's not in any valid state we mark it as failed and move it to the failed queue. Since the process was terminated unexpectedly (think OOM) we do not have any exception context and we can't run any custom exception handlers.\r\n    \r\nThere is still a chance that the job will finish successfully but the work-horse process will be killed before the job is marked as finished and we will erroneously mark it as failed. The users should take care to write idempotent jobs.","created_at":"2016-05-25T19:33:22Z","updated_at":"2016-07-24T11:14:53Z","closed_at":null,"merged_at":null,"merge_commit_sha":"9b75bbb41f0e86134b34142390b29e80521f23af","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/nvie/rq/pulls/710/commits","review_comments_url":"https://api.github.com/repos/nvie/rq/pulls/710/comments","review_comment_url":"https://api.github.com/repos/nvie/rq/pulls/comments{/number}","comments_url":"https://api.github.com/repos/nvie/rq/issues/710/comments","statuses_url":"https://api.github.com/repos/nvie/rq/statuses/9b774771dbef2c00eecde93bf6cd13820f616349","head":{"label":"spiliopoulos:fix_issue_702","ref":"fix_issue_702","sha":"9b774771dbef2c00eecde93bf6cd13820f616349","user":{"login":"spiliopoulos","id":8586010,"avatar_url":"https://avatars.githubusercontent.com/u/8586010?v=3","gravatar_id":"","url":"https://api.github.com/users/spiliopoulos","html_url":"https://github.com/spiliopoulos","followers_url":"https://api.github.com/users/spiliopoulos/followers","following_url":"https://api.github.com/users/spiliopoulos/following{/other_user}","gists_url":"https://api.github.com/users/spiliopoulos/gists{/gist_id}","starred_url":"https://api.github.com/users/spiliopoulos/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/spiliopoulos/subscriptions","organizations_url":"https://api.github.com/users/spiliopoulos/orgs","repos_url":"https://api.github.com/users/spiliopoulos/repos","events_url":"https://api.github.com/users/spiliopoulos/events{/privacy}","received_events_url":"https://api.github.com/users/spiliopoulos/received_events","type":"User","site_admin":false},"repo":{"id":59303129,"name":"rq","full_name":"spiliopoulos/rq","owner":{"login":"spiliopoulos","id":8586010,"avatar_url":"https://avatars.githubusercontent.com/u/8586010?v=3","gravatar_id":"","url":"https://api.github.com/users/spiliopoulos","html_url":"https://github.com/spiliopoulos","followers_url":"https://api.github.com/users/spiliopoulos/followers","following_url":"https://api.github.com/users/spiliopoulos/following{/other_user}","gists_url":"https://api.github.com/users/spiliopoulos/gists{/gist_id}","starred_url":"https://api.github.com/users/spiliopoulos/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/spiliopoulos/subscriptions","organizations_url":"https://api.github.com/users/spiliopoulos/orgs","repos_url":"https://api.github.com/users/spiliopoulos/repos","events_url":"https://api.github.com/users/spiliopoulos/events{/privacy}","received_events_url":"https://api.github.com/users/spiliopoulos/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/spiliopoulos/rq","description":"Simple job queues for Python","fork":true,"url":"https://api.github.com/repos/spiliopoulos/rq","forks_url":"https://api.github.com/repos/spiliopoulos/rq/forks","keys_url":"https://api.github.com/repos/spiliopoulos/rq/keys{/key_id}","collaborators_url":"https://api.github.com/repos/spiliopoulos/rq/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/spiliopoulos/rq/teams","hooks_url":"https://api.github.com/repos/spiliopoulos/rq/hooks","issue_events_url":"https://api.github.com/repos/spiliopoulos/rq/issues/events{/number}","events_url":"https://api.github.com/repos/spiliopoulos/rq/events","assignees_url":"https://api.github.com/repos/spiliopoulos/rq/assignees{/user}","branches_url":"https://api.github.com/repos/spiliopoulos/rq/branches{/branch}","tags_url":"https://api.github.com/repos/spiliopoulos/rq/tags","blobs_url":"https://api.github.com/repos/spiliopoulos/rq/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/spiliopoulos/rq/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/spiliopoulos/rq/git/refs{/sha}","trees_url":"https://api.github.com/repos/spiliopoulos/rq/git/trees{/sha}","statuses_url":"https://api.github.com/repos/spiliopoulos/rq/statuses/{sha}","languages_url":"https://api.github.com/repos/spiliopoulos/rq/languages","stargazers_url":"https://api.github.com/repos/spiliopoulos/rq/stargazers","contributors_url":"https://api.github.com/repos/spiliopoulos/rq/contributors","subscribers_url":"https://api.github.com/repos/spiliopoulos/rq/subscribers","subscription_url":"https://api.github.com/repos/spiliopoulos/rq/subscription","commits_url":"https://api.github.com/repos/spiliopoulos/rq/commits{/sha}","git_commits_url":"https://api.github.com/repos/spiliopoulos/rq/git/commits{/sha}","comments_url":"https://api.github.com/repos/spiliopoulos/rq/comments{/number}","issue_comment_url":"https://api.github.com/repos/spiliopoulos/rq/issues/comments{/number}","contents_url":"https://api.github.com/repos/spiliopoulos/rq/contents/{+path}","compare_url":"https://api.github.com/repos/spiliopoulos/rq/compare/{base}...{head}","merges_url":"https://api.github.com/repos/spiliopoulos/rq/merges","archive_url":"https://api.github.com/repos/spiliopoulos/rq/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/spiliopoulos/rq/downloads","issues_url":"https://api.github.com/repos/spiliopoulos/rq/issues{/number}","pulls_url":"https://api.github.com/repos/spiliopoulos/rq/pulls{/number}","milestones_url":"https://api.github.com/repos/spiliopoulos/rq/milestones{/number}","notifications_url":"https://api.github.com/repos/spiliopoulos/rq/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/spiliopoulos/rq/labels{/name}","releases_url":"https://api.github.com/repos/spiliopoulos/rq/releases{/id}","deployments_url":"https://api.github.com/repos/spiliopoulos/rq/deployments","created_at":"2016-05-20T14:55:43Z","updated_at":"2016-05-20T14:55:44Z","pushed_at":"2016-07-16T06:26:40Z","git_url":"git://github.com/spiliopoulos/rq.git","ssh_url":"git@github.com:spiliopoulos/rq.git","clone_url":"https://github.com/spiliopoulos/rq.git","svn_url":"https://github.com/spiliopoulos/rq","homepage":"http://python-rq.org","size":2704,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"nvie:master","ref":"master","sha":"07adca6ca35d1f121bd58ada4d7ed990313925f8","user":{"login":"nvie","id":83844,"avatar_url":"https://avatars.githubusercontent.com/u/83844?v=3","gravatar_id":"","url":"https://api.github.com/users/nvie","html_url":"https://github.com/nvie","followers_url":"https://api.github.com/users/nvie/followers","following_url":"https://api.github.com/users/nvie/following{/other_user}","gists_url":"https://api.github.com/users/nvie/gists{/gist_id}","starred_url":"https://api.github.com/users/nvie/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/nvie/subscriptions","organizations_url":"https://api.github.com/users/nvie/orgs","repos_url":"https://api.github.com/users/nvie/repos","events_url":"https://api.github.com/users/nvie/events{/privacy}","received_events_url":"https://api.github.com/users/nvie/received_events","type":"User","site_admin":false},"repo":{"id":2771544,"name":"rq","full_name":"nvie/rq","owner":{"login":"nvie","id":83844,"avatar_url":"https://avatars.githubusercontent.com/u/83844?v=3","gravatar_id":"","url":"https://api.github.com/users/nvie","html_url":"https://github.com/nvie","followers_url":"https://api.github.com/users/nvie/followers","following_url":"https://api.github.com/users/nvie/following{/other_user}","gists_url":"https://api.github.com/users/nvie/gists{/gist_id}","starred_url":"https://api.github.com/users/nvie/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/nvie/subscriptions","organizations_url":"https://api.github.com/users/nvie/orgs","repos_url":"https://api.github.com/users/nvie/repos","events_url":"https://api.github.com/users/nvie/events{/privacy}","received_events_url":"https://api.github.com/users/nvie/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/nvie/rq","description":"Simple job queues for Python","fork":false,"url":"https://api.github.com/repos/nvie/rq","forks_url":"https://api.github.com/repos/nvie/rq/forks","keys_url":"https://api.github.com/repos/nvie/rq/keys{/key_id}","collaborators_url":"https://api.github.com/repos/nvie/rq/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/nvie/rq/teams","hooks_url":"https://api.github.com/repos/nvie/rq/hooks","issue_events_url":"https://api.github.com/repos/nvie/rq/issues/events{/number}","events_url":"https://api.github.com/repos/nvie/rq/events","assignees_url":"https://api.github.com/repos/nvie/rq/assignees{/user}","branches_url":"https://api.github.com/repos/nvie/rq/branches{/branch}","tags_url":"https://api.github.com/repos/nvie/rq/tags","blobs_url":"https://api.github.com/repos/nvie/rq/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/nvie/rq/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/nvie/rq/git/refs{/sha}","trees_url":"https://api.github.com/repos/nvie/rq/git/trees{/sha}","statuses_url":"https://api.github.com/repos/nvie/rq/statuses/{sha}","languages_url":"https://api.github.com/repos/nvie/rq/languages","stargazers_url":"https://api.github.com/repos/nvie/rq/stargazers","contributors_url":"https://api.github.com/repos/nvie/rq/contributors","subscribers_url":"https://api.github.com/repos/nvie/rq/subscribers","subscription_url":"https://api.github.com/repos/nvie/rq/subscription","commits_url":"https://api.github.com/repos/nvie/rq/commits{/sha}","git_commits_url":"https://api.github.com/repos/nvie/rq/git/commits{/sha}","comments_url":"https://api.github.com/repos/nvie/rq/comments{/number}","issue_comment_url":"https://api.github.com/repos/nvie/rq/issues/comments{/number}","contents_url":"https://api.github.com/repos/nvie/rq/contents/{+path}","compare_url":"https://api.github.com/repos/nvie/rq/compare/{base}...{head}","merges_url":"https://api.github.com/repos/nvie/rq/merges","archive_url":"https://api.github.com/repos/nvie/rq/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/nvie/rq/downloads","issues_url":"https://api.github.com/repos/nvie/rq/issues{/number}","pulls_url":"https://api.github.com/repos/nvie/rq/pulls{/number}","milestones_url":"https://api.github.com/repos/nvie/rq/milestones{/number}","notifications_url":"https://api.github.com/repos/nvie/rq/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/nvie/rq/labels{/name}","releases_url":"https://api.github.com/repos/nvie/rq/releases{/id}","deployments_url":"https://api.github.com/repos/nvie/rq/deployments","created_at":"2011-11-14T10:53:48Z","updated_at":"2016-07-24T09:53:01Z","pushed_at":"2016-07-21T07:52:09Z","git_url":"git://github.com/nvie/rq.git","ssh_url":"git@github.com:nvie/rq.git","clone_url":"https://github.com/nvie/rq.git","svn_url":"https://github.com/nvie/rq","homepage":"http://python-rq.org","size":3574,"stargazers_count":3109,"watchers_count":3109,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":true,"forks_count":618,"mirror_url":null,"open_issues_count":101,"forks":618,"open_issues":101,"watchers":3109,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/nvie/rq/pulls/710"},"html":{"href":"https://github.com/nvie/rq/pull/710"},"issue":{"href":"https://api.github.com/repos/nvie/rq/issues/710"},"comments":{"href":"https://api.github.com/repos/nvie/rq/issues/710/comments"},"review_comments":{"href":"https://api.github.com/repos/nvie/rq/pulls/710/comments"},"review_comment":{"href":"https://api.github.com/repos/nvie/rq/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/nvie/rq/pulls/710/commits"},"statuses":{"href":"https://api.github.com/repos/nvie/rq/statuses/9b774771dbef2c00eecde93bf6cd13820f616349"}}}}
{"actor":{"display_login":"selwin"}}