{"action":"created","comment":{"url":"https://api.github.com/repos/rackerlabs/otter/pulls/comments/75568346","id":75568346,"diff_hunk":"@@ -796,58 +812,60 @@ def modifier(_group, _state):\n         self.connection.execute.assert_called_once_with(\n             expectedCql, expectedData, ConsistencyLevel.QUORUM)\n \n-        self.kz_client.Lock.assert_called_once_with(\n-            '/locks/' + self.group.uuid)\n-\n-        self.lock._acquire.assert_called_once_with(timeout=10)\n-        self.lock.release.assert_called_once_with()\n+        self.assertEqual(self.lock.path, '/locks/' + self.group.uuid)\n+        self.assertFalse(self.lock.acquired)\n \n     def test_modify_state_local_lock_before_kz_lock(self):\n         \"\"\"\n         ``modify_state`` first acquires local lock then acquires kz lock\n         \"\"\"\n+        modifier_d = defer.Deferred()\n+        group_state = GroupState(tenant_id=self.tenant_id,\n+                                 group_id=self.group_id,\n+                                 group_name='a',\n+                                 active={},\n+                                 pending={},\n+                                 group_touched=None,\n+                                 policy_touched={},\n+                                 paused=True,\n+                                 status=ScalingGroupStatus.ACTIVE,\n+                                 desired=5)\n+\n         def modifier(_group, _state):\n-            group_state = GroupState(tenant_id=self.tenant_id,\n-                                     group_id=self.group_id,\n-                                     group_name='a',\n-                                     active={},\n-                                     pending={},\n-                                     group_touched=None,\n-                                     policy_touched={},\n-                                     paused=True,\n-                                     status=ScalingGroupStatus.ACTIVE,\n-                                     desired=5)\n-            return group_state\n+            return modifier_d\n \n         self.group.view_state = mock.Mock(return_value=defer.succeed('state'))\n-        # setup local lock\n+\n         llock = defer.DeferredLock()\n-        self.group.local_locks = mock.Mock(\n-            get_lock=mock.Mock(return_value=llock))\n+\n+        class Locks(object):","path":"otter/test/models/test_cass_models.py","position":103,"original_position":103,"commit_id":"a4577f6e1d7e7da6c7a395e0c66a9b6709450b38","original_commit_id":"a4577f6e1d7e7da6c7a395e0c66a9b6709450b38","user":{"login":"manishtomar","id":758529,"avatar_url":"https://avatars.githubusercontent.com/u/758529?v=3","gravatar_id":"","url":"https://api.github.com/users/manishtomar","html_url":"https://github.com/manishtomar","followers_url":"https://api.github.com/users/manishtomar/followers","following_url":"https://api.github.com/users/manishtomar/following{/other_user}","gists_url":"https://api.github.com/users/manishtomar/gists{/gist_id}","starred_url":"https://api.github.com/users/manishtomar/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/manishtomar/subscriptions","organizations_url":"https://api.github.com/users/manishtomar/orgs","repos_url":"https://api.github.com/users/manishtomar/repos","events_url":"https://api.github.com/users/manishtomar/events{/privacy}","received_events_url":"https://api.github.com/users/manishtomar/received_events","type":"User","site_admin":false},"body":"It's a stub for [`WeakLocks`](https://github.com/rackerlabs/otter/blob/master/otter/util/weaklocks.py)","created_at":"2016-08-20T00:32:27Z","updated_at":"2016-08-20T00:32:27Z","html_url":"https://github.com/rackerlabs/otter/pull/1927#discussion_r75568346","pull_request_url":"https://api.github.com/repos/rackerlabs/otter/pulls/1927","_links":{"self":{"href":"https://api.github.com/repos/rackerlabs/otter/pulls/comments/75568346"},"html":{"href":"https://github.com/rackerlabs/otter/pull/1927#discussion_r75568346"},"pull_request":{"href":"https://api.github.com/repos/rackerlabs/otter/pulls/1927"}}},"pull_request":{"url":"https://api.github.com/repos/rackerlabs/otter/pulls/1927","id":81562063,"html_url":"https://github.com/rackerlabs/otter/pull/1927","diff_url":"https://github.com/rackerlabs/otter/pull/1927.diff","patch_url":"https://github.com/rackerlabs/otter/pull/1927.patch","issue_url":"https://api.github.com/repos/rackerlabs/otter/issues/1927","number":1927,"state":"open","locked":false,"title":"Use polling lock when modifying state","user":{"login":"manishtomar","id":758529,"avatar_url":"https://avatars.githubusercontent.com/u/758529?v=3","gravatar_id":"","url":"https://api.github.com/users/manishtomar","html_url":"https://github.com/manishtomar","followers_url":"https://api.github.com/users/manishtomar/followers","following_url":"https://api.github.com/users/manishtomar/following{/other_user}","gists_url":"https://api.github.com/users/manishtomar/gists{/gist_id}","starred_url":"https://api.github.com/users/manishtomar/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/manishtomar/subscriptions","organizations_url":"https://api.github.com/users/manishtomar/orgs","repos_url":"https://api.github.com/users/manishtomar/repos","events_url":"https://api.github.com/users/manishtomar/events{/privacy}","received_events_url":"https://api.github.com/users/manishtomar/received_events","type":"User","site_admin":false},"body":"Using the `PollingLock` recipe implemented in #1925 when modifying group state and deleting group. This fixes #1915. Refactored tests to use `ZKLock` instead of fragile mocked lock object (thanks @glyph!; tracked in #1904). \r\n\r\nThis was tested on dev vm with 3 otter instances connected to single zk on the same vm load balanced via nginx talking. I had removed [local lock](https://github.com/rackerlabs/otter/blob/d15e6659436f6822bf6952d4e1751b31c3d66759/otter/models/cass.py#L827) in `modify_state` and directly used kazoo lock as I could not reproduce the issue with local lock. This is still a mystery to me. Following are test results:\r\n\r\n|Lock type|acquire timeout|later acquire timeout|webhook calls|locktimeouts|total errors|child nodes left|\r\n| ------ | ------ | ------ | ------ | ------ | ------ | ------ |\r\n||||||||\r\n|kazoo|20|30|100|22|100|11|\r\n|kazoo|20|30|200|18|200|24|\r\n|kazoo|20|300|200|167|200|1|\r\n|polling with 0.1 delay|20|25|100|0|0|0|\r\n|polling with 0.1 delay|20|25|200|93|93|0|\r\n|polling with 0.1 delay|20|25|200|17|17|0|\r\n|polling with 0.01 delay|20|25|200|108|108|0|\r\n|polling with 0.01 delay|20|25|200|109|109|0|\r\n\r\nAbove, *webhook calls* represent number of concurrent calls made. You will notice that `LockTimeout`s increases as the number of concurrent webhook calls increase. However, \"child nodes left\" is 0 when using `PollingLock` in any combination. This is important as this causes one lock failure to sustain in subsequent calls and is what `PollingLock` is trying to solve. ","created_at":"2016-08-16T23:28:54Z","updated_at":"2016-08-20T00:32:27Z","closed_at":null,"merged_at":null,"merge_commit_sha":"dd2cf38f3a79cbdca67719ea84d7963d5d91d003","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/rackerlabs/otter/pulls/1927/commits","review_comments_url":"https://api.github.com/repos/rackerlabs/otter/pulls/1927/comments","review_comment_url":"https://api.github.com/repos/rackerlabs/otter/pulls/comments{/number}","comments_url":"https://api.github.com/repos/rackerlabs/otter/issues/1927/comments","statuses_url":"https://api.github.com/repos/rackerlabs/otter/statuses/a4577f6e1d7e7da6c7a395e0c66a9b6709450b38","head":{"label":"rackerlabs:use-polling-lock","ref":"use-polling-lock","sha":"a4577f6e1d7e7da6c7a395e0c66a9b6709450b38","user":{"login":"rackerlabs","id":84293,"avatar_url":"https://avatars.githubusercontent.com/u/84293?v=3","gravatar_id":"","url":"https://api.github.com/users/rackerlabs","html_url":"https://github.com/rackerlabs","followers_url":"https://api.github.com/users/rackerlabs/followers","following_url":"https://api.github.com/users/rackerlabs/following{/other_user}","gists_url":"https://api.github.com/users/rackerlabs/gists{/gist_id}","starred_url":"https://api.github.com/users/rackerlabs/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/rackerlabs/subscriptions","organizations_url":"https://api.github.com/users/rackerlabs/orgs","repos_url":"https://api.github.com/users/rackerlabs/repos","events_url":"https://api.github.com/users/rackerlabs/events{/privacy}","received_events_url":"https://api.github.com/users/rackerlabs/received_events","type":"Organization","site_admin":false},"repo":{"id":6097139,"name":"otter","full_name":"rackerlabs/otter","owner":{"login":"rackerlabs","id":84293,"avatar_url":"https://avatars.githubusercontent.com/u/84293?v=3","gravatar_id":"","url":"https://api.github.com/users/rackerlabs","html_url":"https://github.com/rackerlabs","followers_url":"https://api.github.com/users/rackerlabs/followers","following_url":"https://api.github.com/users/rackerlabs/following{/other_user}","gists_url":"https://api.github.com/users/rackerlabs/gists{/gist_id}","starred_url":"https://api.github.com/users/rackerlabs/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/rackerlabs/subscriptions","organizations_url":"https://api.github.com/users/rackerlabs/orgs","repos_url":"https://api.github.com/users/rackerlabs/repos","events_url":"https://api.github.com/users/rackerlabs/events{/privacy}","received_events_url":"https://api.github.com/users/rackerlabs/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/rackerlabs/otter","description":"Rackspace Auto Scale","fork":false,"url":"https://api.github.com/repos/rackerlabs/otter","forks_url":"https://api.github.com/repos/rackerlabs/otter/forks","keys_url":"https://api.github.com/repos/rackerlabs/otter/keys{/key_id}","collaborators_url":"https://api.github.com/repos/rackerlabs/otter/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/rackerlabs/otter/teams","hooks_url":"https://api.github.com/repos/rackerlabs/otter/hooks","issue_events_url":"https://api.github.com/repos/rackerlabs/otter/issues/events{/number}","events_url":"https://api.github.com/repos/rackerlabs/otter/events","assignees_url":"https://api.github.com/repos/rackerlabs/otter/assignees{/user}","branches_url":"https://api.github.com/repos/rackerlabs/otter/branches{/branch}","tags_url":"https://api.github.com/repos/rackerlabs/otter/tags","blobs_url":"https://api.github.com/repos/rackerlabs/otter/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/rackerlabs/otter/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/rackerlabs/otter/git/refs{/sha}","trees_url":"https://api.github.com/repos/rackerlabs/otter/git/trees{/sha}","statuses_url":"https://api.github.com/repos/rackerlabs/otter/statuses/{sha}","languages_url":"https://api.github.com/repos/rackerlabs/otter/languages","stargazers_url":"https://api.github.com/repos/rackerlabs/otter/stargazers","contributors_url":"https://api.github.com/repos/rackerlabs/otter/contributors","subscribers_url":"https://api.github.com/repos/rackerlabs/otter/subscribers","subscription_url":"https://api.github.com/repos/rackerlabs/otter/subscription","commits_url":"https://api.github.com/repos/rackerlabs/otter/commits{/sha}","git_commits_url":"https://api.github.com/repos/rackerlabs/otter/git/commits{/sha}","comments_url":"https://api.github.com/repos/rackerlabs/otter/comments{/number}","issue_comment_url":"https://api.github.com/repos/rackerlabs/otter/issues/comments{/number}","contents_url":"https://api.github.com/repos/rackerlabs/otter/contents/{+path}","compare_url":"https://api.github.com/repos/rackerlabs/otter/compare/{base}...{head}","merges_url":"https://api.github.com/repos/rackerlabs/otter/merges","archive_url":"https://api.github.com/repos/rackerlabs/otter/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/rackerlabs/otter/downloads","issues_url":"https://api.github.com/repos/rackerlabs/otter/issues{/number}","pulls_url":"https://api.github.com/repos/rackerlabs/otter/pulls{/number}","milestones_url":"https://api.github.com/repos/rackerlabs/otter/milestones{/number}","notifications_url":"https://api.github.com/repos/rackerlabs/otter/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/rackerlabs/otter/labels{/name}","releases_url":"https://api.github.com/repos/rackerlabs/otter/releases{/id}","deployments_url":"https://api.github.com/repos/rackerlabs/otter/deployments","created_at":"2012-10-05T21:35:59Z","updated_at":"2016-04-20T16:06:24Z","pushed_at":"2016-08-16T23:28:54Z","git_url":"git://github.com/rackerlabs/otter.git","ssh_url":"git@github.com:rackerlabs/otter.git","clone_url":"https://github.com/rackerlabs/otter.git","svn_url":"https://github.com/rackerlabs/otter","homepage":"http://www.rackspace.com/cloud/auto-scale/","size":35067,"stargazers_count":44,"watchers_count":44,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":21,"mirror_url":null,"open_issues_count":111,"forks":21,"open_issues":111,"watchers":44,"default_branch":"master"}},"base":{"label":"rackerlabs:master","ref":"master","sha":"d15e6659436f6822bf6952d4e1751b31c3d66759","user":{"login":"rackerlabs","id":84293,"avatar_url":"https://avatars.githubusercontent.com/u/84293?v=3","gravatar_id":"","url":"https://api.github.com/users/rackerlabs","html_url":"https://github.com/rackerlabs","followers_url":"https://api.github.com/users/rackerlabs/followers","following_url":"https://api.github.com/users/rackerlabs/following{/other_user}","gists_url":"https://api.github.com/users/rackerlabs/gists{/gist_id}","starred_url":"https://api.github.com/users/rackerlabs/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/rackerlabs/subscriptions","organizations_url":"https://api.github.com/users/rackerlabs/orgs","repos_url":"https://api.github.com/users/rackerlabs/repos","events_url":"https://api.github.com/users/rackerlabs/events{/privacy}","received_events_url":"https://api.github.com/users/rackerlabs/received_events","type":"Organization","site_admin":false},"repo":{"id":6097139,"name":"otter","full_name":"rackerlabs/otter","owner":{"login":"rackerlabs","id":84293,"avatar_url":"https://avatars.githubusercontent.com/u/84293?v=3","gravatar_id":"","url":"https://api.github.com/users/rackerlabs","html_url":"https://github.com/rackerlabs","followers_url":"https://api.github.com/users/rackerlabs/followers","following_url":"https://api.github.com/users/rackerlabs/following{/other_user}","gists_url":"https://api.github.com/users/rackerlabs/gists{/gist_id}","starred_url":"https://api.github.com/users/rackerlabs/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/rackerlabs/subscriptions","organizations_url":"https://api.github.com/users/rackerlabs/orgs","repos_url":"https://api.github.com/users/rackerlabs/repos","events_url":"https://api.github.com/users/rackerlabs/events{/privacy}","received_events_url":"https://api.github.com/users/rackerlabs/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/rackerlabs/otter","description":"Rackspace Auto Scale","fork":false,"url":"https://api.github.com/repos/rackerlabs/otter","forks_url":"https://api.github.com/repos/rackerlabs/otter/forks","keys_url":"https://api.github.com/repos/rackerlabs/otter/keys{/key_id}","collaborators_url":"https://api.github.com/repos/rackerlabs/otter/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/rackerlabs/otter/teams","hooks_url":"https://api.github.com/repos/rackerlabs/otter/hooks","issue_events_url":"https://api.github.com/repos/rackerlabs/otter/issues/events{/number}","events_url":"https://api.github.com/repos/rackerlabs/otter/events","assignees_url":"https://api.github.com/repos/rackerlabs/otter/assignees{/user}","branches_url":"https://api.github.com/repos/rackerlabs/otter/branches{/branch}","tags_url":"https://api.github.com/repos/rackerlabs/otter/tags","blobs_url":"https://api.github.com/repos/rackerlabs/otter/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/rackerlabs/otter/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/rackerlabs/otter/git/refs{/sha}","trees_url":"https://api.github.com/repos/rackerlabs/otter/git/trees{/sha}","statuses_url":"https://api.github.com/repos/rackerlabs/otter/statuses/{sha}","languages_url":"https://api.github.com/repos/rackerlabs/otter/languages","stargazers_url":"https://api.github.com/repos/rackerlabs/otter/stargazers","contributors_url":"https://api.github.com/repos/rackerlabs/otter/contributors","subscribers_url":"https://api.github.com/repos/rackerlabs/otter/subscribers","subscription_url":"https://api.github.com/repos/rackerlabs/otter/subscription","commits_url":"https://api.github.com/repos/rackerlabs/otter/commits{/sha}","git_commits_url":"https://api.github.com/repos/rackerlabs/otter/git/commits{/sha}","comments_url":"https://api.github.com/repos/rackerlabs/otter/comments{/number}","issue_comment_url":"https://api.github.com/repos/rackerlabs/otter/issues/comments{/number}","contents_url":"https://api.github.com/repos/rackerlabs/otter/contents/{+path}","compare_url":"https://api.github.com/repos/rackerlabs/otter/compare/{base}...{head}","merges_url":"https://api.github.com/repos/rackerlabs/otter/merges","archive_url":"https://api.github.com/repos/rackerlabs/otter/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/rackerlabs/otter/downloads","issues_url":"https://api.github.com/repos/rackerlabs/otter/issues{/number}","pulls_url":"https://api.github.com/repos/rackerlabs/otter/pulls{/number}","milestones_url":"https://api.github.com/repos/rackerlabs/otter/milestones{/number}","notifications_url":"https://api.github.com/repos/rackerlabs/otter/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/rackerlabs/otter/labels{/name}","releases_url":"https://api.github.com/repos/rackerlabs/otter/releases{/id}","deployments_url":"https://api.github.com/repos/rackerlabs/otter/deployments","created_at":"2012-10-05T21:35:59Z","updated_at":"2016-04-20T16:06:24Z","pushed_at":"2016-08-16T23:28:54Z","git_url":"git://github.com/rackerlabs/otter.git","ssh_url":"git@github.com:rackerlabs/otter.git","clone_url":"https://github.com/rackerlabs/otter.git","svn_url":"https://github.com/rackerlabs/otter","homepage":"http://www.rackspace.com/cloud/auto-scale/","size":35067,"stargazers_count":44,"watchers_count":44,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":21,"mirror_url":null,"open_issues_count":111,"forks":21,"open_issues":111,"watchers":44,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/rackerlabs/otter/pulls/1927"},"html":{"href":"https://github.com/rackerlabs/otter/pull/1927"},"issue":{"href":"https://api.github.com/repos/rackerlabs/otter/issues/1927"},"comments":{"href":"https://api.github.com/repos/rackerlabs/otter/issues/1927/comments"},"review_comments":{"href":"https://api.github.com/repos/rackerlabs/otter/pulls/1927/comments"},"review_comment":{"href":"https://api.github.com/repos/rackerlabs/otter/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/rackerlabs/otter/pulls/1927/commits"},"statuses":{"href":"https://api.github.com/repos/rackerlabs/otter/statuses/a4577f6e1d7e7da6c7a395e0c66a9b6709450b38"}}}}
{"actor":{"display_login":"manishtomar"}}