{"action":"created","comment":{"url":"https://api.github.com/repos/dimagi/commcare-hq/pulls/comments/74423254","id":74423254,"diff_hunk":"@@ -0,0 +1,64 @@\n+# -*- coding: utf-8 -*-\n+from __future__ import unicode_literals\n+\n+from django.db import models, migrations\n+from corehq.sql_db.operations import HqRunPython\n+\n+\n+RENAMES = (\n+    (\"applications-to-elasticsearch\", \"ApplicationToElasticsearchPillow-hqapps_2016-07-11_2128\"),\n+    (\"all-cases-to-elasticsearch\", \"CaseToElasticsearchPillow-hqcases_2016-03-04\"),\n+    (\"all-xforms-to-elasticsearch\", \"XFormToElasticsearchPillow-xforms_2016-07-07\"),\n+    (\"case-search-to-elasticsearch\", \"CaseSearchToElasticsearchPillow-case_search_2016-03-15\"),\n+    (\"GroupPillow\", \"GroupPillow-hqgroups_20150403_1501\"),\n+    (\"GroupToUserPillow\", \"GroupToUserPillow-hqusers_2016-07-19\"),\n+    (\"KafkaDomainPillow\", \"KafkaDomainPillow-hqdomains_2016-08-08\"),\n+    (\"ledger-to-elasticsearch\", \"LedgerToElasticsearchPillow-ledgers_2016-03-15\"),\n+    (\"report-cases-to-elasticsearch\", \"ReportCaseToElasticsearchPillow-report_cases_czei39du507m9mmpqk3y01x72a3ux4p0\"),\n+    (\"report-xforms-to-elasticsearch\", \"ReportXFormToElasticsearchPillow-report_xforms_20160707_2322\"),\n+    (\"sql-sms-to-es\", \"SqlSMSPillow-smslogs_708c77f8e5fe00286fa5791e9fa7d45f\"),\n+    (\"UnknownUsersPillow\", \"UnknownUsersPillow-hqusers_2016-07-19\"),\n+    (\"UserPillow\", \"UserPillow-hqusers_2016-07-19\"),\n+)\n+\n+\n+def copy_checkpoints(apps, schema_editor):\n+    from pillowtop import get_all_pillow_instances\n+    DjangoPillowCheckpoint = apps.get_model('pillowtop', 'DjangoPillowCheckpoint')\n+    current_checkoint_ids = set(\n+        [pillow_instance.checkpoint.checkpoint_id for pillow_instance in get_all_pillow_instances()]\n+    )\n+\n+    for old_id, new_id in RENAMES:\n+        try:\n+            checkpoint = DjangoPillowCheckpoint.objects.get(checkpoint_id=old_id)\n+            # since checkpoint_id is the primary key, this should make a new model\n+            # which is good in case we need to rollback\n+            checkpoint.checkpoint_id = new_id\n+            checkpoint.save()\n+            assert new_id in current_checkoint_ids\n+        except DjangoPillowCheckpoint.DoesNotExist:\n+            print 'warning: legacy pillow checkpoint with ID {} not found'.format(old_id)\n+        except AssertionError:\n+            print 'warning: no current pillow found with checkpoint ID {}'.format(new_id)\n+\n+\n+def delete_checkpoints(apps, schema_editor):\n+    DjangoPillowCheckpoint = apps.get_model('pillowtop', 'DjangoPillowCheckpoint')\n+    for _, new_id in RENAMES:","path":"corehq/apps/cleanup/migrations/0012_add_es_index_to_checkpoint_ids.py","position":48,"original_position":48,"commit_id":"5a28177c63ce074e566513ee8f50b96c7fa1f39d","original_commit_id":"2d9a69b6f2852d31d427a3198ce57656a597d383","user":{"login":"czue","id":66555,"avatar_url":"https://avatars.githubusercontent.com/u/66555?v=3","gravatar_id":"","url":"https://api.github.com/users/czue","html_url":"https://github.com/czue","followers_url":"https://api.github.com/users/czue/followers","following_url":"https://api.github.com/users/czue/following{/other_user}","gists_url":"https://api.github.com/users/czue/gists{/gist_id}","starred_url":"https://api.github.com/users/czue/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/czue/subscriptions","organizations_url":"https://api.github.com/users/czue/orgs","repos_url":"https://api.github.com/users/czue/repos","events_url":"https://api.github.com/users/czue/events{/privacy}","received_events_url":"https://api.github.com/users/czue/received_events","type":"User","site_admin":false},"body":"it's updated below","created_at":"2016-08-11T13:40:30Z","updated_at":"2016-08-11T13:40:31Z","html_url":"https://github.com/dimagi/commcare-hq/pull/12774#discussion_r74423254","pull_request_url":"https://api.github.com/repos/dimagi/commcare-hq/pulls/12774","_links":{"self":{"href":"https://api.github.com/repos/dimagi/commcare-hq/pulls/comments/74423254"},"html":{"href":"https://github.com/dimagi/commcare-hq/pull/12774#discussion_r74423254"},"pull_request":{"href":"https://api.github.com/repos/dimagi/commcare-hq/pulls/12774"}}},"pull_request":{"url":"https://api.github.com/repos/dimagi/commcare-hq/pulls/12774","id":80838139,"html_url":"https://github.com/dimagi/commcare-hq/pull/12774","diff_url":"https://github.com/dimagi/commcare-hq/pull/12774.diff","patch_url":"https://github.com/dimagi/commcare-hq/pull/12774.patch","issue_url":"https://api.github.com/repos/dimagi/commcare-hq/issues/12774","number":12774,"state":"closed","locked":false,"title":"fix the reindexing gap","user":{"login":"czue","id":66555,"avatar_url":"https://avatars.githubusercontent.com/u/66555?v=3","gravatar_id":"","url":"https://api.github.com/users/czue","html_url":"https://github.com/czue","followers_url":"https://api.github.com/users/czue/followers","following_url":"https://api.github.com/users/czue/following{/other_user}","gists_url":"https://api.github.com/users/czue/gists{/gist_id}","starred_url":"https://api.github.com/users/czue/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/czue/subscriptions","organizations_url":"https://api.github.com/users/czue/orgs","repos_url":"https://api.github.com/users/czue/repos","events_url":"https://api.github.com/users/czue/events{/privacy}","received_events_url":"https://api.github.com/users/czue/received_events","type":"User","site_admin":false},"body":"this does two main things:\r\n\r\n1. ties elastic pillow checkpoint IDs to the elastic indices (and migrates the current checkpoints)\r\n2. sets a checkpoint on reindex for the first pillow that touches the index\r\n\r\ni'm not totally happy with this. in particular it feels pretty hacky that, e.g. the 'form' reindexer sets the checkpoint and the 'form-sql' doesn't (i think this is currently necessary due to the fact that they will be run in serial and therefore by the time 'form-sql' runs it would be setting a checkpoint that is now too late and would cause us to miss some couch forms that have come in while the 'form' reindexer was running). I think the better thing to do would have the notion of a \"reindexer-group\" or something, but I haven't done that yet.\r\n\r\ni also want to clean up how we're using pillow_id a little, and do a bit more testing.\r\n\r\n@snopoke @emord cc @benrudolph ","created_at":"2016-08-10T17:53:42Z","updated_at":"2016-08-11T13:40:31Z","closed_at":"2016-08-11T11:11:27Z","merged_at":"2016-08-11T11:11:27Z","merge_commit_sha":"28e9d1bd1d7f7410db9b85efa89a9491053cad37","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/dimagi/commcare-hq/pulls/12774/commits","review_comments_url":"https://api.github.com/repos/dimagi/commcare-hq/pulls/12774/comments","review_comment_url":"https://api.github.com/repos/dimagi/commcare-hq/pulls/comments{/number}","comments_url":"https://api.github.com/repos/dimagi/commcare-hq/issues/12774/comments","statuses_url":"https://api.github.com/repos/dimagi/commcare-hq/statuses/5a28177c63ce074e566513ee8f50b96c7fa1f39d","head":{"label":"dimagi:cz/reindexer-resiliency","ref":"cz/reindexer-resiliency","sha":"5a28177c63ce074e566513ee8f50b96c7fa1f39d","user":{"login":"dimagi","id":94514,"avatar_url":"https://avatars.githubusercontent.com/u/94514?v=3","gravatar_id":"","url":"https://api.github.com/users/dimagi","html_url":"https://github.com/dimagi","followers_url":"https://api.github.com/users/dimagi/followers","following_url":"https://api.github.com/users/dimagi/following{/other_user}","gists_url":"https://api.github.com/users/dimagi/gists{/gist_id}","starred_url":"https://api.github.com/users/dimagi/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dimagi/subscriptions","organizations_url":"https://api.github.com/users/dimagi/orgs","repos_url":"https://api.github.com/users/dimagi/repos","events_url":"https://api.github.com/users/dimagi/events{/privacy}","received_events_url":"https://api.github.com/users/dimagi/received_events","type":"Organization","site_admin":false},"repo":{"id":247278,"name":"commcare-hq","full_name":"dimagi/commcare-hq","owner":{"login":"dimagi","id":94514,"avatar_url":"https://avatars.githubusercontent.com/u/94514?v=3","gravatar_id":"","url":"https://api.github.com/users/dimagi","html_url":"https://github.com/dimagi","followers_url":"https://api.github.com/users/dimagi/followers","following_url":"https://api.github.com/users/dimagi/following{/other_user}","gists_url":"https://api.github.com/users/dimagi/gists{/gist_id}","starred_url":"https://api.github.com/users/dimagi/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dimagi/subscriptions","organizations_url":"https://api.github.com/users/dimagi/orgs","repos_url":"https://api.github.com/users/dimagi/repos","events_url":"https://api.github.com/users/dimagi/events{/privacy}","received_events_url":"https://api.github.com/users/dimagi/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/dimagi/commcare-hq","description":"A server-side tool to help manage community health workers, written in Django","fork":false,"url":"https://api.github.com/repos/dimagi/commcare-hq","forks_url":"https://api.github.com/repos/dimagi/commcare-hq/forks","keys_url":"https://api.github.com/repos/dimagi/commcare-hq/keys{/key_id}","collaborators_url":"https://api.github.com/repos/dimagi/commcare-hq/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/dimagi/commcare-hq/teams","hooks_url":"https://api.github.com/repos/dimagi/commcare-hq/hooks","issue_events_url":"https://api.github.com/repos/dimagi/commcare-hq/issues/events{/number}","events_url":"https://api.github.com/repos/dimagi/commcare-hq/events","assignees_url":"https://api.github.com/repos/dimagi/commcare-hq/assignees{/user}","branches_url":"https://api.github.com/repos/dimagi/commcare-hq/branches{/branch}","tags_url":"https://api.github.com/repos/dimagi/commcare-hq/tags","blobs_url":"https://api.github.com/repos/dimagi/commcare-hq/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/dimagi/commcare-hq/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/dimagi/commcare-hq/git/refs{/sha}","trees_url":"https://api.github.com/repos/dimagi/commcare-hq/git/trees{/sha}","statuses_url":"https://api.github.com/repos/dimagi/commcare-hq/statuses/{sha}","languages_url":"https://api.github.com/repos/dimagi/commcare-hq/languages","stargazers_url":"https://api.github.com/repos/dimagi/commcare-hq/stargazers","contributors_url":"https://api.github.com/repos/dimagi/commcare-hq/contributors","subscribers_url":"https://api.github.com/repos/dimagi/commcare-hq/subscribers","subscription_url":"https://api.github.com/repos/dimagi/commcare-hq/subscription","commits_url":"https://api.github.com/repos/dimagi/commcare-hq/commits{/sha}","git_commits_url":"https://api.github.com/repos/dimagi/commcare-hq/git/commits{/sha}","comments_url":"https://api.github.com/repos/dimagi/commcare-hq/comments{/number}","issue_comment_url":"https://api.github.com/repos/dimagi/commcare-hq/issues/comments{/number}","contents_url":"https://api.github.com/repos/dimagi/commcare-hq/contents/{+path}","compare_url":"https://api.github.com/repos/dimagi/commcare-hq/compare/{base}...{head}","merges_url":"https://api.github.com/repos/dimagi/commcare-hq/merges","archive_url":"https://api.github.com/repos/dimagi/commcare-hq/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/dimagi/commcare-hq/downloads","issues_url":"https://api.github.com/repos/dimagi/commcare-hq/issues{/number}","pulls_url":"https://api.github.com/repos/dimagi/commcare-hq/pulls{/number}","milestones_url":"https://api.github.com/repos/dimagi/commcare-hq/milestones{/number}","notifications_url":"https://api.github.com/repos/dimagi/commcare-hq/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/dimagi/commcare-hq/labels{/name}","releases_url":"https://api.github.com/repos/dimagi/commcare-hq/releases{/id}","deployments_url":"https://api.github.com/repos/dimagi/commcare-hq/deployments","created_at":"2009-07-09T17:00:07Z","updated_at":"2016-05-16T15:23:03Z","pushed_at":"2016-08-11T13:00:29Z","git_url":"git://github.com/dimagi/commcare-hq.git","ssh_url":"git@github.com:dimagi/commcare-hq.git","clone_url":"https://github.com/dimagi/commcare-hq.git","svn_url":"https://github.com/dimagi/commcare-hq","homepage":"http://www.commcarehq.org","size":317078,"stargazers_count":72,"watchers_count":72,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":true,"forks_count":70,"mirror_url":null,"open_issues_count":51,"forks":70,"open_issues":51,"watchers":72,"default_branch":"master"}},"base":{"label":"dimagi:master","ref":"master","sha":"b4c168d521bb1d8dd5b7ce7077575a701b3cb2ed","user":{"login":"dimagi","id":94514,"avatar_url":"https://avatars.githubusercontent.com/u/94514?v=3","gravatar_id":"","url":"https://api.github.com/users/dimagi","html_url":"https://github.com/dimagi","followers_url":"https://api.github.com/users/dimagi/followers","following_url":"https://api.github.com/users/dimagi/following{/other_user}","gists_url":"https://api.github.com/users/dimagi/gists{/gist_id}","starred_url":"https://api.github.com/users/dimagi/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dimagi/subscriptions","organizations_url":"https://api.github.com/users/dimagi/orgs","repos_url":"https://api.github.com/users/dimagi/repos","events_url":"https://api.github.com/users/dimagi/events{/privacy}","received_events_url":"https://api.github.com/users/dimagi/received_events","type":"Organization","site_admin":false},"repo":{"id":247278,"name":"commcare-hq","full_name":"dimagi/commcare-hq","owner":{"login":"dimagi","id":94514,"avatar_url":"https://avatars.githubusercontent.com/u/94514?v=3","gravatar_id":"","url":"https://api.github.com/users/dimagi","html_url":"https://github.com/dimagi","followers_url":"https://api.github.com/users/dimagi/followers","following_url":"https://api.github.com/users/dimagi/following{/other_user}","gists_url":"https://api.github.com/users/dimagi/gists{/gist_id}","starred_url":"https://api.github.com/users/dimagi/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dimagi/subscriptions","organizations_url":"https://api.github.com/users/dimagi/orgs","repos_url":"https://api.github.com/users/dimagi/repos","events_url":"https://api.github.com/users/dimagi/events{/privacy}","received_events_url":"https://api.github.com/users/dimagi/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/dimagi/commcare-hq","description":"A server-side tool to help manage community health workers, written in Django","fork":false,"url":"https://api.github.com/repos/dimagi/commcare-hq","forks_url":"https://api.github.com/repos/dimagi/commcare-hq/forks","keys_url":"https://api.github.com/repos/dimagi/commcare-hq/keys{/key_id}","collaborators_url":"https://api.github.com/repos/dimagi/commcare-hq/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/dimagi/commcare-hq/teams","hooks_url":"https://api.github.com/repos/dimagi/commcare-hq/hooks","issue_events_url":"https://api.github.com/repos/dimagi/commcare-hq/issues/events{/number}","events_url":"https://api.github.com/repos/dimagi/commcare-hq/events","assignees_url":"https://api.github.com/repos/dimagi/commcare-hq/assignees{/user}","branches_url":"https://api.github.com/repos/dimagi/commcare-hq/branches{/branch}","tags_url":"https://api.github.com/repos/dimagi/commcare-hq/tags","blobs_url":"https://api.github.com/repos/dimagi/commcare-hq/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/dimagi/commcare-hq/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/dimagi/commcare-hq/git/refs{/sha}","trees_url":"https://api.github.com/repos/dimagi/commcare-hq/git/trees{/sha}","statuses_url":"https://api.github.com/repos/dimagi/commcare-hq/statuses/{sha}","languages_url":"https://api.github.com/repos/dimagi/commcare-hq/languages","stargazers_url":"https://api.github.com/repos/dimagi/commcare-hq/stargazers","contributors_url":"https://api.github.com/repos/dimagi/commcare-hq/contributors","subscribers_url":"https://api.github.com/repos/dimagi/commcare-hq/subscribers","subscription_url":"https://api.github.com/repos/dimagi/commcare-hq/subscription","commits_url":"https://api.github.com/repos/dimagi/commcare-hq/commits{/sha}","git_commits_url":"https://api.github.com/repos/dimagi/commcare-hq/git/commits{/sha}","comments_url":"https://api.github.com/repos/dimagi/commcare-hq/comments{/number}","issue_comment_url":"https://api.github.com/repos/dimagi/commcare-hq/issues/comments{/number}","contents_url":"https://api.github.com/repos/dimagi/commcare-hq/contents/{+path}","compare_url":"https://api.github.com/repos/dimagi/commcare-hq/compare/{base}...{head}","merges_url":"https://api.github.com/repos/dimagi/commcare-hq/merges","archive_url":"https://api.github.com/repos/dimagi/commcare-hq/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/dimagi/commcare-hq/downloads","issues_url":"https://api.github.com/repos/dimagi/commcare-hq/issues{/number}","pulls_url":"https://api.github.com/repos/dimagi/commcare-hq/pulls{/number}","milestones_url":"https://api.github.com/repos/dimagi/commcare-hq/milestones{/number}","notifications_url":"https://api.github.com/repos/dimagi/commcare-hq/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/dimagi/commcare-hq/labels{/name}","releases_url":"https://api.github.com/repos/dimagi/commcare-hq/releases{/id}","deployments_url":"https://api.github.com/repos/dimagi/commcare-hq/deployments","created_at":"2009-07-09T17:00:07Z","updated_at":"2016-05-16T15:23:03Z","pushed_at":"2016-08-11T13:00:29Z","git_url":"git://github.com/dimagi/commcare-hq.git","ssh_url":"git@github.com:dimagi/commcare-hq.git","clone_url":"https://github.com/dimagi/commcare-hq.git","svn_url":"https://github.com/dimagi/commcare-hq","homepage":"http://www.commcarehq.org","size":317078,"stargazers_count":72,"watchers_count":72,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":true,"forks_count":70,"mirror_url":null,"open_issues_count":51,"forks":70,"open_issues":51,"watchers":72,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/dimagi/commcare-hq/pulls/12774"},"html":{"href":"https://github.com/dimagi/commcare-hq/pull/12774"},"issue":{"href":"https://api.github.com/repos/dimagi/commcare-hq/issues/12774"},"comments":{"href":"https://api.github.com/repos/dimagi/commcare-hq/issues/12774/comments"},"review_comments":{"href":"https://api.github.com/repos/dimagi/commcare-hq/pulls/12774/comments"},"review_comment":{"href":"https://api.github.com/repos/dimagi/commcare-hq/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/dimagi/commcare-hq/pulls/12774/commits"},"statuses":{"href":"https://api.github.com/repos/dimagi/commcare-hq/statuses/5a28177c63ce074e566513ee8f50b96c7fa1f39d"}}}}
{"actor":{"display_login":"czue"}}