{"action":"created","comment":{"url":"https://api.github.com/repos/medic/medic-analytics/pulls/comments/69397976","id":69397976,"diff_hunk":"@@ -11,50 +11,101 @@ var couch2pg = require('./libs/couch2pg/importer')(\n       db, couchdb,\n       env.couch2pgDocLimit,\n       env.couch2pgChangesLimit),\n-    xmlforms = require('./libs/xmlforms/extractor')(db);\n-\n-var batchLoop = function() {\n-  var completeBatch = function(lastSeq) {\n-    return couch2pg.storeSeq(lastSeq)\n-      .then(function() {\n-        log.info('Batch completed up to ' + lastSeq);\n-      });\n-  };\n-\n-  log.info('Beginning couch2pg and xmlforms batch');\n-\n-  return couch2pg.importBatch()\n-  .then(function(changes) {\n-    var allDocs = changes.deleted.concat(changes.edited);\n-    log.debug('There are ' + allDocs.length + ' changes');\n-\n-    if (allDocs.length > 0) {\n-      return xmlforms.extract(allDocs)\n-      .then(function() {\n-        return completeBatch(changes.lastSeq);\n-      })\n-      .then(batchLoop);\n+    xmlforms = require('./libs/xmlforms/updater')(db);\n+\n+var fallback = 0;\n+var sleepMs = function(errored) {\n+  if (errored) {\n+    var fallbackMs = fallback * 1000 * 60;\n+    if (fallbackMs < env.sleepMs) {\n+      fallback++;\n+      return fallbackMs;\n+    } else {\n+      return env.sleepMs;\n+    }\n+  } else {\n+    fallback = 0;\n+    return env.sleepMs;\n+  }\n+};\n+\n+var migrateCouch2pg = function() {\n+  return couch2pgMigrator(env.postgresqlUrl)();\n+};\n+var migrateXmlforms = function() {\n+  return xmlformsMigrator(env.postgresqlUrl)();\n+};\n+\n+var delayLoop = function(loopFn, errored) {\n+  return new Promise(function(resolve) {\n+    var ms = sleepMs(errored);\n+    log.info('Run '+(errored ? 'errored' : 'complete') + '. Next run at ' + new Date(new Date().getTime() + ms));\n+    if (ms === 0) {\n+      resolve();\n     } else {\n-      return completeBatch(changes.lastSeq);\n+      setInterval(resolve, ms);\n     }\n   });\n };\n \n var run = function() {\n-  log.info('Beginning couch2pg and xmlforms run');\n+  log.info('Beginning couch2pg and xmlforms run at ' + new Date());\n \n-  return batchLoop()\n+  return couch2pg.importAll()\n   .catch(function(err) {\n+    log.error('Couch2PG import failed');\n     log.error(err.stack);\n   })\n-  .then(function() {\n-    log.info('Run complete. Next run at ' + new Date(new Date().getTime() + env.sleepMs));\n-    setInterval(run, env.sleepMs);\n-  });\n+  .then(function(results) {\n+    if ((results.deleted.length + results.edited.length) > 0) {\n+      return xmlforms.update();\n+    }\n+  })\n+  .then(\n+    function() {\n+      return delayLoop(run);\n+    },\n+    function(err) {\n+      log.error('XMLForms support failed');\n+      log.error(err.stack);\n+      return delayLoop(run, true);\n+    })","path":"index.js","position":100,"original_position":100,"commit_id":"f9f3d1cc3d9db72509c47b765d518681f077fa6f","original_commit_id":"f9f3d1cc3d9db72509c47b765d518681f077fa6f","user":{"login":"garethbowen","id":2064938,"avatar_url":"https://avatars.githubusercontent.com/u/2064938?v=3","gravatar_id":"","url":"https://api.github.com/users/garethbowen","html_url":"https://github.com/garethbowen","followers_url":"https://api.github.com/users/garethbowen/followers","following_url":"https://api.github.com/users/garethbowen/following{/other_user}","gists_url":"https://api.github.com/users/garethbowen/gists{/gist_id}","starred_url":"https://api.github.com/users/garethbowen/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/garethbowen/subscriptions","organizations_url":"https://api.github.com/users/garethbowen/orgs","repos_url":"https://api.github.com/users/garethbowen/repos","events_url":"https://api.github.com/users/garethbowen/events{/privacy}","received_events_url":"https://api.github.com/users/garethbowen/received_events","type":"User","site_admin":false},"body":"Usually we use the `.catch` form, rather than passing in two functions to the `then`. Any reason you chose this form this time?","created_at":"2016-07-03T22:04:19Z","updated_at":"2016-07-03T22:04:19Z","html_url":"https://github.com/medic/medic-analytics/pull/33#discussion_r69397976","pull_request_url":"https://api.github.com/repos/medic/medic-analytics/pulls/33","_links":{"self":{"href":"https://api.github.com/repos/medic/medic-analytics/pulls/comments/69397976"},"html":{"href":"https://github.com/medic/medic-analytics/pull/33#discussion_r69397976"},"pull_request":{"href":"https://api.github.com/repos/medic/medic-analytics/pulls/33"}}},"pull_request":{"url":"https://api.github.com/repos/medic/medic-analytics/pulls/33","id":75484603,"html_url":"https://github.com/medic/medic-analytics/pull/33","diff_url":"https://github.com/medic/medic-analytics/pull/33.diff","patch_url":"https://github.com/medic/medic-analytics/pull/33.patch","issue_url":"https://api.github.com/repos/medic/medic-analytics/issues/33","number":33,"state":"open","locked":false,"title":"Remove XML parsing medic/medic-webapp#2432","user":{"login":"SCdF","id":583851,"avatar_url":"https://avatars.githubusercontent.com/u/583851?v=3","gravatar_id":"","url":"https://api.github.com/users/SCdF","html_url":"https://github.com/SCdF","followers_url":"https://api.github.com/users/SCdF/followers","following_url":"https://api.github.com/users/SCdF/following{/other_user}","gists_url":"https://api.github.com/users/SCdF/gists{/gist_id}","starred_url":"https://api.github.com/users/SCdF/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/SCdF/subscriptions","organizations_url":"https://api.github.com/users/SCdF/orgs","repos_url":"https://api.github.com/users/SCdF/repos","events_url":"https://api.github.com/users/SCdF/events{/privacy}","received_events_url":"https://api.github.com/users/SCdF/received_events","type":"User","site_admin":false},"body":"Removes all XML parsing. Because there is no XML parsing we no longer\r\nneed to pass the changes from couch2pg (and couch2pg no longer needs\r\nto generate them, but we do it anyway for now). Because we no longer\r\nneed to pass changes we don't need to worry about losing data, so we\r\nrun importAll() now instead.\r\n\r\nAll xmlforms does now is make sure the helper SQL exists (views and\r\nthe like) and that at the end of each run any materialised views that\r\nexist are updated.","created_at":"2016-06-28T14:14:40Z","updated_at":"2016-07-03T22:04:19Z","closed_at":null,"merged_at":null,"merge_commit_sha":"5a866c8b99b70d2c8be9b530efbd3f97325e379a","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/medic/medic-analytics/pulls/33/commits","review_comments_url":"https://api.github.com/repos/medic/medic-analytics/pulls/33/comments","review_comment_url":"https://api.github.com/repos/medic/medic-analytics/pulls/comments{/number}","comments_url":"https://api.github.com/repos/medic/medic-analytics/issues/33/comments","statuses_url":"https://api.github.com/repos/medic/medic-analytics/statuses/f9f3d1cc3d9db72509c47b765d518681f077fa6f","head":{"label":"medic:2432_no_more_xml","ref":"2432_no_more_xml","sha":"f9f3d1cc3d9db72509c47b765d518681f077fa6f","user":{"login":"medic","id":474424,"avatar_url":"https://avatars.githubusercontent.com/u/474424?v=3","gravatar_id":"","url":"https://api.github.com/users/medic","html_url":"https://github.com/medic","followers_url":"https://api.github.com/users/medic/followers","following_url":"https://api.github.com/users/medic/following{/other_user}","gists_url":"https://api.github.com/users/medic/gists{/gist_id}","starred_url":"https://api.github.com/users/medic/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/medic/subscriptions","organizations_url":"https://api.github.com/users/medic/orgs","repos_url":"https://api.github.com/users/medic/repos","events_url":"https://api.github.com/users/medic/events{/privacy}","received_events_url":"https://api.github.com/users/medic/received_events","type":"Organization","site_admin":false},"repo":{"id":38839516,"name":"medic-analytics","full_name":"medic/medic-analytics","owner":{"login":"medic","id":474424,"avatar_url":"https://avatars.githubusercontent.com/u/474424?v=3","gravatar_id":"","url":"https://api.github.com/users/medic","html_url":"https://github.com/medic","followers_url":"https://api.github.com/users/medic/followers","following_url":"https://api.github.com/users/medic/following{/other_user}","gists_url":"https://api.github.com/users/medic/gists{/gist_id}","starred_url":"https://api.github.com/users/medic/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/medic/subscriptions","organizations_url":"https://api.github.com/users/medic/orgs","repos_url":"https://api.github.com/users/medic/repos","events_url":"https://api.github.com/users/medic/events{/privacy}","received_events_url":"https://api.github.com/users/medic/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/medic/medic-analytics","description":"Software for creating read-only replicas of Medic Mobile data, using PostgreSQL v9.4","fork":false,"url":"https://api.github.com/repos/medic/medic-analytics","forks_url":"https://api.github.com/repos/medic/medic-analytics/forks","keys_url":"https://api.github.com/repos/medic/medic-analytics/keys{/key_id}","collaborators_url":"https://api.github.com/repos/medic/medic-analytics/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/medic/medic-analytics/teams","hooks_url":"https://api.github.com/repos/medic/medic-analytics/hooks","issue_events_url":"https://api.github.com/repos/medic/medic-analytics/issues/events{/number}","events_url":"https://api.github.com/repos/medic/medic-analytics/events","assignees_url":"https://api.github.com/repos/medic/medic-analytics/assignees{/user}","branches_url":"https://api.github.com/repos/medic/medic-analytics/branches{/branch}","tags_url":"https://api.github.com/repos/medic/medic-analytics/tags","blobs_url":"https://api.github.com/repos/medic/medic-analytics/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/medic/medic-analytics/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/medic/medic-analytics/git/refs{/sha}","trees_url":"https://api.github.com/repos/medic/medic-analytics/git/trees{/sha}","statuses_url":"https://api.github.com/repos/medic/medic-analytics/statuses/{sha}","languages_url":"https://api.github.com/repos/medic/medic-analytics/languages","stargazers_url":"https://api.github.com/repos/medic/medic-analytics/stargazers","contributors_url":"https://api.github.com/repos/medic/medic-analytics/contributors","subscribers_url":"https://api.github.com/repos/medic/medic-analytics/subscribers","subscription_url":"https://api.github.com/repos/medic/medic-analytics/subscription","commits_url":"https://api.github.com/repos/medic/medic-analytics/commits{/sha}","git_commits_url":"https://api.github.com/repos/medic/medic-analytics/git/commits{/sha}","comments_url":"https://api.github.com/repos/medic/medic-analytics/comments{/number}","issue_comment_url":"https://api.github.com/repos/medic/medic-analytics/issues/comments{/number}","contents_url":"https://api.github.com/repos/medic/medic-analytics/contents/{+path}","compare_url":"https://api.github.com/repos/medic/medic-analytics/compare/{base}...{head}","merges_url":"https://api.github.com/repos/medic/medic-analytics/merges","archive_url":"https://api.github.com/repos/medic/medic-analytics/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/medic/medic-analytics/downloads","issues_url":"https://api.github.com/repos/medic/medic-analytics/issues{/number}","pulls_url":"https://api.github.com/repos/medic/medic-analytics/pulls{/number}","milestones_url":"https://api.github.com/repos/medic/medic-analytics/milestones{/number}","notifications_url":"https://api.github.com/repos/medic/medic-analytics/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/medic/medic-analytics/labels{/name}","releases_url":"https://api.github.com/repos/medic/medic-analytics/releases{/id}","deployments_url":"https://api.github.com/repos/medic/medic-analytics/deployments","created_at":"2015-07-09T19:00:16Z","updated_at":"2016-01-28T20:23:55Z","pushed_at":"2016-07-01T13:20:55Z","git_url":"git://github.com/medic/medic-analytics.git","ssh_url":"git@github.com:medic/medic-analytics.git","clone_url":"https://github.com/medic/medic-analytics.git","svn_url":"https://github.com/medic/medic-analytics","homepage":null,"size":207,"stargazers_count":0,"watchers_count":0,"language":"JavaScript","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":1,"forks":0,"open_issues":1,"watchers":0,"default_branch":"master"}},"base":{"label":"medic:master","ref":"master","sha":"56ce6550eb5fdf6e811269902c4117e931aa91b5","user":{"login":"medic","id":474424,"avatar_url":"https://avatars.githubusercontent.com/u/474424?v=3","gravatar_id":"","url":"https://api.github.com/users/medic","html_url":"https://github.com/medic","followers_url":"https://api.github.com/users/medic/followers","following_url":"https://api.github.com/users/medic/following{/other_user}","gists_url":"https://api.github.com/users/medic/gists{/gist_id}","starred_url":"https://api.github.com/users/medic/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/medic/subscriptions","organizations_url":"https://api.github.com/users/medic/orgs","repos_url":"https://api.github.com/users/medic/repos","events_url":"https://api.github.com/users/medic/events{/privacy}","received_events_url":"https://api.github.com/users/medic/received_events","type":"Organization","site_admin":false},"repo":{"id":38839516,"name":"medic-analytics","full_name":"medic/medic-analytics","owner":{"login":"medic","id":474424,"avatar_url":"https://avatars.githubusercontent.com/u/474424?v=3","gravatar_id":"","url":"https://api.github.com/users/medic","html_url":"https://github.com/medic","followers_url":"https://api.github.com/users/medic/followers","following_url":"https://api.github.com/users/medic/following{/other_user}","gists_url":"https://api.github.com/users/medic/gists{/gist_id}","starred_url":"https://api.github.com/users/medic/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/medic/subscriptions","organizations_url":"https://api.github.com/users/medic/orgs","repos_url":"https://api.github.com/users/medic/repos","events_url":"https://api.github.com/users/medic/events{/privacy}","received_events_url":"https://api.github.com/users/medic/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/medic/medic-analytics","description":"Software for creating read-only replicas of Medic Mobile data, using PostgreSQL v9.4","fork":false,"url":"https://api.github.com/repos/medic/medic-analytics","forks_url":"https://api.github.com/repos/medic/medic-analytics/forks","keys_url":"https://api.github.com/repos/medic/medic-analytics/keys{/key_id}","collaborators_url":"https://api.github.com/repos/medic/medic-analytics/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/medic/medic-analytics/teams","hooks_url":"https://api.github.com/repos/medic/medic-analytics/hooks","issue_events_url":"https://api.github.com/repos/medic/medic-analytics/issues/events{/number}","events_url":"https://api.github.com/repos/medic/medic-analytics/events","assignees_url":"https://api.github.com/repos/medic/medic-analytics/assignees{/user}","branches_url":"https://api.github.com/repos/medic/medic-analytics/branches{/branch}","tags_url":"https://api.github.com/repos/medic/medic-analytics/tags","blobs_url":"https://api.github.com/repos/medic/medic-analytics/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/medic/medic-analytics/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/medic/medic-analytics/git/refs{/sha}","trees_url":"https://api.github.com/repos/medic/medic-analytics/git/trees{/sha}","statuses_url":"https://api.github.com/repos/medic/medic-analytics/statuses/{sha}","languages_url":"https://api.github.com/repos/medic/medic-analytics/languages","stargazers_url":"https://api.github.com/repos/medic/medic-analytics/stargazers","contributors_url":"https://api.github.com/repos/medic/medic-analytics/contributors","subscribers_url":"https://api.github.com/repos/medic/medic-analytics/subscribers","subscription_url":"https://api.github.com/repos/medic/medic-analytics/subscription","commits_url":"https://api.github.com/repos/medic/medic-analytics/commits{/sha}","git_commits_url":"https://api.github.com/repos/medic/medic-analytics/git/commits{/sha}","comments_url":"https://api.github.com/repos/medic/medic-analytics/comments{/number}","issue_comment_url":"https://api.github.com/repos/medic/medic-analytics/issues/comments{/number}","contents_url":"https://api.github.com/repos/medic/medic-analytics/contents/{+path}","compare_url":"https://api.github.com/repos/medic/medic-analytics/compare/{base}...{head}","merges_url":"https://api.github.com/repos/medic/medic-analytics/merges","archive_url":"https://api.github.com/repos/medic/medic-analytics/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/medic/medic-analytics/downloads","issues_url":"https://api.github.com/repos/medic/medic-analytics/issues{/number}","pulls_url":"https://api.github.com/repos/medic/medic-analytics/pulls{/number}","milestones_url":"https://api.github.com/repos/medic/medic-analytics/milestones{/number}","notifications_url":"https://api.github.com/repos/medic/medic-analytics/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/medic/medic-analytics/labels{/name}","releases_url":"https://api.github.com/repos/medic/medic-analytics/releases{/id}","deployments_url":"https://api.github.com/repos/medic/medic-analytics/deployments","created_at":"2015-07-09T19:00:16Z","updated_at":"2016-01-28T20:23:55Z","pushed_at":"2016-07-01T13:20:55Z","git_url":"git://github.com/medic/medic-analytics.git","ssh_url":"git@github.com:medic/medic-analytics.git","clone_url":"https://github.com/medic/medic-analytics.git","svn_url":"https://github.com/medic/medic-analytics","homepage":null,"size":207,"stargazers_count":0,"watchers_count":0,"language":"JavaScript","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":1,"forks":0,"open_issues":1,"watchers":0,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/medic/medic-analytics/pulls/33"},"html":{"href":"https://github.com/medic/medic-analytics/pull/33"},"issue":{"href":"https://api.github.com/repos/medic/medic-analytics/issues/33"},"comments":{"href":"https://api.github.com/repos/medic/medic-analytics/issues/33/comments"},"review_comments":{"href":"https://api.github.com/repos/medic/medic-analytics/pulls/33/comments"},"review_comment":{"href":"https://api.github.com/repos/medic/medic-analytics/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/medic/medic-analytics/pulls/33/commits"},"statuses":{"href":"https://api.github.com/repos/medic/medic-analytics/statuses/f9f3d1cc3d9db72509c47b765d518681f077fa6f"}}}}
{"actor":{"display_login":"garethbowen"}}