{"action":"created","comment":{"url":"https://api.github.com/repos/alphagov/email-alert-api/pulls/comments/171787948","pull_request_review_id":100703023,"id":171787948,"diff_hunk":"@@ -0,0 +1,85 @@\n+class EmailArchiveWorker\n+  include Sidekiq::Worker\n+\n+  sidekiq_options queue: :cleanup\n+\n+  LOCK_NAME = \"email_archive_worker\".freeze\n+\n+  def perform\n+    Email.with_advisory_lock(LOCK_NAME, timeout_seconds: 0) do\n+      start_time = Time.zone.now\n+      archived = 0\n+\n+      EmailArchiveQuery.call.in_batches do |batch|\n+        Email.transaction do\n+          archived += archive_batch(batch)\n+        end\n+      end\n+\n+      log_complete(archived, start_time, Time.zone.now)\n+    end\n+  end\n+\n+private\n+\n+  def archive_batch(batch)\n+    archived_at = Time.zone.now\n+\n+    import_batch(batch, archived_at)\n+    Email.where(id: batch.pluck(:id)).update_all(archived_at: archived_at)\n+  end\n+\n+  def import_batch(batch, archived_at)\n+    to_import = batch.as_json.map { |e| build_email_archive(e, archived_at) }\n+    columns = to_import.first.keys.map(&:to_s)\n+    values = to_import.map(&:values)\n+    EmailArchive.import(columns, values, validate: false)\n+  end\n+\n+  def build_email_archive(email_data, archived_at)\n+    content_change = build_content_change(email_data)\n+\n+    {\n+      archived_at: archived_at,\n+      content_change: content_change,\n+      created_at: email_data[\"created_at\"],","path":"app/workers/email_archive_worker.rb","position":45,"original_position":45,"commit_id":"7113ed075306fd6ae1c62877874d47f1a28b3c8b","original_commit_id":"7113ed075306fd6ae1c62877874d47f1a28b3c8b","user":{"login":"thomasleese","id":510498,"avatar_url":"https://avatars2.githubusercontent.com/u/510498?v=4","gravatar_id":"","url":"https://api.github.com/users/thomasleese","html_url":"https://github.com/thomasleese","followers_url":"https://api.github.com/users/thomasleese/followers","following_url":"https://api.github.com/users/thomasleese/following{/other_user}","gists_url":"https://api.github.com/users/thomasleese/gists{/gist_id}","starred_url":"https://api.github.com/users/thomasleese/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/thomasleese/subscriptions","organizations_url":"https://api.github.com/users/thomasleese/orgs","repos_url":"https://api.github.com/users/thomasleese/repos","events_url":"https://api.github.com/users/thomasleese/events{/privacy}","received_events_url":"https://api.github.com/users/thomasleese/received_events","type":"User","site_admin":false},"body":"Is it worth using `.fetch` for a lot of these methods to error out early?","created_at":"2018-03-02T08:34:15Z","updated_at":"2018-03-02T08:34:36Z","html_url":"https://github.com/alphagov/email-alert-api/pull/476#discussion_r171787948","pull_request_url":"https://api.github.com/repos/alphagov/email-alert-api/pulls/476","author_association":"CONTRIBUTOR","_links":{"self":{"href":"https://api.github.com/repos/alphagov/email-alert-api/pulls/comments/171787948"},"html":{"href":"https://github.com/alphagov/email-alert-api/pull/476#discussion_r171787948"},"pull_request":{"href":"https://api.github.com/repos/alphagov/email-alert-api/pulls/476"}}},"pull_request":{"url":"https://api.github.com/repos/alphagov/email-alert-api/pulls/476","id":172290871,"html_url":"https://github.com/alphagov/email-alert-api/pull/476","diff_url":"https://github.com/alphagov/email-alert-api/pull/476.diff","patch_url":"https://github.com/alphagov/email-alert-api/pull/476.patch","issue_url":"https://api.github.com/repos/alphagov/email-alert-api/issues/476","number":476,"state":"open","locked":false,"title":"Email archiving tasks","user":{"login":"kevindew","id":282717,"avatar_url":"https://avatars0.githubusercontent.com/u/282717?v=4","gravatar_id":"","url":"https://api.github.com/users/kevindew","html_url":"https://github.com/kevindew","followers_url":"https://api.github.com/users/kevindew/followers","following_url":"https://api.github.com/users/kevindew/following{/other_user}","gists_url":"https://api.github.com/users/kevindew/gists{/gist_id}","starred_url":"https://api.github.com/users/kevindew/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/kevindew/subscriptions","organizations_url":"https://api.github.com/users/kevindew/orgs","repos_url":"https://api.github.com/users/kevindew/repos","events_url":"https://api.github.com/users/kevindew/events{/privacy}","received_events_url":"https://api.github.com/users/kevindew/received_events","type":"User","site_admin":false},"body":"Trello: https://trello.com/c/R14kXWU9/633-set-up-rudimentary-email-archive-system\r\n\r\nThis sets up the appropriate extra data fields for email archiving as per [data retention strategy](https://github.com/alphagov/email-alert-api/blob/68fb5c40863114a18655c12db01c87e60eb5dc77/doc/arch/adr-003-data-retention.md) and the worker classes to perform the tasks.\r\n\r\nI've set them to run every hour and I imagine they'll be pretty fast. My maths has them averaging at about 30,000 emails an hour and running locally (for a dataset of 30,000) I get:\r\n\r\n```\r\nirb(main):009:0> 30_000.times { FactoryBot.create(:subscription_content, :with_archivable_email) }\r\n=> 30000\r\nirb(main):011:0> { archivable_emails: Email.archivable.count, deleteable_emails: Email.deleteable.count, subscription_contents: SubscriptionContent.count, email_archives: EmailArchive.count }\r\n=> {:archivable_emails=>30000, :deleteable_emails=>0, :subscription_contents=>30000, :email_archives=>0}\r\nirb(main):012:0> EmailArchiveWorker.new.perform\r\n{\"@timestamp\":\"2018-03-01T16:41:58Z\",\"@fields\":{\"pid\":24478,\"tid\":\"TID-gndlb4qq0\",\"context\":\"\",\"program_name\":null,\"worker\":null},\"@type\":\"sidekiq\",\"@status\":null,\"@severity\":\"INFO\",\"@run_time\":null,\"@message\":\"Archived 30000 emails in 12.77 seconds\"}\r\n=> true\r\nirb(main):013:0> { archivable_emails: Email.archivable.count, deleteable_emails: Email.deleteable.count, subscription_contents: SubscriptionContent.count, email_archives: EmailArchive.count }\r\n=> {:archivable_emails=>0, :deleteable_emails=>0, :subscription_contents=>30000, :email_archives=>30000}\r\nirb(main):014:0> Email.update_all(finished_sending_at: 15.days.ago)\r\n=> 30000\r\nirb(main):015:0> { archivable_emails: Email.archivable.count, deleteable_emails: Email.deleteable.count, subscription_contents: SubscriptionContent.count, email_archives: EmailArchive.count }\r\n=> {:archivable_emails=>0, :deleteable_emails=>30000, :subscription_contents=>30000, :email_archives=>30000}\r\nirb(main):016:0> EmailDeletionWorker.new.perform\r\n{\"@timestamp\":\"2018-03-01T16:42:40Z\",\"@fields\":{\"pid\":24478,\"tid\":\"TID-gndlb4qq0\",\"context\":\"\",\"program_name\":null,\"worker\":null},\"@type\":\"sidekiq\",\"@status\":null,\"@severity\":\"INFO\",\"@run_time\":null,\"@message\":\"Deleted 30000 emails in 7.18 seconds\"}\r\n=> true\r\nirb(main):017:0> { archivable_emails: Email.archivable.count, deleteable_emails: Email.deleteable.count, subscription_contents: SubscriptionContent.count, email_archives: EmailArchive.count }\r\n=> {:archivable_emails=>0, :deleteable_emails=>0, :subscription_contents=>30000, :email_archives=>30000}\r\nirb(main):018:0> Email.count\r\n=> 0\r\n```\r\n\r\nIt is set to wait 14 days before deleting emails. Until we've upped the postgres disk space this might be a little optimistic, but hopefully that will be resolved soon - but we may still need to adjust this.","created_at":"2018-03-01T16:44:04Z","updated_at":"2018-03-02T08:34:36Z","closed_at":null,"merged_at":null,"merge_commit_sha":"90ab7ee5fd2d8ccf4fa77b353f9b215f40048525","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/alphagov/email-alert-api/pulls/476/commits","review_comments_url":"https://api.github.com/repos/alphagov/email-alert-api/pulls/476/comments","review_comment_url":"https://api.github.com/repos/alphagov/email-alert-api/pulls/comments{/number}","comments_url":"https://api.github.com/repos/alphagov/email-alert-api/issues/476/comments","statuses_url":"https://api.github.com/repos/alphagov/email-alert-api/statuses/7113ed075306fd6ae1c62877874d47f1a28b3c8b","head":{"label":"alphagov:email-archive","ref":"email-archive","sha":"7113ed075306fd6ae1c62877874d47f1a28b3c8b","user":{"login":"alphagov","id":596977,"avatar_url":"https://avatars0.githubusercontent.com/u/596977?v=4","gravatar_id":"","url":"https://api.github.com/users/alphagov","html_url":"https://github.com/alphagov","followers_url":"https://api.github.com/users/alphagov/followers","following_url":"https://api.github.com/users/alphagov/following{/other_user}","gists_url":"https://api.github.com/users/alphagov/gists{/gist_id}","starred_url":"https://api.github.com/users/alphagov/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/alphagov/subscriptions","organizations_url":"https://api.github.com/users/alphagov/orgs","repos_url":"https://api.github.com/users/alphagov/repos","events_url":"https://api.github.com/users/alphagov/events{/privacy}","received_events_url":"https://api.github.com/users/alphagov/received_events","type":"Organization","site_admin":false},"repo":{"id":24410607,"name":"email-alert-api","full_name":"alphagov/email-alert-api","owner":{"login":"alphagov","id":596977,"avatar_url":"https://avatars0.githubusercontent.com/u/596977?v=4","gravatar_id":"","url":"https://api.github.com/users/alphagov","html_url":"https://github.com/alphagov","followers_url":"https://api.github.com/users/alphagov/followers","following_url":"https://api.github.com/users/alphagov/following{/other_user}","gists_url":"https://api.github.com/users/alphagov/gists{/gist_id}","starred_url":"https://api.github.com/users/alphagov/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/alphagov/subscriptions","organizations_url":"https://api.github.com/users/alphagov/orgs","repos_url":"https://api.github.com/users/alphagov/repos","events_url":"https://api.github.com/users/alphagov/events{/privacy}","received_events_url":"https://api.github.com/users/alphagov/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/alphagov/email-alert-api","description":"Sends email alerts to the public for GOV.UK","fork":false,"url":"https://api.github.com/repos/alphagov/email-alert-api","forks_url":"https://api.github.com/repos/alphagov/email-alert-api/forks","keys_url":"https://api.github.com/repos/alphagov/email-alert-api/keys{/key_id}","collaborators_url":"https://api.github.com/repos/alphagov/email-alert-api/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/alphagov/email-alert-api/teams","hooks_url":"https://api.github.com/repos/alphagov/email-alert-api/hooks","issue_events_url":"https://api.github.com/repos/alphagov/email-alert-api/issues/events{/number}","events_url":"https://api.github.com/repos/alphagov/email-alert-api/events","assignees_url":"https://api.github.com/repos/alphagov/email-alert-api/assignees{/user}","branches_url":"https://api.github.com/repos/alphagov/email-alert-api/branches{/branch}","tags_url":"https://api.github.com/repos/alphagov/email-alert-api/tags","blobs_url":"https://api.github.com/repos/alphagov/email-alert-api/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/alphagov/email-alert-api/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/alphagov/email-alert-api/git/refs{/sha}","trees_url":"https://api.github.com/repos/alphagov/email-alert-api/git/trees{/sha}","statuses_url":"https://api.github.com/repos/alphagov/email-alert-api/statuses/{sha}","languages_url":"https://api.github.com/repos/alphagov/email-alert-api/languages","stargazers_url":"https://api.github.com/repos/alphagov/email-alert-api/stargazers","contributors_url":"https://api.github.com/repos/alphagov/email-alert-api/contributors","subscribers_url":"https://api.github.com/repos/alphagov/email-alert-api/subscribers","subscription_url":"https://api.github.com/repos/alphagov/email-alert-api/subscription","commits_url":"https://api.github.com/repos/alphagov/email-alert-api/commits{/sha}","git_commits_url":"https://api.github.com/repos/alphagov/email-alert-api/git/commits{/sha}","comments_url":"https://api.github.com/repos/alphagov/email-alert-api/comments{/number}","issue_comment_url":"https://api.github.com/repos/alphagov/email-alert-api/issues/comments{/number}","contents_url":"https://api.github.com/repos/alphagov/email-alert-api/contents/{+path}","compare_url":"https://api.github.com/repos/alphagov/email-alert-api/compare/{base}...{head}","merges_url":"https://api.github.com/repos/alphagov/email-alert-api/merges","archive_url":"https://api.github.com/repos/alphagov/email-alert-api/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/alphagov/email-alert-api/downloads","issues_url":"https://api.github.com/repos/alphagov/email-alert-api/issues{/number}","pulls_url":"https://api.github.com/repos/alphagov/email-alert-api/pulls{/number}","milestones_url":"https://api.github.com/repos/alphagov/email-alert-api/milestones{/number}","notifications_url":"https://api.github.com/repos/alphagov/email-alert-api/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/alphagov/email-alert-api/labels{/name}","releases_url":"https://api.github.com/repos/alphagov/email-alert-api/releases{/id}","deployments_url":"https://api.github.com/repos/alphagov/email-alert-api/deployments","created_at":"2014-09-24T10:22:32Z","updated_at":"2017-09-06T13:30:54Z","pushed_at":"2018-03-01T18:41:58Z","git_url":"git://github.com/alphagov/email-alert-api.git","ssh_url":"git@github.com:alphagov/email-alert-api.git","clone_url":"https://github.com/alphagov/email-alert-api.git","svn_url":"https://github.com/alphagov/email-alert-api","homepage":"https://docs.publishing.service.gov.uk/apps/email-alert-api.html","size":3513,"stargazers_count":1,"watchers_count":1,"language":"Ruby","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":5,"license":{"key":"mit","name":"MIT License","spdx_id":"MIT","url":"https://api.github.com/licenses/mit"},"forks":0,"open_issues":5,"watchers":1,"default_branch":"master"}},"base":{"label":"alphagov:master","ref":"master","sha":"68fb5c40863114a18655c12db01c87e60eb5dc77","user":{"login":"alphagov","id":596977,"avatar_url":"https://avatars0.githubusercontent.com/u/596977?v=4","gravatar_id":"","url":"https://api.github.com/users/alphagov","html_url":"https://github.com/alphagov","followers_url":"https://api.github.com/users/alphagov/followers","following_url":"https://api.github.com/users/alphagov/following{/other_user}","gists_url":"https://api.github.com/users/alphagov/gists{/gist_id}","starred_url":"https://api.github.com/users/alphagov/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/alphagov/subscriptions","organizations_url":"https://api.github.com/users/alphagov/orgs","repos_url":"https://api.github.com/users/alphagov/repos","events_url":"https://api.github.com/users/alphagov/events{/privacy}","received_events_url":"https://api.github.com/users/alphagov/received_events","type":"Organization","site_admin":false},"repo":{"id":24410607,"name":"email-alert-api","full_name":"alphagov/email-alert-api","owner":{"login":"alphagov","id":596977,"avatar_url":"https://avatars0.githubusercontent.com/u/596977?v=4","gravatar_id":"","url":"https://api.github.com/users/alphagov","html_url":"https://github.com/alphagov","followers_url":"https://api.github.com/users/alphagov/followers","following_url":"https://api.github.com/users/alphagov/following{/other_user}","gists_url":"https://api.github.com/users/alphagov/gists{/gist_id}","starred_url":"https://api.github.com/users/alphagov/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/alphagov/subscriptions","organizations_url":"https://api.github.com/users/alphagov/orgs","repos_url":"https://api.github.com/users/alphagov/repos","events_url":"https://api.github.com/users/alphagov/events{/privacy}","received_events_url":"https://api.github.com/users/alphagov/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/alphagov/email-alert-api","description":"Sends email alerts to the public for GOV.UK","fork":false,"url":"https://api.github.com/repos/alphagov/email-alert-api","forks_url":"https://api.github.com/repos/alphagov/email-alert-api/forks","keys_url":"https://api.github.com/repos/alphagov/email-alert-api/keys{/key_id}","collaborators_url":"https://api.github.com/repos/alphagov/email-alert-api/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/alphagov/email-alert-api/teams","hooks_url":"https://api.github.com/repos/alphagov/email-alert-api/hooks","issue_events_url":"https://api.github.com/repos/alphagov/email-alert-api/issues/events{/number}","events_url":"https://api.github.com/repos/alphagov/email-alert-api/events","assignees_url":"https://api.github.com/repos/alphagov/email-alert-api/assignees{/user}","branches_url":"https://api.github.com/repos/alphagov/email-alert-api/branches{/branch}","tags_url":"https://api.github.com/repos/alphagov/email-alert-api/tags","blobs_url":"https://api.github.com/repos/alphagov/email-alert-api/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/alphagov/email-alert-api/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/alphagov/email-alert-api/git/refs{/sha}","trees_url":"https://api.github.com/repos/alphagov/email-alert-api/git/trees{/sha}","statuses_url":"https://api.github.com/repos/alphagov/email-alert-api/statuses/{sha}","languages_url":"https://api.github.com/repos/alphagov/email-alert-api/languages","stargazers_url":"https://api.github.com/repos/alphagov/email-alert-api/stargazers","contributors_url":"https://api.github.com/repos/alphagov/email-alert-api/contributors","subscribers_url":"https://api.github.com/repos/alphagov/email-alert-api/subscribers","subscription_url":"https://api.github.com/repos/alphagov/email-alert-api/subscription","commits_url":"https://api.github.com/repos/alphagov/email-alert-api/commits{/sha}","git_commits_url":"https://api.github.com/repos/alphagov/email-alert-api/git/commits{/sha}","comments_url":"https://api.github.com/repos/alphagov/email-alert-api/comments{/number}","issue_comment_url":"https://api.github.com/repos/alphagov/email-alert-api/issues/comments{/number}","contents_url":"https://api.github.com/repos/alphagov/email-alert-api/contents/{+path}","compare_url":"https://api.github.com/repos/alphagov/email-alert-api/compare/{base}...{head}","merges_url":"https://api.github.com/repos/alphagov/email-alert-api/merges","archive_url":"https://api.github.com/repos/alphagov/email-alert-api/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/alphagov/email-alert-api/downloads","issues_url":"https://api.github.com/repos/alphagov/email-alert-api/issues{/number}","pulls_url":"https://api.github.com/repos/alphagov/email-alert-api/pulls{/number}","milestones_url":"https://api.github.com/repos/alphagov/email-alert-api/milestones{/number}","notifications_url":"https://api.github.com/repos/alphagov/email-alert-api/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/alphagov/email-alert-api/labels{/name}","releases_url":"https://api.github.com/repos/alphagov/email-alert-api/releases{/id}","deployments_url":"https://api.github.com/repos/alphagov/email-alert-api/deployments","created_at":"2014-09-24T10:22:32Z","updated_at":"2017-09-06T13:30:54Z","pushed_at":"2018-03-01T18:41:58Z","git_url":"git://github.com/alphagov/email-alert-api.git","ssh_url":"git@github.com:alphagov/email-alert-api.git","clone_url":"https://github.com/alphagov/email-alert-api.git","svn_url":"https://github.com/alphagov/email-alert-api","homepage":"https://docs.publishing.service.gov.uk/apps/email-alert-api.html","size":3513,"stargazers_count":1,"watchers_count":1,"language":"Ruby","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":5,"license":{"key":"mit","name":"MIT License","spdx_id":"MIT","url":"https://api.github.com/licenses/mit"},"forks":0,"open_issues":5,"watchers":1,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/alphagov/email-alert-api/pulls/476"},"html":{"href":"https://github.com/alphagov/email-alert-api/pull/476"},"issue":{"href":"https://api.github.com/repos/alphagov/email-alert-api/issues/476"},"comments":{"href":"https://api.github.com/repos/alphagov/email-alert-api/issues/476/comments"},"review_comments":{"href":"https://api.github.com/repos/alphagov/email-alert-api/pulls/476/comments"},"review_comment":{"href":"https://api.github.com/repos/alphagov/email-alert-api/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/alphagov/email-alert-api/pulls/476/commits"},"statuses":{"href":"https://api.github.com/repos/alphagov/email-alert-api/statuses/7113ed075306fd6ae1c62877874d47f1a28b3c8b"}},"author_association":"MEMBER"}}
{"actor":{"display_login":"thomasleese"}}