{"action":"created","comment":{"url":"https://api.github.com/repos/hashicorp/vault/pulls/comments/50904867","id":50904867,"diff_hunk":"@@ -72,11 +73,182 @@ type Policy struct {\n \t// for decryption\n \tMinDecryptionVersion int `json:\"min_decryption_version\"`\n \n+\t// The latest key version in this policy\n+\tLatestVersion int `json:\"latest_version\"`\n+\n+\t// The latest key version in the archive. We never delete these, so this is\n+\t// a max.\n+\tArchiveVersion int `json:\"archive_version\"`\n+\n \t// Whether the key is allowed to be deleted\n \tDeletionAllowed bool `json:\"deletion_allowed\"`\n }\n \n-func (p *Policy) Persist(storage logical.Storage, name string) error {\n+// ArchivedKeys stores old keys. This is used to keep the key loading time sane\n+// when there are huge numbers of rotations.\n+type ArchivedKeys struct {\n+\tKeys []KeyEntry `json:\"keys\"`\n+}\n+\n+func (p *Policy) loadArchive(storage logical.Storage) (*ArchivedKeys, error) {\n+\tarchive := &ArchivedKeys{}\n+\n+\traw, err := storage.Get(\"policy/\" + p.Name + \"/archive\")\n+\tif err != nil {\n+\t\treturn nil, err\n+\t}\n+\tif raw == nil {\n+\t\tarchive.Keys = make([]KeyEntry, 0)\n+\t\treturn archive, nil\n+\t}\n+\n+\tif err := json.Unmarshal(raw.Value, archive); err != nil {\n+\t\treturn nil, err\n+\t}\n+\n+\treturn archive, nil\n+}\n+\n+func (p *Policy) storeArchive(archive *ArchivedKeys, storage logical.Storage) error {\n+\t// Encode the policy\n+\tbuf, err := json.Marshal(archive)\n+\tif err != nil {\n+\t\treturn err\n+\t}\n+\n+\t// Write the policy into storage\n+\terr = storage.Put(&logical.StorageEntry{\n+\t\tKey:   \"policy/\" + p.Name + \"/archive\",\n+\t\tValue: buf,\n+\t})\n+\tif err != nil {\n+\t\treturn err\n+\t}\n+\n+\treturn nil\n+}\n+\n+// handleArchiving manages the movement of keys to and from the policy archive.\n+// This should *ONLY* be called from Persist() since it assumes that the policy\n+// will be persisted afterwards.\n+func (p *Policy) handleArchiving(storage logical.Storage) error {\n+\t// We need to move keys that are no longer accessible to ArchivedKeys, and keys\n+\t// that now need to be accessible back here.\n+\t//\n+\t// For safety, because there isn't really a good reason to, we never delete\n+\t// keys from the archive even when we move them back.\n+\n+\t// Sanity checks\n+\tswitch {\n+\tcase p.MinDecryptionVersion < 1:\n+\t\treturn fmt.Errorf(\"minimum decryption version of %d is less than 1\", p.MinDecryptionVersion)\n+\tcase p.LatestVersion < 1:\n+\t\treturn fmt.Errorf(\"latest version of %d is less than 1\", p.LatestVersion)\n+\tcase p.MinDecryptionVersion > p.LatestVersion:\n+\t\treturn fmt.Errorf(\"minimum decryption version of %d is greater than the latest version %d\",\n+\t\t\tp.MinDecryptionVersion, p.LatestVersion)\n+\t}\n+\n+\t// Check if we have the latest minimum version in the current set of keys\n+\t_, keysContainsMinimum := p.Keys[p.MinDecryptionVersion]\n+\n+\t// If keys contains the minimum value, we are moving keys *to* the archive,\n+\t// but we only need to do this if the archive doesn't contain those key\n+\t// versions, since we don't remove key versions from the archive.\n+\tif keysContainsMinimum &&\n+\t\tp.ArchiveVersion >= p.MinDecryptionVersion-1 {\n+\t\treturn nil\n+\t}\n+\n+\tarchive, err := p.loadArchive(storage)\n+\tif err != nil {\n+\t\treturn err\n+\t}\n+\n+\tif keysContainsMinimum {\n+\t\t// Need to move keys *to* archive\n+\n+\t\t// We need a size that is equivalent to the minimum decryption version\n+\t\t// minus 1, but adding one since slice numbering starts at 0 and we're\n+\t\t// indexing by key version\n+\t\tif len(archive.Keys) < p.MinDecryptionVersion {\n+\t\t\t// Increase the size of the archive slice\n+\t\t\tnewKeys := make([]KeyEntry, p.MinDecryptionVersion)\n+\t\t\tcopy(newKeys, archive.Keys)\n+\t\t\tarchive.Keys = newKeys\n+\t\t}\n+\n+\t\t// As we are archiving progressively, we should only have to archive\n+\t\t// from the min version down to the latest version minus however many\n+\t\t// keys are in the policy's map. For example, if we have never\n+\t\t// archived, the latest version is 10, and we move the min decryption\n+\t\t// version to 5, we will archive from 4 down to (10-10) non-inclusive.\n+\t\t// If the latest version now becomes 8, we will archive from 7 down to\n+\t\t// (10-6) non-inclusive, e.g. keys 5, 6, and 7.\n+\t\tfor i := p.LatestVersion - len(p.Keys) + 1; i < p.MinDecryptionVersion; i++ {","path":"builtin/logical/transit/policy.go","position":125,"original_position":125,"commit_id":"934be180d67905f8b5b80710c91edbe53d3e989f","original_commit_id":"934be180d67905f8b5b80710c91edbe53d3e989f","user":{"login":"armon","id":592032,"avatar_url":"https://avatars.githubusercontent.com/u/592032?v=3","gravatar_id":"","url":"https://api.github.com/users/armon","html_url":"https://github.com/armon","followers_url":"https://api.github.com/users/armon/followers","following_url":"https://api.github.com/users/armon/following{/other_user}","gists_url":"https://api.github.com/users/armon/gists{/gist_id}","starred_url":"https://api.github.com/users/armon/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/armon/subscriptions","organizations_url":"https://api.github.com/users/armon/orgs","repos_url":"https://api.github.com/users/armon/repos","events_url":"https://api.github.com/users/armon/events{/privacy}","received_events_url":"https://api.github.com/users/armon/received_events","type":"User","site_admin":false},"body":"Can we initialize i to just `len(archive.Keys)`?","created_at":"2016-01-26T21:41:48Z","updated_at":"2016-01-26T21:41:48Z","html_url":"https://github.com/hashicorp/vault/pull/977#discussion_r50904867","pull_request_url":"https://api.github.com/repos/hashicorp/vault/pulls/977","_links":{"self":{"href":"https://api.github.com/repos/hashicorp/vault/pulls/comments/50904867"},"html":{"href":"https://github.com/hashicorp/vault/pull/977#discussion_r50904867"},"pull_request":{"href":"https://api.github.com/repos/hashicorp/vault/pulls/977"}}},"pull_request":{"url":"https://api.github.com/repos/hashicorp/vault/pulls/977","id":57264346,"html_url":"https://github.com/hashicorp/vault/pull/977","diff_url":"https://github.com/hashicorp/vault/pull/977.diff","patch_url":"https://github.com/hashicorp/vault/pull/977.patch","issue_url":"https://api.github.com/repos/hashicorp/vault/issues/977","number":977,"state":"open","locked":false,"title":"Transit key archiving","user":{"login":"jefferai","id":28627,"avatar_url":"https://avatars.githubusercontent.com/u/28627?v=3","gravatar_id":"","url":"https://api.github.com/users/jefferai","html_url":"https://github.com/jefferai","followers_url":"https://api.github.com/users/jefferai/followers","following_url":"https://api.github.com/users/jefferai/following{/other_user}","gists_url":"https://api.github.com/users/jefferai/gists{/gist_id}","starred_url":"https://api.github.com/users/jefferai/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jefferai/subscriptions","organizations_url":"https://api.github.com/users/jefferai/orgs","repos_url":"https://api.github.com/users/jefferai/repos","events_url":"https://api.github.com/users/jefferai/events{/privacy}","received_events_url":"https://api.github.com/users/jefferai/received_events","type":"User","site_admin":false},"body":"Archive transit keys that are below the `MinDecryptionVersion`. This allows lookups of in-use keys to only contain those keys, drastically speeding things up when keys have been rotated many (hundreds/thousands+) times. \r\n\r\nIf the `MinDecryptionVersion` is lowered, the keys will be moved back, although keys are never deleted from the archive -- it provides safety and there isn't much reason since it's okay if the rotation is a slower operation.\r\n\r\nAlso forces `MinDecryptionVersion` to be at least 1 since key numbering starts at 1 since 0.3; this makes logic easier and also consistency for the user is nice.\r\n\r\nFixes #925 ","created_at":"2016-01-26T20:07:41Z","updated_at":"2016-01-26T21:41:48Z","closed_at":null,"merged_at":null,"merge_commit_sha":"07f767178cf94364090f72f4115a58f9449521f7","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/hashicorp/vault/pulls/977/commits","review_comments_url":"https://api.github.com/repos/hashicorp/vault/pulls/977/comments","review_comment_url":"https://api.github.com/repos/hashicorp/vault/pulls/comments{/number}","comments_url":"https://api.github.com/repos/hashicorp/vault/issues/977/comments","statuses_url":"https://api.github.com/repos/hashicorp/vault/statuses/934be180d67905f8b5b80710c91edbe53d3e989f","head":{"label":"hashicorp:transit-key-archiving","ref":"transit-key-archiving","sha":"934be180d67905f8b5b80710c91edbe53d3e989f","user":{"login":"hashicorp","id":761456,"avatar_url":"https://avatars.githubusercontent.com/u/761456?v=3","gravatar_id":"","url":"https://api.github.com/users/hashicorp","html_url":"https://github.com/hashicorp","followers_url":"https://api.github.com/users/hashicorp/followers","following_url":"https://api.github.com/users/hashicorp/following{/other_user}","gists_url":"https://api.github.com/users/hashicorp/gists{/gist_id}","starred_url":"https://api.github.com/users/hashicorp/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/hashicorp/subscriptions","organizations_url":"https://api.github.com/users/hashicorp/orgs","repos_url":"https://api.github.com/users/hashicorp/repos","events_url":"https://api.github.com/users/hashicorp/events{/privacy}","received_events_url":"https://api.github.com/users/hashicorp/received_events","type":"Organization","site_admin":false},"repo":{"id":31288958,"name":"vault","full_name":"hashicorp/vault","owner":{"login":"hashicorp","id":761456,"avatar_url":"https://avatars.githubusercontent.com/u/761456?v=3","gravatar_id":"","url":"https://api.github.com/users/hashicorp","html_url":"https://github.com/hashicorp","followers_url":"https://api.github.com/users/hashicorp/followers","following_url":"https://api.github.com/users/hashicorp/following{/other_user}","gists_url":"https://api.github.com/users/hashicorp/gists{/gist_id}","starred_url":"https://api.github.com/users/hashicorp/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/hashicorp/subscriptions","organizations_url":"https://api.github.com/users/hashicorp/orgs","repos_url":"https://api.github.com/users/hashicorp/repos","events_url":"https://api.github.com/users/hashicorp/events{/privacy}","received_events_url":"https://api.github.com/users/hashicorp/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/hashicorp/vault","description":"A tool for managing secrets.","fork":false,"url":"https://api.github.com/repos/hashicorp/vault","forks_url":"https://api.github.com/repos/hashicorp/vault/forks","keys_url":"https://api.github.com/repos/hashicorp/vault/keys{/key_id}","collaborators_url":"https://api.github.com/repos/hashicorp/vault/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/hashicorp/vault/teams","hooks_url":"https://api.github.com/repos/hashicorp/vault/hooks","issue_events_url":"https://api.github.com/repos/hashicorp/vault/issues/events{/number}","events_url":"https://api.github.com/repos/hashicorp/vault/events","assignees_url":"https://api.github.com/repos/hashicorp/vault/assignees{/user}","branches_url":"https://api.github.com/repos/hashicorp/vault/branches{/branch}","tags_url":"https://api.github.com/repos/hashicorp/vault/tags","blobs_url":"https://api.github.com/repos/hashicorp/vault/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/hashicorp/vault/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/hashicorp/vault/git/refs{/sha}","trees_url":"https://api.github.com/repos/hashicorp/vault/git/trees{/sha}","statuses_url":"https://api.github.com/repos/hashicorp/vault/statuses/{sha}","languages_url":"https://api.github.com/repos/hashicorp/vault/languages","stargazers_url":"https://api.github.com/repos/hashicorp/vault/stargazers","contributors_url":"https://api.github.com/repos/hashicorp/vault/contributors","subscribers_url":"https://api.github.com/repos/hashicorp/vault/subscribers","subscription_url":"https://api.github.com/repos/hashicorp/vault/subscription","commits_url":"https://api.github.com/repos/hashicorp/vault/commits{/sha}","git_commits_url":"https://api.github.com/repos/hashicorp/vault/git/commits{/sha}","comments_url":"https://api.github.com/repos/hashicorp/vault/comments{/number}","issue_comment_url":"https://api.github.com/repos/hashicorp/vault/issues/comments{/number}","contents_url":"https://api.github.com/repos/hashicorp/vault/contents/{+path}","compare_url":"https://api.github.com/repos/hashicorp/vault/compare/{base}...{head}","merges_url":"https://api.github.com/repos/hashicorp/vault/merges","archive_url":"https://api.github.com/repos/hashicorp/vault/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/hashicorp/vault/downloads","issues_url":"https://api.github.com/repos/hashicorp/vault/issues{/number}","pulls_url":"https://api.github.com/repos/hashicorp/vault/pulls{/number}","milestones_url":"https://api.github.com/repos/hashicorp/vault/milestones{/number}","notifications_url":"https://api.github.com/repos/hashicorp/vault/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/hashicorp/vault/labels{/name}","releases_url":"https://api.github.com/repos/hashicorp/vault/releases{/id}","deployments_url":"https://api.github.com/repos/hashicorp/vault/deployments","created_at":"2015-02-25T00:15:59Z","updated_at":"2016-01-26T18:16:11Z","pushed_at":"2016-01-26T20:27:32Z","git_url":"git://github.com/hashicorp/vault.git","ssh_url":"git@github.com:hashicorp/vault.git","clone_url":"https://github.com/hashicorp/vault.git","svn_url":"https://github.com/hashicorp/vault","homepage":"http://vaultproject.io","size":11209,"stargazers_count":3971,"watchers_count":3971,"language":"JavaScript","has_issues":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":322,"mirror_url":null,"open_issues_count":144,"forks":322,"open_issues":144,"watchers":3971,"default_branch":"master"}},"base":{"label":"hashicorp:master","ref":"master","sha":"352f3abe874f4dd87007cbc5a03732dae1611560","user":{"login":"hashicorp","id":761456,"avatar_url":"https://avatars.githubusercontent.com/u/761456?v=3","gravatar_id":"","url":"https://api.github.com/users/hashicorp","html_url":"https://github.com/hashicorp","followers_url":"https://api.github.com/users/hashicorp/followers","following_url":"https://api.github.com/users/hashicorp/following{/other_user}","gists_url":"https://api.github.com/users/hashicorp/gists{/gist_id}","starred_url":"https://api.github.com/users/hashicorp/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/hashicorp/subscriptions","organizations_url":"https://api.github.com/users/hashicorp/orgs","repos_url":"https://api.github.com/users/hashicorp/repos","events_url":"https://api.github.com/users/hashicorp/events{/privacy}","received_events_url":"https://api.github.com/users/hashicorp/received_events","type":"Organization","site_admin":false},"repo":{"id":31288958,"name":"vault","full_name":"hashicorp/vault","owner":{"login":"hashicorp","id":761456,"avatar_url":"https://avatars.githubusercontent.com/u/761456?v=3","gravatar_id":"","url":"https://api.github.com/users/hashicorp","html_url":"https://github.com/hashicorp","followers_url":"https://api.github.com/users/hashicorp/followers","following_url":"https://api.github.com/users/hashicorp/following{/other_user}","gists_url":"https://api.github.com/users/hashicorp/gists{/gist_id}","starred_url":"https://api.github.com/users/hashicorp/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/hashicorp/subscriptions","organizations_url":"https://api.github.com/users/hashicorp/orgs","repos_url":"https://api.github.com/users/hashicorp/repos","events_url":"https://api.github.com/users/hashicorp/events{/privacy}","received_events_url":"https://api.github.com/users/hashicorp/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/hashicorp/vault","description":"A tool for managing secrets.","fork":false,"url":"https://api.github.com/repos/hashicorp/vault","forks_url":"https://api.github.com/repos/hashicorp/vault/forks","keys_url":"https://api.github.com/repos/hashicorp/vault/keys{/key_id}","collaborators_url":"https://api.github.com/repos/hashicorp/vault/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/hashicorp/vault/teams","hooks_url":"https://api.github.com/repos/hashicorp/vault/hooks","issue_events_url":"https://api.github.com/repos/hashicorp/vault/issues/events{/number}","events_url":"https://api.github.com/repos/hashicorp/vault/events","assignees_url":"https://api.github.com/repos/hashicorp/vault/assignees{/user}","branches_url":"https://api.github.com/repos/hashicorp/vault/branches{/branch}","tags_url":"https://api.github.com/repos/hashicorp/vault/tags","blobs_url":"https://api.github.com/repos/hashicorp/vault/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/hashicorp/vault/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/hashicorp/vault/git/refs{/sha}","trees_url":"https://api.github.com/repos/hashicorp/vault/git/trees{/sha}","statuses_url":"https://api.github.com/repos/hashicorp/vault/statuses/{sha}","languages_url":"https://api.github.com/repos/hashicorp/vault/languages","stargazers_url":"https://api.github.com/repos/hashicorp/vault/stargazers","contributors_url":"https://api.github.com/repos/hashicorp/vault/contributors","subscribers_url":"https://api.github.com/repos/hashicorp/vault/subscribers","subscription_url":"https://api.github.com/repos/hashicorp/vault/subscription","commits_url":"https://api.github.com/repos/hashicorp/vault/commits{/sha}","git_commits_url":"https://api.github.com/repos/hashicorp/vault/git/commits{/sha}","comments_url":"https://api.github.com/repos/hashicorp/vault/comments{/number}","issue_comment_url":"https://api.github.com/repos/hashicorp/vault/issues/comments{/number}","contents_url":"https://api.github.com/repos/hashicorp/vault/contents/{+path}","compare_url":"https://api.github.com/repos/hashicorp/vault/compare/{base}...{head}","merges_url":"https://api.github.com/repos/hashicorp/vault/merges","archive_url":"https://api.github.com/repos/hashicorp/vault/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/hashicorp/vault/downloads","issues_url":"https://api.github.com/repos/hashicorp/vault/issues{/number}","pulls_url":"https://api.github.com/repos/hashicorp/vault/pulls{/number}","milestones_url":"https://api.github.com/repos/hashicorp/vault/milestones{/number}","notifications_url":"https://api.github.com/repos/hashicorp/vault/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/hashicorp/vault/labels{/name}","releases_url":"https://api.github.com/repos/hashicorp/vault/releases{/id}","deployments_url":"https://api.github.com/repos/hashicorp/vault/deployments","created_at":"2015-02-25T00:15:59Z","updated_at":"2016-01-26T18:16:11Z","pushed_at":"2016-01-26T20:27:32Z","git_url":"git://github.com/hashicorp/vault.git","ssh_url":"git@github.com:hashicorp/vault.git","clone_url":"https://github.com/hashicorp/vault.git","svn_url":"https://github.com/hashicorp/vault","homepage":"http://vaultproject.io","size":11209,"stargazers_count":3971,"watchers_count":3971,"language":"JavaScript","has_issues":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":322,"mirror_url":null,"open_issues_count":144,"forks":322,"open_issues":144,"watchers":3971,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/hashicorp/vault/pulls/977"},"html":{"href":"https://github.com/hashicorp/vault/pull/977"},"issue":{"href":"https://api.github.com/repos/hashicorp/vault/issues/977"},"comments":{"href":"https://api.github.com/repos/hashicorp/vault/issues/977/comments"},"review_comments":{"href":"https://api.github.com/repos/hashicorp/vault/pulls/977/comments"},"review_comment":{"href":"https://api.github.com/repos/hashicorp/vault/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/hashicorp/vault/pulls/977/commits"},"statuses":{"href":"https://api.github.com/repos/hashicorp/vault/statuses/934be180d67905f8b5b80710c91edbe53d3e989f"}}}}