{"action":"created","comment":{"url":"https://api.github.com/repos/m3db/m3/pulls/comments/342513950","pull_request_review_id":311670046,"id":342513950,"node_id":"MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDM0MjUxMzk1MA==","diff_hunk":"@@ -372,3 +376,66 @@ func (b bootstrapProcess) newRunOptions() RunOptions {\n \t\t).\n \t\tSetInitialTopologyState(b.initialTopologyState)\n }\n+\n+// NewNamespaces returns a new set of bootstrappable namespaces.\n+func NewNamespaces(\n+\tnamespaces []ProcessNamespace,\n+) Namespaces {\n+\tnamespacesMap := NewNamespacesMap(NamespacesMapOptions{})\n+\tfor _, ns := range namespaces {\n+\t\tnamespacesMap.Set(ns.Metadata.ID(), Namespace{\n+\t\t\tMetadata:        ns.Metadata,\n+\t\t\tShards:          ns.Shards,\n+\t\t\tDataAccumulator: ns.DataAccumulator,\n+\t\t})\n+\t}\n+\treturn Namespaces{\n+\t\tNamespaces: namespacesMap,\n+\t}\n+}\n+\n+// NewNamespaceResults creates a\n+// namespace results map with an entry for each\n+// namespace spoecified by a namespaces map.\n+func NewNamespaceResults(\n+\tnamespaces Namespaces,\n+) NamespaceResults {\n+\tresultsMap := NewNamespaceResultsMap(NamespaceResultsMapOptions{})\n+\tfor _, entry := range namespaces.Namespaces.Iter() {\n+\t\tkey := entry.Key()\n+\t\tvalue := entry.Value()\n+\t\tresultsMap.Set(key, NamespaceResult{\n+\t\t\tMetadata:    value.Metadata,\n+\t\t\tShards:      value.Shards,\n+\t\t\tDataResult:  result.NewDataBootstrapResult(),\n+\t\t\tIndexResult: result.NewIndexBootstrapResult(),\n+\t\t})\n+\t}\n+\treturn NamespaceResults{\n+\t\tResults: resultsMap,\n+\t}\n+}\n+\n+// MergeNamespaceResults merges two namespace results.\n+func MergeNamespaceResults(a, b NamespaceResults) NamespaceResults {","path":"src/dbnode/storage/bootstrap/process.go","position":330,"original_position":330,"commit_id":"f1380360221cfda5eb9e46c38200a76f6a59aa68","original_commit_id":"aa900f40f2147b7291a6983e86ebd86bc84c50ac","user":{"login":"robskillington","id":339615,"node_id":"MDQ6VXNlcjMzOTYxNQ==","avatar_url":"https://avatars1.githubusercontent.com/u/339615?v=4","gravatar_id":"","url":"https://api.github.com/users/robskillington","html_url":"https://github.com/robskillington","followers_url":"https://api.github.com/users/robskillington/followers","following_url":"https://api.github.com/users/robskillington/following{/other_user}","gists_url":"https://api.github.com/users/robskillington/gists{/gist_id}","starred_url":"https://api.github.com/users/robskillington/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/robskillington/subscriptions","organizations_url":"https://api.github.com/users/robskillington/orgs","repos_url":"https://api.github.com/users/robskillington/repos","events_url":"https://api.github.com/users/robskillington/events{/privacy}","received_events_url":"https://api.github.com/users/robskillington/received_events","type":"User","site_admin":false},"body":"They are both structs thankfully.  Let me add a comment about them mutating both (this is the contract the other MergeFoo(...) functions have.","created_at":"2019-11-05T11:32:35Z","updated_at":"2019-11-05T11:32:35Z","html_url":"https://github.com/m3db/m3/pull/1989#discussion_r342513950","pull_request_url":"https://api.github.com/repos/m3db/m3/pulls/1989","author_association":"COLLABORATOR","_links":{"self":{"href":"https://api.github.com/repos/m3db/m3/pulls/comments/342513950"},"html":{"href":"https://github.com/m3db/m3/pull/1989#discussion_r342513950"},"pull_request":{"href":"https://api.github.com/repos/m3db/m3/pulls/1989"}},"in_reply_to_id":339793355},"pull_request":{"url":"https://api.github.com/repos/m3db/m3/pulls/1989","id":325885456,"node_id":"MDExOlB1bGxSZXF1ZXN0MzI1ODg1NDU2","html_url":"https://github.com/m3db/m3/pull/1989","diff_url":"https://github.com/m3db/m3/pull/1989.diff","patch_url":"https://github.com/m3db/m3/pull/1989.patch","issue_url":"https://api.github.com/repos/m3db/m3/issues/1989","number":1989,"state":"open","locked":false,"title":"[dbnode] Single pass bootstrap and raw bootstrap memory/performance enhancements","user":{"login":"robskillington","id":339615,"node_id":"MDQ6VXNlcjMzOTYxNQ==","avatar_url":"https://avatars1.githubusercontent.com/u/339615?v=4","gravatar_id":"","url":"https://api.github.com/users/robskillington","html_url":"https://github.com/robskillington","followers_url":"https://api.github.com/users/robskillington/followers","following_url":"https://api.github.com/users/robskillington/following{/other_user}","gists_url":"https://api.github.com/users/robskillington/gists{/gist_id}","starred_url":"https://api.github.com/users/robskillington/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/robskillington/subscriptions","organizations_url":"https://api.github.com/users/robskillington/orgs","repos_url":"https://api.github.com/users/robskillington/repos","events_url":"https://api.github.com/users/robskillington/events{/privacy}","received_events_url":"https://api.github.com/users/robskillington/received_events","type":"User","site_admin":false},"body":"<!--  Thanks for sending a pull request!  Here are some tips for you:\r\n1. If this is your first time, please read our contributor guidelines: https://github.com/m3db/m3/blob/master/CONTRIBUTING.md and developer notes https://github.com/m3db/m3/blob/master/DEVELOPER.md\r\n2. Please prefix the name of the pull request with the component you are updating in the format \"[component] Change title\" (for example \"[dbnode] Support out of order writes\") and also label this pull request according to what type of issue you are addressing. Furthermore, if this is a WIP or DRAFT PR, please create a draft PR instead: https://github.blog/2019-02-14-introducing-draft-pull-requests/\r\n3. Ensure you have added or ran the appropriate tests for your PR: read more at https://github.com/m3db/m3/blob/master/DEVELOPER.md#testing-changes\r\n4. Follow the instructions for writing a changelog note: read more at https://github.com/m3db/m3/blob/master/DEVELOPER.md#adding-a-changelog\r\n-->\r\n\r\n**What this PR does / why we need it**:\r\n<!--\r\nIf you have an issue this change addresses, please add the following details:\r\n*Automatically closes linked issue when PR is merged.\r\nUsage: `Fixes #<issue number>`, or `Fixes (paste link of issue)`.\r\n-->\r\nThis change alters the bootstrap to perform work in a single pass, instead of multiplier of the number of namespaces multiplied by the number of types of bootstraps (just series data or series data and series index metadata).\r\n\r\nIf you had 3 namespaces and you use both the time series data and index metadata, that would previously have resulted in 6 different bootstrap passes.\r\n\r\nThe commit log bootstrapper also now writes data to an accumulator directly populating the database, avoiding having to stage any of the data and creating multiple copies of a series ID and tags (as part of bootstrapping + part of taking active writes). \r\n\r\nIt also avoids having to do any merging of out of order buffers as part of the bootstrapping phase, using the background tick to perform all these merges, the vast majority of which can be done lazily after a bootstrap is complete and the data is readable.\r\n\r\nAll these changes combined should greatly improve the time to bootstrap and the memory and CPU required to bootstrap a DB node.\r\n\r\n**Special notes for your reviewer**:\r\n\r\n**Does this PR introduce a user-facing and/or backwards incompatible change?**:\r\n<!--\r\nIf no, just write \"NONE\" in the release-note block below.\r\nIf yes, a release note is required:\r\nEnter your extended release note in the block below. If the PR requires additional action from users switching to the new release, include the string \"action required\".\r\n-->\r\n```release-note\r\nTBD\r\n```\r\n\r\n**Does this PR require updating code package or user-facing documentation?**:\r\n<!--\r\nIf no, just write \"NONE\" in the documentation-note block below.\r\nIf yes, describe which documentation you updated.\r\nNote: Any changes that significantly updates how a code package is architectured or functions requires code package documentation updates in the form of updates to the README.md file in that package.\r\n-->\r\n```documentation-note\r\nTBD\r\n```\r\n","created_at":"2019-10-08T17:42:22Z","updated_at":"2019-11-05T11:32:35Z","closed_at":null,"merged_at":null,"merge_commit_sha":"7f85bc05af28e06188030a0685e6811001fda72b","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/m3db/m3/pulls/1989/commits","review_comments_url":"https://api.github.com/repos/m3db/m3/pulls/1989/comments","review_comment_url":"https://api.github.com/repos/m3db/m3/pulls/comments{/number}","comments_url":"https://api.github.com/repos/m3db/m3/issues/1989/comments","statuses_url":"https://api.github.com/repos/m3db/m3/statuses/f1380360221cfda5eb9e46c38200a76f6a59aa68","head":{"label":"m3db:r/bootstrapping-changes","ref":"r/bootstrapping-changes","sha":"f1380360221cfda5eb9e46c38200a76f6a59aa68","user":{"login":"m3db","id":19937171,"node_id":"MDEyOk9yZ2FuaXphdGlvbjE5OTM3MTcx","avatar_url":"https://avatars0.githubusercontent.com/u/19937171?v=4","gravatar_id":"","url":"https://api.github.com/users/m3db","html_url":"https://github.com/m3db","followers_url":"https://api.github.com/users/m3db/followers","following_url":"https://api.github.com/users/m3db/following{/other_user}","gists_url":"https://api.github.com/users/m3db/gists{/gist_id}","starred_url":"https://api.github.com/users/m3db/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/m3db/subscriptions","organizations_url":"https://api.github.com/users/m3db/orgs","repos_url":"https://api.github.com/users/m3db/repos","events_url":"https://api.github.com/users/m3db/events{/privacy}","received_events_url":"https://api.github.com/users/m3db/received_events","type":"Organization","site_admin":false},"repo":{"id":61153677,"node_id":"MDEwOlJlcG9zaXRvcnk2MTE1MzY3Nw==","name":"m3","full_name":"m3db/m3","private":false,"owner":{"login":"m3db","id":19937171,"node_id":"MDEyOk9yZ2FuaXphdGlvbjE5OTM3MTcx","avatar_url":"https://avatars0.githubusercontent.com/u/19937171?v=4","gravatar_id":"","url":"https://api.github.com/users/m3db","html_url":"https://github.com/m3db","followers_url":"https://api.github.com/users/m3db/followers","following_url":"https://api.github.com/users/m3db/following{/other_user}","gists_url":"https://api.github.com/users/m3db/gists{/gist_id}","starred_url":"https://api.github.com/users/m3db/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/m3db/subscriptions","organizations_url":"https://api.github.com/users/m3db/orgs","repos_url":"https://api.github.com/users/m3db/repos","events_url":"https://api.github.com/users/m3db/events{/privacy}","received_events_url":"https://api.github.com/users/m3db/received_events","type":"Organization","site_admin":false},"html_url":"https://github.com/m3db/m3","description":"M3 monorepo - Distributed TSDB, Aggregator and Query Engine, Prometheus Sidecar, Graphite Compatible, Metrics Platform","fork":false,"url":"https://api.github.com/repos/m3db/m3","forks_url":"https://api.github.com/repos/m3db/m3/forks","keys_url":"https://api.github.com/repos/m3db/m3/keys{/key_id}","collaborators_url":"https://api.github.com/repos/m3db/m3/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/m3db/m3/teams","hooks_url":"https://api.github.com/repos/m3db/m3/hooks","issue_events_url":"https://api.github.com/repos/m3db/m3/issues/events{/number}","events_url":"https://api.github.com/repos/m3db/m3/events","assignees_url":"https://api.github.com/repos/m3db/m3/assignees{/user}","branches_url":"https://api.github.com/repos/m3db/m3/branches{/branch}","tags_url":"https://api.github.com/repos/m3db/m3/tags","blobs_url":"https://api.github.com/repos/m3db/m3/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/m3db/m3/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/m3db/m3/git/refs{/sha}","trees_url":"https://api.github.com/repos/m3db/m3/git/trees{/sha}","statuses_url":"https://api.github.com/repos/m3db/m3/statuses/{sha}","languages_url":"https://api.github.com/repos/m3db/m3/languages","stargazers_url":"https://api.github.com/repos/m3db/m3/stargazers","contributors_url":"https://api.github.com/repos/m3db/m3/contributors","subscribers_url":"https://api.github.com/repos/m3db/m3/subscribers","subscription_url":"https://api.github.com/repos/m3db/m3/subscription","commits_url":"https://api.github.com/repos/m3db/m3/commits{/sha}","git_commits_url":"https://api.github.com/repos/m3db/m3/git/commits{/sha}","comments_url":"https://api.github.com/repos/m3db/m3/comments{/number}","issue_comment_url":"https://api.github.com/repos/m3db/m3/issues/comments{/number}","contents_url":"https://api.github.com/repos/m3db/m3/contents/{+path}","compare_url":"https://api.github.com/repos/m3db/m3/compare/{base}...{head}","merges_url":"https://api.github.com/repos/m3db/m3/merges","archive_url":"https://api.github.com/repos/m3db/m3/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/m3db/m3/downloads","issues_url":"https://api.github.com/repos/m3db/m3/issues{/number}","pulls_url":"https://api.github.com/repos/m3db/m3/pulls{/number}","milestones_url":"https://api.github.com/repos/m3db/m3/milestones{/number}","notifications_url":"https://api.github.com/repos/m3db/m3/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/m3db/m3/labels{/name}","releases_url":"https://api.github.com/repos/m3db/m3/releases{/id}","deployments_url":"https://api.github.com/repos/m3db/m3/deployments","created_at":"2016-06-14T20:29:45Z","updated_at":"2019-11-05T11:07:54Z","pushed_at":"2019-11-05T02:27:16Z","git_url":"git://github.com/m3db/m3.git","ssh_url":"git@github.com:m3db/m3.git","clone_url":"https://github.com/m3db/m3.git","svn_url":"https://github.com/m3db/m3","homepage":"https://m3db.io/","size":49724,"stargazers_count":2257,"watchers_count":2257,"language":"Go","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":true,"forks_count":202,"mirror_url":null,"archived":false,"disabled":false,"open_issues_count":286,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0","node_id":"MDc6TGljZW5zZTI="},"forks":202,"open_issues":286,"watchers":2257,"default_branch":"master"}},"base":{"label":"m3db:master","ref":"master","sha":"0c6a0fe5c159869684c765e95ffb9db1109d431b","user":{"login":"m3db","id":19937171,"node_id":"MDEyOk9yZ2FuaXphdGlvbjE5OTM3MTcx","avatar_url":"https://avatars0.githubusercontent.com/u/19937171?v=4","gravatar_id":"","url":"https://api.github.com/users/m3db","html_url":"https://github.com/m3db","followers_url":"https://api.github.com/users/m3db/followers","following_url":"https://api.github.com/users/m3db/following{/other_user}","gists_url":"https://api.github.com/users/m3db/gists{/gist_id}","starred_url":"https://api.github.com/users/m3db/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/m3db/subscriptions","organizations_url":"https://api.github.com/users/m3db/orgs","repos_url":"https://api.github.com/users/m3db/repos","events_url":"https://api.github.com/users/m3db/events{/privacy}","received_events_url":"https://api.github.com/users/m3db/received_events","type":"Organization","site_admin":false},"repo":{"id":61153677,"node_id":"MDEwOlJlcG9zaXRvcnk2MTE1MzY3Nw==","name":"m3","full_name":"m3db/m3","private":false,"owner":{"login":"m3db","id":19937171,"node_id":"MDEyOk9yZ2FuaXphdGlvbjE5OTM3MTcx","avatar_url":"https://avatars0.githubusercontent.com/u/19937171?v=4","gravatar_id":"","url":"https://api.github.com/users/m3db","html_url":"https://github.com/m3db","followers_url":"https://api.github.com/users/m3db/followers","following_url":"https://api.github.com/users/m3db/following{/other_user}","gists_url":"https://api.github.com/users/m3db/gists{/gist_id}","starred_url":"https://api.github.com/users/m3db/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/m3db/subscriptions","organizations_url":"https://api.github.com/users/m3db/orgs","repos_url":"https://api.github.com/users/m3db/repos","events_url":"https://api.github.com/users/m3db/events{/privacy}","received_events_url":"https://api.github.com/users/m3db/received_events","type":"Organization","site_admin":false},"html_url":"https://github.com/m3db/m3","description":"M3 monorepo - Distributed TSDB, Aggregator and Query Engine, Prometheus Sidecar, Graphite Compatible, Metrics Platform","fork":false,"url":"https://api.github.com/repos/m3db/m3","forks_url":"https://api.github.com/repos/m3db/m3/forks","keys_url":"https://api.github.com/repos/m3db/m3/keys{/key_id}","collaborators_url":"https://api.github.com/repos/m3db/m3/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/m3db/m3/teams","hooks_url":"https://api.github.com/repos/m3db/m3/hooks","issue_events_url":"https://api.github.com/repos/m3db/m3/issues/events{/number}","events_url":"https://api.github.com/repos/m3db/m3/events","assignees_url":"https://api.github.com/repos/m3db/m3/assignees{/user}","branches_url":"https://api.github.com/repos/m3db/m3/branches{/branch}","tags_url":"https://api.github.com/repos/m3db/m3/tags","blobs_url":"https://api.github.com/repos/m3db/m3/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/m3db/m3/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/m3db/m3/git/refs{/sha}","trees_url":"https://api.github.com/repos/m3db/m3/git/trees{/sha}","statuses_url":"https://api.github.com/repos/m3db/m3/statuses/{sha}","languages_url":"https://api.github.com/repos/m3db/m3/languages","stargazers_url":"https://api.github.com/repos/m3db/m3/stargazers","contributors_url":"https://api.github.com/repos/m3db/m3/contributors","subscribers_url":"https://api.github.com/repos/m3db/m3/subscribers","subscription_url":"https://api.github.com/repos/m3db/m3/subscription","commits_url":"https://api.github.com/repos/m3db/m3/commits{/sha}","git_commits_url":"https://api.github.com/repos/m3db/m3/git/commits{/sha}","comments_url":"https://api.github.com/repos/m3db/m3/comments{/number}","issue_comment_url":"https://api.github.com/repos/m3db/m3/issues/comments{/number}","contents_url":"https://api.github.com/repos/m3db/m3/contents/{+path}","compare_url":"https://api.github.com/repos/m3db/m3/compare/{base}...{head}","merges_url":"https://api.github.com/repos/m3db/m3/merges","archive_url":"https://api.github.com/repos/m3db/m3/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/m3db/m3/downloads","issues_url":"https://api.github.com/repos/m3db/m3/issues{/number}","pulls_url":"https://api.github.com/repos/m3db/m3/pulls{/number}","milestones_url":"https://api.github.com/repos/m3db/m3/milestones{/number}","notifications_url":"https://api.github.com/repos/m3db/m3/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/m3db/m3/labels{/name}","releases_url":"https://api.github.com/repos/m3db/m3/releases{/id}","deployments_url":"https://api.github.com/repos/m3db/m3/deployments","created_at":"2016-06-14T20:29:45Z","updated_at":"2019-11-05T11:07:54Z","pushed_at":"2019-11-05T02:27:16Z","git_url":"git://github.com/m3db/m3.git","ssh_url":"git@github.com:m3db/m3.git","clone_url":"https://github.com/m3db/m3.git","svn_url":"https://github.com/m3db/m3","homepage":"https://m3db.io/","size":49724,"stargazers_count":2257,"watchers_count":2257,"language":"Go","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":true,"forks_count":202,"mirror_url":null,"archived":false,"disabled":false,"open_issues_count":286,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0","node_id":"MDc6TGljZW5zZTI="},"forks":202,"open_issues":286,"watchers":2257,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/m3db/m3/pulls/1989"},"html":{"href":"https://github.com/m3db/m3/pull/1989"},"issue":{"href":"https://api.github.com/repos/m3db/m3/issues/1989"},"comments":{"href":"https://api.github.com/repos/m3db/m3/issues/1989/comments"},"review_comments":{"href":"https://api.github.com/repos/m3db/m3/pulls/1989/comments"},"review_comment":{"href":"https://api.github.com/repos/m3db/m3/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/m3db/m3/pulls/1989/commits"},"statuses":{"href":"https://api.github.com/repos/m3db/m3/statuses/f1380360221cfda5eb9e46c38200a76f6a59aa68"}},"author_association":"COLLABORATOR"}}
{"actor":{"display_login":"robskillington"}}