{"action":"created","comment":{"url":"https://api.github.com/repos/ethcore/parity/pulls/comments/83451026","pull_request_review_id":4296238,"id":83451026,"diff_hunk":"@@ -0,0 +1,137 @@\n+// Copyright 2015, 2016 Ethcore (UK) Ltd.\n+// This file is part of Parity.\n+\n+// Parity is free software: you can redistribute it and/or modify\n+// it under the terms of the GNU General Public License as published by\n+// the Free Software Foundation, either version 3 of the License, or\n+// (at your option) any later version.\n+\n+// Parity is distributed in the hope that it will be useful,\n+// but WITHOUT ANY WARRANTY; without even the implied warranty of\n+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n+// GNU General Public License for more details.\n+\n+// You should have received a copy of the GNU General Public License\n+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.\n+\n+//! Bloom upgrade\n+\n+use db::{COL_EXTRA, COL_HEADERS, COL_STATE, NUM_COLUMNS, COL_ACCOUNT_BLOOM};\n+use state_db::{ACCOUNT_BLOOM_SPACE, DEFAULT_ACCOUNT_PRESET, StateDB, ACCOUNT_BLOOM_HASHCOUNT_KEY, ACCOUNT_BLOOM_SPACE_KEY};\n+use util::trie::TrieDB;\n+use views::HeaderView;\n+use bloom_journal::Bloom;\n+use util::migration::Error;\n+use util::{journaldb, Trie};\n+use util::{H256, FixedHash};\n+use util::{Database, DatabaseConfig, DBTransaction, CompactionProfile};\n+use std::path::Path;\n+use std::collections::HashMap;\n+use byteorder::{LittleEndian, ByteOrder};\n+\n+fn check_bloom_exists(db: &Database) -> bool {\n+\tlet hash_count_entry = db.get(COL_ACCOUNT_BLOOM, ACCOUNT_BLOOM_HASHCOUNT_KEY)\n+\t\t.expect(\"Low-level database error\");\n+\n+\thash_count_entry.is_some()\n+}\n+\n+fn check_space_match(db: &Database) -> Result<(), usize> {\n+\tlet db_space = db.get(COL_ACCOUNT_BLOOM, ACCOUNT_BLOOM_SPACE_KEY)\n+\t\t.expect(\"Low-level database error\")\n+\t\t.map(|val| LittleEndian::read_u64(&val[..]) as usize)\n+\t\t// this was the initial size of the bloom which was not written in the database\n+\t\t.unwrap_or(1048576);\n+\n+\tif db_space == ACCOUNT_BLOOM_SPACE { Ok(()) } else { Err(db_space) }\n+}\n+\n+/// Account bloom upgrade routine. If bloom already present, does nothing.\n+/// If database empty (no best block), does nothing.\n+/// Can be called on upgraded database with no issues (will do nothing).\n+pub fn upgrade_account_bloom(db_path: &Path) -> Result<(), Error> {\n+\tlet mut progress = ::util::migration::Progress::default();\n+\n+\tlet path = try!(db_path.to_str().ok_or(Error::MigrationImpossible));\n+\ttrace!(target: \"migration\", \"Account bloom upgrade at {:?}\", db_path);\n+\n+\tlet source = try!(Database::open(&DatabaseConfig {\n+\t\tmax_open_files: 64,\n+\t\tcache_sizes: HashMap::new(),\n+\t\tcompaction: CompactionProfile::default(),\n+\t\tcolumns: NUM_COLUMNS,\n+\t\twal: true,\n+\t}, path));\n+\n+\tlet best_block_hash = match try!(source.get(COL_EXTRA, b\"best\")) {\n+\t\t// no migration needed\n+\t\tNone => {\n+\t\t\ttrace!(target: \"migration\", \"No best block hash, skipping\");\n+\t\t\treturn Ok(());\n+\t\t},\n+\t\tSome(hash) => hash,\n+\t};\n+\tlet best_block_header = match try!(source.get(COL_HEADERS, &best_block_hash)) {\n+\t\t// no best block, nothing to do\n+\t\tNone => {\n+\t\t\ttrace!(target: \"migration\", \"No best block header, skipping\");\n+\t\t\treturn Ok(())\n+\t\t},\n+\t\tSome(x) => x,\n+\t};\n+\tlet state_root = HeaderView::new(&best_block_header).state_root();\n+\n+\tlet db = ::std::sync::Arc::new(source);\n+\tlet mut batch = DBTransaction::new(&db);\n+\n+\tif check_bloom_exists(&*db) {\n+\t\tmatch check_space_match(&*db) {\n+\t\t\tOk(_) => {\n+\t\t\t\t// bloom already exists and desired and stored spaces match, nothing to do\n+\t\t\t\ttrace!(target: \"migration\", \"Bloom already present of the right space, skipping\");\n+\t\t\t\treturn Ok(())\n+\t\t\t},\n+\t\t\tErr(wrong_size) => {\n+\t\t\t\t// nullify existing bloom entries\n+\t\t\t\ttrace!(target: \"migration\", \"Clearing old bloom of space {}\", &wrong_size);\n+\t\t\t\tlet mut key = [0u8; 8];\n+\t\t\t\tfor idx in 0..(wrong_size as u64/8) {\n+\t\t\t\t\tLittleEndian::write_u64(&mut key, idx);\n+\t\t\t\t\tbatch.put(COL_ACCOUNT_BLOOM, &key, &[0u8; 8]);\n+\n+\t\t\t\t\tif idx % 10000 == 1 { progress.tick(); };\n+\t\t\t\t}\n+\n+\t\t\t\tLittleEndian::write_u64(&mut key, ACCOUNT_BLOOM_SPACE as u64);\n+\t\t\t\tbatch.put(COL_ACCOUNT_BLOOM, ACCOUNT_BLOOM_SPACE_KEY, &key);\n+\t\t\t},\n+\t\t}\n+\t}\n+\n+\tprintln!(\"Adding/expanding accounts bloom (one-time upgrade)\");\n+\tlet bloom_journal = {\n+\t\tlet mut bloom = Bloom::new(ACCOUNT_BLOOM_SPACE, DEFAULT_ACCOUNT_PRESET);\n+\t\t// no difference what algorithm is passed, since there will be no writes\n+\t\tlet state_db = journaldb::new(","path":"ethcore/src/migrations/upgrade_bloom.rs","position":115,"original_position":115,"commit_id":"1d3934b2ff0d9fbb6258651d9cfd5a2ed0886e49","original_commit_id":"1d3934b2ff0d9fbb6258651d9cfd5a2ed0886e49","user":{"login":"NikVolf","id":3186518,"avatar_url":"https://avatars.githubusercontent.com/u/3186518?v=3","gravatar_id":"","url":"https://api.github.com/users/NikVolf","html_url":"https://github.com/NikVolf","followers_url":"https://api.github.com/users/NikVolf/followers","following_url":"https://api.github.com/users/NikVolf/following{/other_user}","gists_url":"https://api.github.com/users/NikVolf/gists{/gist_id}","starred_url":"https://api.github.com/users/NikVolf/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/NikVolf/subscriptions","organizations_url":"https://api.github.com/users/NikVolf/orgs","repos_url":"https://api.github.com/users/NikVolf/repos","events_url":"https://api.github.com/users/NikVolf/events{/privacy}","received_events_url":"https://api.github.com/users/NikVolf/received_events","type":"User","site_admin":false},"body":"why it will fail with this error?\r\nthere is only one state used there, which is for the best block, no?","created_at":"2016-10-14T15:56:57Z","updated_at":"2016-10-14T15:56:57Z","html_url":"https://github.com/ethcore/parity/pull/2624#discussion_r83451026","pull_request_url":"https://api.github.com/repos/ethcore/parity/pulls/2624","_links":{"self":{"href":"https://api.github.com/repos/ethcore/parity/pulls/comments/83451026"},"html":{"href":"https://github.com/ethcore/parity/pull/2624#discussion_r83451026"},"pull_request":{"href":"https://api.github.com/repos/ethcore/parity/pulls/2624"}}},"pull_request":{"url":"https://api.github.com/repos/ethcore/parity/pulls/2624","id":89362648,"html_url":"https://github.com/ethcore/parity/pull/2624","diff_url":"https://github.com/ethcore/parity/pull/2624.diff","patch_url":"https://github.com/ethcore/parity/pull/2624.patch","issue_url":"https://api.github.com/repos/ethcore/parity/issues/2624","number":2624,"state":"open","locked":false,"title":"Bloom upgrade in master","user":{"login":"NikVolf","id":3186518,"avatar_url":"https://avatars.githubusercontent.com/u/3186518?v=3","gravatar_id":"","url":"https://api.github.com/users/NikVolf","html_url":"https://github.com/NikVolf","followers_url":"https://api.github.com/users/NikVolf/followers","following_url":"https://api.github.com/users/NikVolf/following{/other_user}","gists_url":"https://api.github.com/users/NikVolf/gists{/gist_id}","starred_url":"https://api.github.com/users/NikVolf/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/NikVolf/subscriptions","organizations_url":"https://api.github.com/users/NikVolf/orgs","repos_url":"https://api.github.com/users/NikVolf/repos","events_url":"https://api.github.com/users/NikVolf/events{/privacy}","received_events_url":"https://api.github.com/users/NikVolf/received_events","type":"User","site_admin":false},"body":"had to move the same in-place upgrade logic that we had in beta","created_at":"2016-10-14T10:41:27Z","updated_at":"2016-10-14T15:56:57Z","closed_at":null,"merged_at":null,"merge_commit_sha":"6ae5e896ca9760de0df22fad7835fdee076754aa","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/ethcore/parity/pulls/2624/commits","review_comments_url":"https://api.github.com/repos/ethcore/parity/pulls/2624/comments","review_comment_url":"https://api.github.com/repos/ethcore/parity/pulls/comments{/number}","comments_url":"https://api.github.com/repos/ethcore/parity/issues/2624/comments","statuses_url":"https://api.github.com/repos/ethcore/parity/statuses/1d3934b2ff0d9fbb6258651d9cfd5a2ed0886e49","head":{"label":"ethcore:bloom-upgrade-master","ref":"bloom-upgrade-master","sha":"1d3934b2ff0d9fbb6258651d9cfd5a2ed0886e49","user":{"login":"ethcore","id":14176906,"avatar_url":"https://avatars.githubusercontent.com/u/14176906?v=3","gravatar_id":"","url":"https://api.github.com/users/ethcore","html_url":"https://github.com/ethcore","followers_url":"https://api.github.com/users/ethcore/followers","following_url":"https://api.github.com/users/ethcore/following{/other_user}","gists_url":"https://api.github.com/users/ethcore/gists{/gist_id}","starred_url":"https://api.github.com/users/ethcore/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ethcore/subscriptions","organizations_url":"https://api.github.com/users/ethcore/orgs","repos_url":"https://api.github.com/users/ethcore/repos","events_url":"https://api.github.com/users/ethcore/events{/privacy}","received_events_url":"https://api.github.com/users/ethcore/received_events","type":"Organization","site_admin":false},"repo":{"id":46714939,"name":"parity","full_name":"ethcore/parity","owner":{"login":"ethcore","id":14176906,"avatar_url":"https://avatars.githubusercontent.com/u/14176906?v=3","gravatar_id":"","url":"https://api.github.com/users/ethcore","html_url":"https://github.com/ethcore","followers_url":"https://api.github.com/users/ethcore/followers","following_url":"https://api.github.com/users/ethcore/following{/other_user}","gists_url":"https://api.github.com/users/ethcore/gists{/gist_id}","starred_url":"https://api.github.com/users/ethcore/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ethcore/subscriptions","organizations_url":"https://api.github.com/users/ethcore/orgs","repos_url":"https://api.github.com/users/ethcore/repos","events_url":"https://api.github.com/users/ethcore/events{/privacy}","received_events_url":"https://api.github.com/users/ethcore/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/ethcore/parity","description":"Fast, light, robust Ethereum implementation","fork":false,"url":"https://api.github.com/repos/ethcore/parity","forks_url":"https://api.github.com/repos/ethcore/parity/forks","keys_url":"https://api.github.com/repos/ethcore/parity/keys{/key_id}","collaborators_url":"https://api.github.com/repos/ethcore/parity/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/ethcore/parity/teams","hooks_url":"https://api.github.com/repos/ethcore/parity/hooks","issue_events_url":"https://api.github.com/repos/ethcore/parity/issues/events{/number}","events_url":"https://api.github.com/repos/ethcore/parity/events","assignees_url":"https://api.github.com/repos/ethcore/parity/assignees{/user}","branches_url":"https://api.github.com/repos/ethcore/parity/branches{/branch}","tags_url":"https://api.github.com/repos/ethcore/parity/tags","blobs_url":"https://api.github.com/repos/ethcore/parity/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/ethcore/parity/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/ethcore/parity/git/refs{/sha}","trees_url":"https://api.github.com/repos/ethcore/parity/git/trees{/sha}","statuses_url":"https://api.github.com/repos/ethcore/parity/statuses/{sha}","languages_url":"https://api.github.com/repos/ethcore/parity/languages","stargazers_url":"https://api.github.com/repos/ethcore/parity/stargazers","contributors_url":"https://api.github.com/repos/ethcore/parity/contributors","subscribers_url":"https://api.github.com/repos/ethcore/parity/subscribers","subscription_url":"https://api.github.com/repos/ethcore/parity/subscription","commits_url":"https://api.github.com/repos/ethcore/parity/commits{/sha}","git_commits_url":"https://api.github.com/repos/ethcore/parity/git/commits{/sha}","comments_url":"https://api.github.com/repos/ethcore/parity/comments{/number}","issue_comment_url":"https://api.github.com/repos/ethcore/parity/issues/comments{/number}","contents_url":"https://api.github.com/repos/ethcore/parity/contents/{+path}","compare_url":"https://api.github.com/repos/ethcore/parity/compare/{base}...{head}","merges_url":"https://api.github.com/repos/ethcore/parity/merges","archive_url":"https://api.github.com/repos/ethcore/parity/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/ethcore/parity/downloads","issues_url":"https://api.github.com/repos/ethcore/parity/issues{/number}","pulls_url":"https://api.github.com/repos/ethcore/parity/pulls{/number}","milestones_url":"https://api.github.com/repos/ethcore/parity/milestones{/number}","notifications_url":"https://api.github.com/repos/ethcore/parity/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/ethcore/parity/labels{/name}","releases_url":"https://api.github.com/repos/ethcore/parity/releases{/id}","deployments_url":"https://api.github.com/repos/ethcore/parity/deployments","created_at":"2015-11-23T11:07:32Z","updated_at":"2016-10-14T12:21:12Z","pushed_at":"2016-10-14T15:50:35Z","git_url":"git://github.com/ethcore/parity.git","ssh_url":"git@github.com:ethcore/parity.git","clone_url":"https://github.com/ethcore/parity.git","svn_url":"https://github.com/ethcore/parity","homepage":"","size":46669,"stargazers_count":606,"watchers_count":606,"language":"Rust","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":true,"forks_count":108,"mirror_url":null,"open_issues_count":208,"forks":108,"open_issues":208,"watchers":606,"default_branch":"master"}},"base":{"label":"ethcore:master","ref":"master","sha":"4581469e780498b6e179e6ccdde6d3a0f6daa040","user":{"login":"ethcore","id":14176906,"avatar_url":"https://avatars.githubusercontent.com/u/14176906?v=3","gravatar_id":"","url":"https://api.github.com/users/ethcore","html_url":"https://github.com/ethcore","followers_url":"https://api.github.com/users/ethcore/followers","following_url":"https://api.github.com/users/ethcore/following{/other_user}","gists_url":"https://api.github.com/users/ethcore/gists{/gist_id}","starred_url":"https://api.github.com/users/ethcore/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ethcore/subscriptions","organizations_url":"https://api.github.com/users/ethcore/orgs","repos_url":"https://api.github.com/users/ethcore/repos","events_url":"https://api.github.com/users/ethcore/events{/privacy}","received_events_url":"https://api.github.com/users/ethcore/received_events","type":"Organization","site_admin":false},"repo":{"id":46714939,"name":"parity","full_name":"ethcore/parity","owner":{"login":"ethcore","id":14176906,"avatar_url":"https://avatars.githubusercontent.com/u/14176906?v=3","gravatar_id":"","url":"https://api.github.com/users/ethcore","html_url":"https://github.com/ethcore","followers_url":"https://api.github.com/users/ethcore/followers","following_url":"https://api.github.com/users/ethcore/following{/other_user}","gists_url":"https://api.github.com/users/ethcore/gists{/gist_id}","starred_url":"https://api.github.com/users/ethcore/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ethcore/subscriptions","organizations_url":"https://api.github.com/users/ethcore/orgs","repos_url":"https://api.github.com/users/ethcore/repos","events_url":"https://api.github.com/users/ethcore/events{/privacy}","received_events_url":"https://api.github.com/users/ethcore/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/ethcore/parity","description":"Fast, light, robust Ethereum implementation","fork":false,"url":"https://api.github.com/repos/ethcore/parity","forks_url":"https://api.github.com/repos/ethcore/parity/forks","keys_url":"https://api.github.com/repos/ethcore/parity/keys{/key_id}","collaborators_url":"https://api.github.com/repos/ethcore/parity/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/ethcore/parity/teams","hooks_url":"https://api.github.com/repos/ethcore/parity/hooks","issue_events_url":"https://api.github.com/repos/ethcore/parity/issues/events{/number}","events_url":"https://api.github.com/repos/ethcore/parity/events","assignees_url":"https://api.github.com/repos/ethcore/parity/assignees{/user}","branches_url":"https://api.github.com/repos/ethcore/parity/branches{/branch}","tags_url":"https://api.github.com/repos/ethcore/parity/tags","blobs_url":"https://api.github.com/repos/ethcore/parity/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/ethcore/parity/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/ethcore/parity/git/refs{/sha}","trees_url":"https://api.github.com/repos/ethcore/parity/git/trees{/sha}","statuses_url":"https://api.github.com/repos/ethcore/parity/statuses/{sha}","languages_url":"https://api.github.com/repos/ethcore/parity/languages","stargazers_url":"https://api.github.com/repos/ethcore/parity/stargazers","contributors_url":"https://api.github.com/repos/ethcore/parity/contributors","subscribers_url":"https://api.github.com/repos/ethcore/parity/subscribers","subscription_url":"https://api.github.com/repos/ethcore/parity/subscription","commits_url":"https://api.github.com/repos/ethcore/parity/commits{/sha}","git_commits_url":"https://api.github.com/repos/ethcore/parity/git/commits{/sha}","comments_url":"https://api.github.com/repos/ethcore/parity/comments{/number}","issue_comment_url":"https://api.github.com/repos/ethcore/parity/issues/comments{/number}","contents_url":"https://api.github.com/repos/ethcore/parity/contents/{+path}","compare_url":"https://api.github.com/repos/ethcore/parity/compare/{base}...{head}","merges_url":"https://api.github.com/repos/ethcore/parity/merges","archive_url":"https://api.github.com/repos/ethcore/parity/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/ethcore/parity/downloads","issues_url":"https://api.github.com/repos/ethcore/parity/issues{/number}","pulls_url":"https://api.github.com/repos/ethcore/parity/pulls{/number}","milestones_url":"https://api.github.com/repos/ethcore/parity/milestones{/number}","notifications_url":"https://api.github.com/repos/ethcore/parity/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/ethcore/parity/labels{/name}","releases_url":"https://api.github.com/repos/ethcore/parity/releases{/id}","deployments_url":"https://api.github.com/repos/ethcore/parity/deployments","created_at":"2015-11-23T11:07:32Z","updated_at":"2016-10-14T12:21:12Z","pushed_at":"2016-10-14T15:50:35Z","git_url":"git://github.com/ethcore/parity.git","ssh_url":"git@github.com:ethcore/parity.git","clone_url":"https://github.com/ethcore/parity.git","svn_url":"https://github.com/ethcore/parity","homepage":"","size":46669,"stargazers_count":606,"watchers_count":606,"language":"Rust","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":true,"forks_count":108,"mirror_url":null,"open_issues_count":208,"forks":108,"open_issues":208,"watchers":606,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/ethcore/parity/pulls/2624"},"html":{"href":"https://github.com/ethcore/parity/pull/2624"},"issue":{"href":"https://api.github.com/repos/ethcore/parity/issues/2624"},"comments":{"href":"https://api.github.com/repos/ethcore/parity/issues/2624/comments"},"review_comments":{"href":"https://api.github.com/repos/ethcore/parity/pulls/2624/comments"},"review_comment":{"href":"https://api.github.com/repos/ethcore/parity/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/ethcore/parity/pulls/2624/commits"},"statuses":{"href":"https://api.github.com/repos/ethcore/parity/statuses/1d3934b2ff0d9fbb6258651d9cfd5a2ed0886e49"}}}}
{"actor":{"display_login":"NikVolf"}}