{"action":"created","comment":{"url":"https://api.github.com/repos/baidu/bfs/pulls/comments/80401573","id":80401573,"diff_hunk":"@@ -685,28 +694,82 @@ bool ChunkServerManager::GetShutdownChunkServerStat() {\n     return !chunkservers_to_offline_.empty();\n }\n \n-bool ChunkServerManager::GetChunkServerBlockMapPtr(int32_t cs_id, ChunkServerBlockMap** cs_block_map)\n+bool ChunkServerManager::GetChunkServerBlockMapPtr(std::map<int32_t, ChunkServerBlockMap*>& src_map,\n+                                                   int32_t cs_id, ChunkServerBlockMap** cs_block_map)\n {\n     MutexLock lock(&mu_);\n-    std::map<int32_t, ChunkServerBlockMap*>::iterator it = chunkserver_block_map_.find(cs_id);\n-    if (it == chunkserver_block_map_.end()) {\n+    std::map<int32_t, ChunkServerBlockMap*>::iterator it = src_map.find(cs_id);\n+    if (it == src_map.end()) {\n         *cs_block_map = NULL;\n         return false;\n     }\n     *cs_block_map = it->second;\n     return true;\n }\n \n-void ChunkServerManager::AddBlock(int32_t id, const::google::protobuf::RepeatedPtrField<ReportBlockInfo>& blocks) {\n+int64_t ChunkServerManager::AddBlock(int32_t id, const std::set<int64_t>& blocks,\n+                                  int64_t start, int64_t end, std::vector<int64_t>* lost,\n+                                  int64_t report_id) {\n     ChunkServerBlockMap* cs_block_map = NULL;\n-    if (!GetChunkServerBlockMapPtr(id, &cs_block_map)) {\n+    if (!GetChunkServerBlockMapPtr(chunkserver_block_map_, id, &cs_block_map)) {\n         LOG(WARNING, \"Can't find chunkserver C%d\", id);\n-        return;\n+        return report_id;\n     }\n     MutexLock lock(cs_block_map->mu);\n-    for (int i = 0; i < blocks.size(); i++) {\n-        cs_block_map->blocks.insert(blocks.Get(i).block_id());\n+    std::set<int64_t>* ns_blocks = &cs_block_map->blocks;\n+    bool pass_check = true;\n+    for (std::set<int64_t>::iterator it = blocks.begin(); it != blocks.end(); ++it) {\n+        pass_check &= ns_blocks->insert(*it).second;\n+    }\n+    if (pass_check) {\n+        LOG(DEBUG, \"C%d pass block check\", id);\n+        return report_id;\n+    }\n+    if (report_id != -1 && report_id <= cs_block_map->report_id) {\n+        LOG(INFO, \"Report out-date C%d current_id %ld report_id %ld\",\n+                id, cs_block_map->report_id, report_id);\n+        return cs_block_map->report_id;\n+    }\n+    cs_block_map->report_id = report_id;\n+\n+    std::set<int64_t>::iterator ns_it = ns_blocks->lower_bound(start);\n+    std::set<int64_t>::iterator cs_it = blocks.begin();\n+    int count = 0;\n+    // cs_blocks is a subset of ns_blocks, cause all cs_blocks have been inserted into ns_blocks\n+    while (cs_it != blocks.end() && ns_it != ns_blocks->end() && *ns_it <= end) {\n+        ++count;\n+        while (ns_it != ns_blocks->end() && *cs_it > *ns_it) {\n+            LOG(WARNING, \"Check Block for C%d missing %ld \", id, *ns_it);\n+            lost->push_back(*ns_it);\n+            ++ns_it;\n+        }\n+        if (ns_it == ns_blocks->end()) {\n+            break;\n+        }\n+        if (*cs_it != *ns_it) {\n+            LOG(WARNING, \"Check failed for C%d, mismatch ns = %ld cs = %ld\", id, *ns_it, *cs_it);\n+            return report_id;\n+        }\n+        ++ns_it;\n+        ++cs_it;\n+    }\n+    while (ns_it != ns_blocks->end() && *ns_it <= end) {\n+        lost->push_back(*ns_it);\n+        ++ns_it;\n+    }\n+    ChunkServerBlockMap* delta_block_map = NULL;\n+    if (!GetChunkServerBlockMapPtr(chunkserver_block_delta_, id, &delta_block_map)) {\n+        LOG(WARNING, \"Can't find chunkserver C%d\", id);\n+        return report_id;\n+    }\n+    delta_block_map->mu->Lock();\n+    std::set<int64_t> delta_blocks;\n+    std::swap(delta_blocks, delta_block_map->blocks);\n+    delta_block_map->mu->Unlock();\n+    for (std::set<int64_t>::iterator it = delta_blocks.begin(); it != delta_blocks.end(); ++it) {","path":"src/nameserver/chunkserver_manager.cc","position":135,"original_position":135,"commit_id":"51455facc4652cb786afcd1c93333a2e2974ab29","original_commit_id":"51455facc4652cb786afcd1c93333a2e2974ab29","user":{"login":"yvxiang","id":1382038,"avatar_url":"https://avatars.githubusercontent.com/u/1382038?v=3","gravatar_id":"","url":"https://api.github.com/users/yvxiang","html_url":"https://github.com/yvxiang","followers_url":"https://api.github.com/users/yvxiang/followers","following_url":"https://api.github.com/users/yvxiang/following{/other_user}","gists_url":"https://api.github.com/users/yvxiang/gists{/gist_id}","starred_url":"https://api.github.com/users/yvxiang/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/yvxiang/subscriptions","organizations_url":"https://api.github.com/users/yvxiang/orgs","repos_url":"https://api.github.com/users/yvxiang/repos","events_url":"https://api.github.com/users/yvxiang/events{/privacy}","received_events_url":"https://api.github.com/users/yvxiang/received_events","type":"User","site_admin":false},"body":"锁外是不是不安全，如果这时有正常的AddBlock/RemoveBlock操作","created_at":"2016-09-26T02:40:34Z","updated_at":"2016-09-26T02:40:34Z","html_url":"https://github.com/baidu/bfs/pull/482#discussion_r80401573","pull_request_url":"https://api.github.com/repos/baidu/bfs/pulls/482","_links":{"self":{"href":"https://api.github.com/repos/baidu/bfs/pulls/comments/80401573"},"html":{"href":"https://github.com/baidu/bfs/pull/482#discussion_r80401573"},"pull_request":{"href":"https://api.github.com/repos/baidu/bfs/pulls/482"}}},"pull_request":{"url":"https://api.github.com/repos/baidu/bfs/pulls/482","id":85700306,"html_url":"https://github.com/baidu/bfs/pull/482","diff_url":"https://github.com/baidu/bfs/pull/482.diff","patch_url":"https://github.com/baidu/bfs/pull/482.patch","issue_url":"https://api.github.com/repos/baidu/bfs/issues/482","number":482,"state":"open","locked":false,"title":"check lost blocks during block report","user":{"login":"lylei","id":2767333,"avatar_url":"https://avatars.githubusercontent.com/u/2767333?v=3","gravatar_id":"","url":"https://api.github.com/users/lylei","html_url":"https://github.com/lylei","followers_url":"https://api.github.com/users/lylei/followers","following_url":"https://api.github.com/users/lylei/following{/other_user}","gists_url":"https://api.github.com/users/lylei/gists{/gist_id}","starred_url":"https://api.github.com/users/lylei/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/lylei/subscriptions","organizations_url":"https://api.github.com/users/lylei/orgs","repos_url":"https://api.github.com/users/lylei/repos","events_url":"https://api.github.com/users/lylei/events{/privacy}","received_events_url":"https://api.github.com/users/lylei/received_events","type":"User","site_admin":false},"body":"#474","created_at":"2016-09-18T07:13:25Z","updated_at":"2016-09-26T02:40:34Z","closed_at":null,"merged_at":null,"merge_commit_sha":"b6fff6729d97d52e34bd64465c519eb36c2f9c15","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/baidu/bfs/pulls/482/commits","review_comments_url":"https://api.github.com/repos/baidu/bfs/pulls/482/comments","review_comment_url":"https://api.github.com/repos/baidu/bfs/pulls/comments{/number}","comments_url":"https://api.github.com/repos/baidu/bfs/issues/482/comments","statuses_url":"https://api.github.com/repos/baidu/bfs/statuses/51455facc4652cb786afcd1c93333a2e2974ab29","head":{"label":"lylei:dead","ref":"dead","sha":"51455facc4652cb786afcd1c93333a2e2974ab29","user":{"login":"lylei","id":2767333,"avatar_url":"https://avatars.githubusercontent.com/u/2767333?v=3","gravatar_id":"","url":"https://api.github.com/users/lylei","html_url":"https://github.com/lylei","followers_url":"https://api.github.com/users/lylei/followers","following_url":"https://api.github.com/users/lylei/following{/other_user}","gists_url":"https://api.github.com/users/lylei/gists{/gist_id}","starred_url":"https://api.github.com/users/lylei/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/lylei/subscriptions","organizations_url":"https://api.github.com/users/lylei/orgs","repos_url":"https://api.github.com/users/lylei/repos","events_url":"https://api.github.com/users/lylei/events{/privacy}","received_events_url":"https://api.github.com/users/lylei/received_events","type":"User","site_admin":false},"repo":{"id":48720316,"name":"dfs","full_name":"lylei/dfs","owner":{"login":"lylei","id":2767333,"avatar_url":"https://avatars.githubusercontent.com/u/2767333?v=3","gravatar_id":"","url":"https://api.github.com/users/lylei","html_url":"https://github.com/lylei","followers_url":"https://api.github.com/users/lylei/followers","following_url":"https://api.github.com/users/lylei/following{/other_user}","gists_url":"https://api.github.com/users/lylei/gists{/gist_id}","starred_url":"https://api.github.com/users/lylei/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/lylei/subscriptions","organizations_url":"https://api.github.com/users/lylei/orgs","repos_url":"https://api.github.com/users/lylei/repos","events_url":"https://api.github.com/users/lylei/events{/privacy}","received_events_url":"https://api.github.com/users/lylei/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/lylei/dfs","description":"突然想写个分布式文件系统~","fork":true,"url":"https://api.github.com/repos/lylei/dfs","forks_url":"https://api.github.com/repos/lylei/dfs/forks","keys_url":"https://api.github.com/repos/lylei/dfs/keys{/key_id}","collaborators_url":"https://api.github.com/repos/lylei/dfs/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/lylei/dfs/teams","hooks_url":"https://api.github.com/repos/lylei/dfs/hooks","issue_events_url":"https://api.github.com/repos/lylei/dfs/issues/events{/number}","events_url":"https://api.github.com/repos/lylei/dfs/events","assignees_url":"https://api.github.com/repos/lylei/dfs/assignees{/user}","branches_url":"https://api.github.com/repos/lylei/dfs/branches{/branch}","tags_url":"https://api.github.com/repos/lylei/dfs/tags","blobs_url":"https://api.github.com/repos/lylei/dfs/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/lylei/dfs/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/lylei/dfs/git/refs{/sha}","trees_url":"https://api.github.com/repos/lylei/dfs/git/trees{/sha}","statuses_url":"https://api.github.com/repos/lylei/dfs/statuses/{sha}","languages_url":"https://api.github.com/repos/lylei/dfs/languages","stargazers_url":"https://api.github.com/repos/lylei/dfs/stargazers","contributors_url":"https://api.github.com/repos/lylei/dfs/contributors","subscribers_url":"https://api.github.com/repos/lylei/dfs/subscribers","subscription_url":"https://api.github.com/repos/lylei/dfs/subscription","commits_url":"https://api.github.com/repos/lylei/dfs/commits{/sha}","git_commits_url":"https://api.github.com/repos/lylei/dfs/git/commits{/sha}","comments_url":"https://api.github.com/repos/lylei/dfs/comments{/number}","issue_comment_url":"https://api.github.com/repos/lylei/dfs/issues/comments{/number}","contents_url":"https://api.github.com/repos/lylei/dfs/contents/{+path}","compare_url":"https://api.github.com/repos/lylei/dfs/compare/{base}...{head}","merges_url":"https://api.github.com/repos/lylei/dfs/merges","archive_url":"https://api.github.com/repos/lylei/dfs/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/lylei/dfs/downloads","issues_url":"https://api.github.com/repos/lylei/dfs/issues{/number}","pulls_url":"https://api.github.com/repos/lylei/dfs/pulls{/number}","milestones_url":"https://api.github.com/repos/lylei/dfs/milestones{/number}","notifications_url":"https://api.github.com/repos/lylei/dfs/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/lylei/dfs/labels{/name}","releases_url":"https://api.github.com/repos/lylei/dfs/releases{/id}","deployments_url":"https://api.github.com/repos/lylei/dfs/deployments","created_at":"2015-12-29T01:20:31Z","updated_at":"2016-02-03T08:42:36Z","pushed_at":"2016-09-23T06:40:04Z","git_url":"git://github.com/lylei/dfs.git","ssh_url":"git@github.com:lylei/dfs.git","clone_url":"https://github.com/lylei/dfs.git","svn_url":"https://github.com/lylei/dfs","homepage":null,"size":11239,"stargazers_count":1,"watchers_count":1,"language":"C++","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":1,"mirror_url":null,"open_issues_count":0,"forks":1,"open_issues":0,"watchers":1,"default_branch":"master"}},"base":{"label":"baidu:master","ref":"master","sha":"b5c0ec2de033b2ada61560f096b08fb2a3079d5b","user":{"login":"baidu","id":13245940,"avatar_url":"https://avatars.githubusercontent.com/u/13245940?v=3","gravatar_id":"","url":"https://api.github.com/users/baidu","html_url":"https://github.com/baidu","followers_url":"https://api.github.com/users/baidu/followers","following_url":"https://api.github.com/users/baidu/following{/other_user}","gists_url":"https://api.github.com/users/baidu/gists{/gist_id}","starred_url":"https://api.github.com/users/baidu/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/baidu/subscriptions","organizations_url":"https://api.github.com/users/baidu/orgs","repos_url":"https://api.github.com/users/baidu/repos","events_url":"https://api.github.com/users/baidu/events{/privacy}","received_events_url":"https://api.github.com/users/baidu/received_events","type":"Organization","site_admin":false},"repo":{"id":26562172,"name":"bfs","full_name":"baidu/bfs","owner":{"login":"baidu","id":13245940,"avatar_url":"https://avatars.githubusercontent.com/u/13245940?v=3","gravatar_id":"","url":"https://api.github.com/users/baidu","html_url":"https://github.com/baidu","followers_url":"https://api.github.com/users/baidu/followers","following_url":"https://api.github.com/users/baidu/following{/other_user}","gists_url":"https://api.github.com/users/baidu/gists{/gist_id}","starred_url":"https://api.github.com/users/baidu/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/baidu/subscriptions","organizations_url":"https://api.github.com/users/baidu/orgs","repos_url":"https://api.github.com/users/baidu/repos","events_url":"https://api.github.com/users/baidu/events{/privacy}","received_events_url":"https://api.github.com/users/baidu/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/baidu/bfs","description":"The Baidu File System.","fork":false,"url":"https://api.github.com/repos/baidu/bfs","forks_url":"https://api.github.com/repos/baidu/bfs/forks","keys_url":"https://api.github.com/repos/baidu/bfs/keys{/key_id}","collaborators_url":"https://api.github.com/repos/baidu/bfs/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/baidu/bfs/teams","hooks_url":"https://api.github.com/repos/baidu/bfs/hooks","issue_events_url":"https://api.github.com/repos/baidu/bfs/issues/events{/number}","events_url":"https://api.github.com/repos/baidu/bfs/events","assignees_url":"https://api.github.com/repos/baidu/bfs/assignees{/user}","branches_url":"https://api.github.com/repos/baidu/bfs/branches{/branch}","tags_url":"https://api.github.com/repos/baidu/bfs/tags","blobs_url":"https://api.github.com/repos/baidu/bfs/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/baidu/bfs/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/baidu/bfs/git/refs{/sha}","trees_url":"https://api.github.com/repos/baidu/bfs/git/trees{/sha}","statuses_url":"https://api.github.com/repos/baidu/bfs/statuses/{sha}","languages_url":"https://api.github.com/repos/baidu/bfs/languages","stargazers_url":"https://api.github.com/repos/baidu/bfs/stargazers","contributors_url":"https://api.github.com/repos/baidu/bfs/contributors","subscribers_url":"https://api.github.com/repos/baidu/bfs/subscribers","subscription_url":"https://api.github.com/repos/baidu/bfs/subscription","commits_url":"https://api.github.com/repos/baidu/bfs/commits{/sha}","git_commits_url":"https://api.github.com/repos/baidu/bfs/git/commits{/sha}","comments_url":"https://api.github.com/repos/baidu/bfs/comments{/number}","issue_comment_url":"https://api.github.com/repos/baidu/bfs/issues/comments{/number}","contents_url":"https://api.github.com/repos/baidu/bfs/contents/{+path}","compare_url":"https://api.github.com/repos/baidu/bfs/compare/{base}...{head}","merges_url":"https://api.github.com/repos/baidu/bfs/merges","archive_url":"https://api.github.com/repos/baidu/bfs/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/baidu/bfs/downloads","issues_url":"https://api.github.com/repos/baidu/bfs/issues{/number}","pulls_url":"https://api.github.com/repos/baidu/bfs/pulls{/number}","milestones_url":"https://api.github.com/repos/baidu/bfs/milestones{/number}","notifications_url":"https://api.github.com/repos/baidu/bfs/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/baidu/bfs/labels{/name}","releases_url":"https://api.github.com/repos/baidu/bfs/releases{/id}","deployments_url":"https://api.github.com/repos/baidu/bfs/deployments","created_at":"2014-11-13T00:00:29Z","updated_at":"2016-09-25T14:44:09Z","pushed_at":"2016-09-26T01:59:49Z","git_url":"git://github.com/baidu/bfs.git","ssh_url":"git@github.com:baidu/bfs.git","clone_url":"https://github.com/baidu/bfs.git","svn_url":"https://github.com/baidu/bfs","homepage":"","size":12108,"stargazers_count":87,"watchers_count":87,"language":"C++","has_issues":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":40,"mirror_url":null,"open_issues_count":62,"forks":40,"open_issues":62,"watchers":87,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/baidu/bfs/pulls/482"},"html":{"href":"https://github.com/baidu/bfs/pull/482"},"issue":{"href":"https://api.github.com/repos/baidu/bfs/issues/482"},"comments":{"href":"https://api.github.com/repos/baidu/bfs/issues/482/comments"},"review_comments":{"href":"https://api.github.com/repos/baidu/bfs/pulls/482/comments"},"review_comment":{"href":"https://api.github.com/repos/baidu/bfs/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/baidu/bfs/pulls/482/commits"},"statuses":{"href":"https://api.github.com/repos/baidu/bfs/statuses/51455facc4652cb786afcd1c93333a2e2974ab29"}}}}
{"actor":{"display_login":"yvxiang"}}