{"action":"created","comment":{"url":"https://api.github.com/repos/LeelaChessZero/lc0/pulls/comments/202514978","pull_request_review_id":137238085,"id":202514978,"node_id":"MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDIwMjUxNDk3OA==","diff_hunk":"@@ -46,25 +50,89 @@ FloatVectors LoadFloatsFromFile(const std::string& filename) {\n     } else {\n       bytes_read += sz;\n       buffer.resize(bytes_read);\n-      // Add newline in the end for the case it was not there.\n-      buffer.push_back('\\n');\n       break;\n     }\n   }\n   gzclose(file);\n \n+  return buffer;\n+}\n+\n+void DenormLayer(const lc0::Weights_Layer& layer, FloatVectors& vecs) {\n+  FloatVector v;\n+  auto& buffer = layer.params();\n+  auto data = reinterpret_cast<const std::uint16_t*>(buffer.data());\n+  int n = buffer.length() / 2;\n+  v.resize(n);\n+  for (int i = 0; i < n; i++) {\n+    v[i] = data[i] / float(0xffff);\n+    v[i] *= layer.max_val() - layer.min_val();\n+    v[i] += layer.min_val();\n+  }\n+  vecs.emplace_back(v);\n+}\n+\n+void DenormConvBlock(const lc0::Weights_ConvBlock& conv, FloatVectors& vecs) {\n+  DenormLayer(conv.weights(), vecs);\n+  DenormLayer(conv.biases(), vecs);\n+  DenormLayer(conv.bn_means(), vecs);\n+  DenormLayer(conv.bn_stddivs(), vecs);\n+}\n+\n+FloatVectors LoadFloatsFromPbFile(const std::string& buffer) {\n+  auto net = lc0::Net();\n+  FloatVectors vecs;\n+  net.ParseFromString(buffer);\n+\n+  std::string min_version(std::to_string(net.min_version().major()) + \".\");\n+  min_version += std::to_string(net.min_version().minor()) + \".\";\n+  min_version += std::to_string(net.min_version().patch());\n+\n+  if (net.min_version().major() > LC0_VERSION_MAJOR)\n+    throw Exception(\"Weights require at least lc0 version: \" + min_version);\n+  if (net.min_version().minor() > LC0_VERSION_MINOR)\n+    throw Exception(\"Weights require at least lc0 version: \" + min_version);\n+  if (net.min_version().patch() > LC0_VERSION_PATCH)\n+    throw Exception(\"Weights require at least lc0 version: \" + min_version);\n+\n+  if (net.format().weights_encoding() != lc0::Format::LINEAR16)\n+    throw Exception(\"Invalid weight encoding\");\n+\n+  auto& w = net.weights();\n+\n+  DenormConvBlock(w.input(), vecs);\n+\n+  for (int i = 0, n = w.residual_size(); i < n; i++) {\n+    DenormConvBlock(w.residual(i).conv1(), vecs);\n+    DenormConvBlock(w.residual(i).conv2(), vecs);\n+  }\n+\n+  DenormConvBlock(w.policy(), vecs);\n+  DenormLayer(w.ip_pol_w(), vecs);\n+  DenormLayer(w.ip_pol_b(), vecs);\n+  DenormConvBlock(w.value(), vecs);\n+  DenormLayer(w.ip1_val_w(), vecs);\n+  DenormLayer(w.ip1_val_b(), vecs);\n+  DenormLayer(w.ip2_val_w(), vecs);\n+  DenormLayer(w.ip2_val_b(), vecs);\n+\n+  return vecs;\n+}\n+\n+FloatVectors LoadFloatsFromFile(std::string* buffer) {","path":"src/neural/loader.cc","position":97,"original_position":97,"commit_id":"0f6f21a17725141132d1486409c33a9a2bd6ee2b","original_commit_id":"0f6f21a17725141132d1486409c33a9a2bd6ee2b","user":{"login":"mooskagh","id":11266455,"node_id":"MDQ6VXNlcjExMjY2NDU1","avatar_url":"https://avatars3.githubusercontent.com/u/11266455?v=4","gravatar_id":"","url":"https://api.github.com/users/mooskagh","html_url":"https://github.com/mooskagh","followers_url":"https://api.github.com/users/mooskagh/followers","following_url":"https://api.github.com/users/mooskagh/following{/other_user}","gists_url":"https://api.github.com/users/mooskagh/gists{/gist_id}","starred_url":"https://api.github.com/users/mooskagh/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mooskagh/subscriptions","organizations_url":"https://api.github.com/users/mooskagh/orgs","repos_url":"https://api.github.com/users/mooskagh/repos","events_url":"https://api.github.com/users/mooskagh/events{/privacy}","received_events_url":"https://api.github.com/users/mooskagh/received_events","type":"User","site_admin":false},"body":"It's better to make buffer parameter a const reference.\r\nIt's a really weird side effect of a function that it adds '\\n'.\r\n\r\nProbably better to remove that and add after the loop:\r\n```cpp\r\n  if (!line.empty()) {\r\n       result.emplace_back();\r\n       result.back().swap(line);\r\n  }\r\n```","created_at":"2018-07-14T13:28:39Z","updated_at":"2018-07-14T13:37:43Z","html_url":"https://github.com/LeelaChessZero/lc0/pull/166#discussion_r202514978","pull_request_url":"https://api.github.com/repos/LeelaChessZero/lc0/pulls/166","author_association":"MEMBER","_links":{"self":{"href":"https://api.github.com/repos/LeelaChessZero/lc0/pulls/comments/202514978"},"html":{"href":"https://github.com/LeelaChessZero/lc0/pull/166#discussion_r202514978"},"pull_request":{"href":"https://api.github.com/repos/LeelaChessZero/lc0/pulls/166"}}},"pull_request":{"url":"https://api.github.com/repos/LeelaChessZero/lc0/pulls/166","id":201004948,"node_id":"MDExOlB1bGxSZXF1ZXN0MjAxMDA0OTQ4","html_url":"https://github.com/LeelaChessZero/lc0/pull/166","diff_url":"https://github.com/LeelaChessZero/lc0/pull/166.diff","patch_url":"https://github.com/LeelaChessZero/lc0/pull/166.patch","issue_url":"https://api.github.com/repos/LeelaChessZero/lc0/issues/166","number":166,"state":"open","locked":false,"title":"gzipped protobuf compressed neural networks","user":{"login":"Error323","id":76089,"node_id":"MDQ6VXNlcjc2MDg5","avatar_url":"https://avatars0.githubusercontent.com/u/76089?v=4","gravatar_id":"","url":"https://api.github.com/users/Error323","html_url":"https://github.com/Error323","followers_url":"https://api.github.com/users/Error323/followers","following_url":"https://api.github.com/users/Error323/following{/other_user}","gists_url":"https://api.github.com/users/Error323/gists{/gist_id}","starred_url":"https://api.github.com/users/Error323/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Error323/subscriptions","organizations_url":"https://api.github.com/users/Error323/orgs","repos_url":"https://api.github.com/users/Error323/repos","events_url":"https://api.github.com/users/Error323/events{/privacy}","received_events_url":"https://api.github.com/users/Error323/received_events","type":"User","site_admin":false},"body":"Tested on testserver network 10068 `lc0 -w 10068.pb.gz --verbose-move-stats --threads=1`\r\n\r\n```\r\n32bit txt\r\ninfo string c2c4  (264 ) N:    1239 (+ 0) (P: 12.50%) (Q:  0.02736) (U: 0.03301) (Q+U:  0.06037) (V: -0.0142) \r\ninfo string d2d4  (293 ) N:    1451 (+ 0) (P:  9.75%) (Q:  0.03747) (U: 0.02198) (Q+U:  0.05945) (V: -0.0142) \r\ninfo string e2e4  (322 ) N:    3001 (+ 0) (P: 14.43%) (Q:  0.04315) (U: 0.01574) (Q+U:  0.05889) (V: -0.0142) \r\nbestmove e2e4\r\n\r\n16bit protobuf\r\ninfo string c2c4  (264 ) N:    1234 (+ 0) (P: 12.50%) (Q:  0.02733) (U: 0.03314) (Q+U:  0.06047) (V: -0.0142) \r\ninfo string d2d4  (293 ) N:    1431 (+ 0) (P:  9.75%) (Q:  0.03749) (U: 0.02229) (Q+U:  0.05978) (V: -0.0142) \r\ninfo string e2e4  (322 ) N:    3024 (+ 0) (P: 14.43%) (Q:  0.04306) (U: 0.01562) (Q+U:  0.05868) (V: -0.0142) \r\nbestmove e2e4\r\n```\r\n\r\nLoads network ~10x faster","created_at":"2018-07-12T13:39:58Z","updated_at":"2018-07-14T13:37:43Z","closed_at":null,"merged_at":null,"merge_commit_sha":"e08c85d424af442a6cc50a62c1ce67f4945ae037","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/LeelaChessZero/lc0/pulls/166/commits","review_comments_url":"https://api.github.com/repos/LeelaChessZero/lc0/pulls/166/comments","review_comment_url":"https://api.github.com/repos/LeelaChessZero/lc0/pulls/comments{/number}","comments_url":"https://api.github.com/repos/LeelaChessZero/lc0/issues/166/comments","statuses_url":"https://api.github.com/repos/LeelaChessZero/lc0/statuses/0f6f21a17725141132d1486409c33a9a2bd6ee2b","head":{"label":"LeelaChessZero:protobuf","ref":"protobuf","sha":"0f6f21a17725141132d1486409c33a9a2bd6ee2b","user":{"login":"LeelaChessZero","id":39777295,"node_id":"MDEyOk9yZ2FuaXphdGlvbjM5Nzc3Mjk1","avatar_url":"https://avatars1.githubusercontent.com/u/39777295?v=4","gravatar_id":"","url":"https://api.github.com/users/LeelaChessZero","html_url":"https://github.com/LeelaChessZero","followers_url":"https://api.github.com/users/LeelaChessZero/followers","following_url":"https://api.github.com/users/LeelaChessZero/following{/other_user}","gists_url":"https://api.github.com/users/LeelaChessZero/gists{/gist_id}","starred_url":"https://api.github.com/users/LeelaChessZero/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/LeelaChessZero/subscriptions","organizations_url":"https://api.github.com/users/LeelaChessZero/orgs","repos_url":"https://api.github.com/users/LeelaChessZero/repos","events_url":"https://api.github.com/users/LeelaChessZero/events{/privacy}","received_events_url":"https://api.github.com/users/LeelaChessZero/received_events","type":"Organization","site_admin":false},"repo":{"id":135489692,"node_id":"MDEwOlJlcG9zaXRvcnkxMzU0ODk2OTI=","name":"lc0","full_name":"LeelaChessZero/lc0","owner":{"login":"LeelaChessZero","id":39777295,"node_id":"MDEyOk9yZ2FuaXphdGlvbjM5Nzc3Mjk1","avatar_url":"https://avatars1.githubusercontent.com/u/39777295?v=4","gravatar_id":"","url":"https://api.github.com/users/LeelaChessZero","html_url":"https://github.com/LeelaChessZero","followers_url":"https://api.github.com/users/LeelaChessZero/followers","following_url":"https://api.github.com/users/LeelaChessZero/following{/other_user}","gists_url":"https://api.github.com/users/LeelaChessZero/gists{/gist_id}","starred_url":"https://api.github.com/users/LeelaChessZero/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/LeelaChessZero/subscriptions","organizations_url":"https://api.github.com/users/LeelaChessZero/orgs","repos_url":"https://api.github.com/users/LeelaChessZero/repos","events_url":"https://api.github.com/users/LeelaChessZero/events{/privacy}","received_events_url":"https://api.github.com/users/LeelaChessZero/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/LeelaChessZero/lc0","description":"The rewritten engine, originally for tensorflow. Now all other backends have been ported here.","fork":false,"url":"https://api.github.com/repos/LeelaChessZero/lc0","forks_url":"https://api.github.com/repos/LeelaChessZero/lc0/forks","keys_url":"https://api.github.com/repos/LeelaChessZero/lc0/keys{/key_id}","collaborators_url":"https://api.github.com/repos/LeelaChessZero/lc0/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/LeelaChessZero/lc0/teams","hooks_url":"https://api.github.com/repos/LeelaChessZero/lc0/hooks","issue_events_url":"https://api.github.com/repos/LeelaChessZero/lc0/issues/events{/number}","events_url":"https://api.github.com/repos/LeelaChessZero/lc0/events","assignees_url":"https://api.github.com/repos/LeelaChessZero/lc0/assignees{/user}","branches_url":"https://api.github.com/repos/LeelaChessZero/lc0/branches{/branch}","tags_url":"https://api.github.com/repos/LeelaChessZero/lc0/tags","blobs_url":"https://api.github.com/repos/LeelaChessZero/lc0/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/LeelaChessZero/lc0/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/LeelaChessZero/lc0/git/refs{/sha}","trees_url":"https://api.github.com/repos/LeelaChessZero/lc0/git/trees{/sha}","statuses_url":"https://api.github.com/repos/LeelaChessZero/lc0/statuses/{sha}","languages_url":"https://api.github.com/repos/LeelaChessZero/lc0/languages","stargazers_url":"https://api.github.com/repos/LeelaChessZero/lc0/stargazers","contributors_url":"https://api.github.com/repos/LeelaChessZero/lc0/contributors","subscribers_url":"https://api.github.com/repos/LeelaChessZero/lc0/subscribers","subscription_url":"https://api.github.com/repos/LeelaChessZero/lc0/subscription","commits_url":"https://api.github.com/repos/LeelaChessZero/lc0/commits{/sha}","git_commits_url":"https://api.github.com/repos/LeelaChessZero/lc0/git/commits{/sha}","comments_url":"https://api.github.com/repos/LeelaChessZero/lc0/comments{/number}","issue_comment_url":"https://api.github.com/repos/LeelaChessZero/lc0/issues/comments{/number}","contents_url":"https://api.github.com/repos/LeelaChessZero/lc0/contents/{+path}","compare_url":"https://api.github.com/repos/LeelaChessZero/lc0/compare/{base}...{head}","merges_url":"https://api.github.com/repos/LeelaChessZero/lc0/merges","archive_url":"https://api.github.com/repos/LeelaChessZero/lc0/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/LeelaChessZero/lc0/downloads","issues_url":"https://api.github.com/repos/LeelaChessZero/lc0/issues{/number}","pulls_url":"https://api.github.com/repos/LeelaChessZero/lc0/pulls{/number}","milestones_url":"https://api.github.com/repos/LeelaChessZero/lc0/milestones{/number}","notifications_url":"https://api.github.com/repos/LeelaChessZero/lc0/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/LeelaChessZero/lc0/labels{/name}","releases_url":"https://api.github.com/repos/LeelaChessZero/lc0/releases{/id}","deployments_url":"https://api.github.com/repos/LeelaChessZero/lc0/deployments","created_at":"2018-05-30T19:41:42Z","updated_at":"2018-07-13T07:37:06Z","pushed_at":"2018-07-14T12:56:11Z","git_url":"git://github.com/LeelaChessZero/lc0.git","ssh_url":"git@github.com:LeelaChessZero/lc0.git","clone_url":"https://github.com/LeelaChessZero/lc0.git","svn_url":"https://github.com/LeelaChessZero/lc0","homepage":"","size":34865,"stargazers_count":28,"watchers_count":28,"language":"C++","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":36,"mirror_url":null,"archived":false,"open_issues_count":64,"license":{"key":"gpl-3.0","name":"GNU General Public License v3.0","spdx_id":"GPL-3.0","url":"https://api.github.com/licenses/gpl-3.0","node_id":"MDc6TGljZW5zZTk="},"forks":36,"open_issues":64,"watchers":28,"default_branch":"master"}},"base":{"label":"LeelaChessZero:master","ref":"master","sha":"97137d297f9d0d78a2b71ddf64dcfdd6c1781103","user":{"login":"LeelaChessZero","id":39777295,"node_id":"MDEyOk9yZ2FuaXphdGlvbjM5Nzc3Mjk1","avatar_url":"https://avatars1.githubusercontent.com/u/39777295?v=4","gravatar_id":"","url":"https://api.github.com/users/LeelaChessZero","html_url":"https://github.com/LeelaChessZero","followers_url":"https://api.github.com/users/LeelaChessZero/followers","following_url":"https://api.github.com/users/LeelaChessZero/following{/other_user}","gists_url":"https://api.github.com/users/LeelaChessZero/gists{/gist_id}","starred_url":"https://api.github.com/users/LeelaChessZero/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/LeelaChessZero/subscriptions","organizations_url":"https://api.github.com/users/LeelaChessZero/orgs","repos_url":"https://api.github.com/users/LeelaChessZero/repos","events_url":"https://api.github.com/users/LeelaChessZero/events{/privacy}","received_events_url":"https://api.github.com/users/LeelaChessZero/received_events","type":"Organization","site_admin":false},"repo":{"id":135489692,"node_id":"MDEwOlJlcG9zaXRvcnkxMzU0ODk2OTI=","name":"lc0","full_name":"LeelaChessZero/lc0","owner":{"login":"LeelaChessZero","id":39777295,"node_id":"MDEyOk9yZ2FuaXphdGlvbjM5Nzc3Mjk1","avatar_url":"https://avatars1.githubusercontent.com/u/39777295?v=4","gravatar_id":"","url":"https://api.github.com/users/LeelaChessZero","html_url":"https://github.com/LeelaChessZero","followers_url":"https://api.github.com/users/LeelaChessZero/followers","following_url":"https://api.github.com/users/LeelaChessZero/following{/other_user}","gists_url":"https://api.github.com/users/LeelaChessZero/gists{/gist_id}","starred_url":"https://api.github.com/users/LeelaChessZero/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/LeelaChessZero/subscriptions","organizations_url":"https://api.github.com/users/LeelaChessZero/orgs","repos_url":"https://api.github.com/users/LeelaChessZero/repos","events_url":"https://api.github.com/users/LeelaChessZero/events{/privacy}","received_events_url":"https://api.github.com/users/LeelaChessZero/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/LeelaChessZero/lc0","description":"The rewritten engine, originally for tensorflow. Now all other backends have been ported here.","fork":false,"url":"https://api.github.com/repos/LeelaChessZero/lc0","forks_url":"https://api.github.com/repos/LeelaChessZero/lc0/forks","keys_url":"https://api.github.com/repos/LeelaChessZero/lc0/keys{/key_id}","collaborators_url":"https://api.github.com/repos/LeelaChessZero/lc0/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/LeelaChessZero/lc0/teams","hooks_url":"https://api.github.com/repos/LeelaChessZero/lc0/hooks","issue_events_url":"https://api.github.com/repos/LeelaChessZero/lc0/issues/events{/number}","events_url":"https://api.github.com/repos/LeelaChessZero/lc0/events","assignees_url":"https://api.github.com/repos/LeelaChessZero/lc0/assignees{/user}","branches_url":"https://api.github.com/repos/LeelaChessZero/lc0/branches{/branch}","tags_url":"https://api.github.com/repos/LeelaChessZero/lc0/tags","blobs_url":"https://api.github.com/repos/LeelaChessZero/lc0/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/LeelaChessZero/lc0/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/LeelaChessZero/lc0/git/refs{/sha}","trees_url":"https://api.github.com/repos/LeelaChessZero/lc0/git/trees{/sha}","statuses_url":"https://api.github.com/repos/LeelaChessZero/lc0/statuses/{sha}","languages_url":"https://api.github.com/repos/LeelaChessZero/lc0/languages","stargazers_url":"https://api.github.com/repos/LeelaChessZero/lc0/stargazers","contributors_url":"https://api.github.com/repos/LeelaChessZero/lc0/contributors","subscribers_url":"https://api.github.com/repos/LeelaChessZero/lc0/subscribers","subscription_url":"https://api.github.com/repos/LeelaChessZero/lc0/subscription","commits_url":"https://api.github.com/repos/LeelaChessZero/lc0/commits{/sha}","git_commits_url":"https://api.github.com/repos/LeelaChessZero/lc0/git/commits{/sha}","comments_url":"https://api.github.com/repos/LeelaChessZero/lc0/comments{/number}","issue_comment_url":"https://api.github.com/repos/LeelaChessZero/lc0/issues/comments{/number}","contents_url":"https://api.github.com/repos/LeelaChessZero/lc0/contents/{+path}","compare_url":"https://api.github.com/repos/LeelaChessZero/lc0/compare/{base}...{head}","merges_url":"https://api.github.com/repos/LeelaChessZero/lc0/merges","archive_url":"https://api.github.com/repos/LeelaChessZero/lc0/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/LeelaChessZero/lc0/downloads","issues_url":"https://api.github.com/repos/LeelaChessZero/lc0/issues{/number}","pulls_url":"https://api.github.com/repos/LeelaChessZero/lc0/pulls{/number}","milestones_url":"https://api.github.com/repos/LeelaChessZero/lc0/milestones{/number}","notifications_url":"https://api.github.com/repos/LeelaChessZero/lc0/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/LeelaChessZero/lc0/labels{/name}","releases_url":"https://api.github.com/repos/LeelaChessZero/lc0/releases{/id}","deployments_url":"https://api.github.com/repos/LeelaChessZero/lc0/deployments","created_at":"2018-05-30T19:41:42Z","updated_at":"2018-07-13T07:37:06Z","pushed_at":"2018-07-14T12:56:11Z","git_url":"git://github.com/LeelaChessZero/lc0.git","ssh_url":"git@github.com:LeelaChessZero/lc0.git","clone_url":"https://github.com/LeelaChessZero/lc0.git","svn_url":"https://github.com/LeelaChessZero/lc0","homepage":"","size":34865,"stargazers_count":28,"watchers_count":28,"language":"C++","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":36,"mirror_url":null,"archived":false,"open_issues_count":64,"license":{"key":"gpl-3.0","name":"GNU General Public License v3.0","spdx_id":"GPL-3.0","url":"https://api.github.com/licenses/gpl-3.0","node_id":"MDc6TGljZW5zZTk="},"forks":36,"open_issues":64,"watchers":28,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/LeelaChessZero/lc0/pulls/166"},"html":{"href":"https://github.com/LeelaChessZero/lc0/pull/166"},"issue":{"href":"https://api.github.com/repos/LeelaChessZero/lc0/issues/166"},"comments":{"href":"https://api.github.com/repos/LeelaChessZero/lc0/issues/166/comments"},"review_comments":{"href":"https://api.github.com/repos/LeelaChessZero/lc0/pulls/166/comments"},"review_comment":{"href":"https://api.github.com/repos/LeelaChessZero/lc0/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/LeelaChessZero/lc0/pulls/166/commits"},"statuses":{"href":"https://api.github.com/repos/LeelaChessZero/lc0/statuses/0f6f21a17725141132d1486409c33a9a2bd6ee2b"}},"author_association":"MEMBER"}}
{"actor":{"display_login":"mooskagh"}}