{"action":"created","comment":{"url":"https://api.github.com/repos/mapsme/omim/pulls/comments/237091058","pull_request_review_id":179304553,"id":237091058,"node_id":"MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDIzNzA5MTA1OA==","diff_hunk":"@@ -0,0 +1,115 @@\n+#include \"generator/descriptions_section_builder.hpp\"\n+\n+#include <iterator>\n+#include <limits>\n+#include <sstream>\n+#include <utility>\n+\n+#include <boost/algorithm/string.hpp>\n+#include <boost/range/iterator_range.hpp>\n+\n+namespace generator\n+{\n+namespace fs = boost::filesystem;\n+\n+std::string DescriptionsCollectionBuilderStat::LangStatisticsToString() const\n+{\n+  std::stringstream stream;\n+  stream << \"Language statistics - \";\n+  if (m_langsStat.empty())\n+    stream << \"(empty)\";\n+\n+  for (size_t code = 0; code < m_langsStat.size(); ++code)\n+  {\n+    if (m_langsStat[code] == 0)\n+      continue;\n+\n+    stream << StringUtf8Multilang::GetLangByCode(static_cast<int8_t>(code))\n+           << \":\" << m_langsStat[code] << \" \";\n+  }\n+\n+  return stream.str();\n+}\n+\n+DescriptionsCollectionBuilder::DescriptionsCollectionBuilder(std::string const & wikipediaDir,\n+                                                             std::string const & mwmFile)\n+  : m_wikipediaDir(wikipediaDir), m_mwmFile(mwmFile) {}\n+\n+// static\n+fs::path DescriptionsCollectionBuilder::MakePath(std::string const & wikipediaDir, std::string wikipediaUrl)\n+{\n+  boost::trim(wikipediaUrl);\n+  boost::replace_first(wikipediaUrl, \"http://\", \"\");\n+  boost::replace_first(wikipediaUrl, \"https://\", \"\");\n+  if (boost::ends_with(wikipediaUrl, \"/\"))\n+    wikipediaUrl.pop_back();\n+\n+  return fs::path(wikipediaDir) / fs::path(wikipediaUrl);\n+}\n+\n+// static\n+int DescriptionsCollectionBuilder::FillStringFromFile(fs::path const & fullPath, int8_t code,\n+                                                      StringUtf8Multilang & str)\n+{\n+\n+  fs::fstream stream(fullPath);\n+  std::string content((std::istreambuf_iterator<char>(stream)), std::istreambuf_iterator<char>());\n+  auto const contentSize = content.size();\n+  if (contentSize != 0)\n+    str.AddString(code, std::move(content));\n+\n+  CHECK_LESS_OR_EQUAL(contentSize, std::numeric_limits<int>::max(), ());\n+  return static_cast<int>(contentSize);\n+}\n+\n+int DescriptionsCollectionBuilder::FindPageAndFill(std::string wikipediaUrl,","path":"generator/descriptions_section_builder.cpp","position":null,"original_position":65,"commit_id":"9a94980c361ffca71c5952a1aa875825f6979516","original_commit_id":"fcb9bb108b065005a6a76b35593b98fddac563a0","user":{"login":"maksimandrianov","id":12376740,"node_id":"MDQ6VXNlcjEyMzc2NzQw","avatar_url":"https://avatars3.githubusercontent.com/u/12376740?v=4","gravatar_id":"","url":"https://api.github.com/users/maksimandrianov","html_url":"https://github.com/maksimandrianov","followers_url":"https://api.github.com/users/maksimandrianov/followers","following_url":"https://api.github.com/users/maksimandrianov/following{/other_user}","gists_url":"https://api.github.com/users/maksimandrianov/gists{/gist_id}","starred_url":"https://api.github.com/users/maksimandrianov/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/maksimandrianov/subscriptions","organizations_url":"https://api.github.com/users/maksimandrianov/orgs","repos_url":"https://api.github.com/users/maksimandrianov/repos","events_url":"https://api.github.com/users/maksimandrianov/events{/privacy}","received_events_url":"https://api.github.com/users/maksimandrianov/received_events","type":"User","site_admin":false},"body":"++ заменил на size_t","created_at":"2018-11-28T13:52:02Z","updated_at":"2018-11-28T13:52:02Z","html_url":"https://github.com/mapsme/omim/pull/9811#discussion_r237091058","pull_request_url":"https://api.github.com/repos/mapsme/omim/pulls/9811","author_association":"CONTRIBUTOR","_links":{"self":{"href":"https://api.github.com/repos/mapsme/omim/pulls/comments/237091058"},"html":{"href":"https://github.com/mapsme/omim/pull/9811#discussion_r237091058"},"pull_request":{"href":"https://api.github.com/repos/mapsme/omim/pulls/9811"}},"in_reply_to_id":237027245},"pull_request":{"url":"https://api.github.com/repos/mapsme/omim/pulls/9811","id":229727697,"node_id":"MDExOlB1bGxSZXF1ZXN0MjI5NzI3Njk3","html_url":"https://github.com/mapsme/omim/pull/9811","diff_url":"https://github.com/mapsme/omim/pull/9811.diff","patch_url":"https://github.com/mapsme/omim/pull/9811.patch","issue_url":"https://api.github.com/repos/mapsme/omim/issues/9811","number":9811,"state":"open","locked":false,"title":"[generator] Added wikipedia descriptions generation.","user":{"login":"maksimandrianov","id":12376740,"node_id":"MDQ6VXNlcjEyMzc2NzQw","avatar_url":"https://avatars3.githubusercontent.com/u/12376740?v=4","gravatar_id":"","url":"https://api.github.com/users/maksimandrianov","html_url":"https://github.com/maksimandrianov","followers_url":"https://api.github.com/users/maksimandrianov/followers","following_url":"https://api.github.com/users/maksimandrianov/following{/other_user}","gists_url":"https://api.github.com/users/maksimandrianov/gists{/gist_id}","starred_url":"https://api.github.com/users/maksimandrianov/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/maksimandrianov/subscriptions","organizations_url":"https://api.github.com/users/maksimandrianov/orgs","repos_url":"https://api.github.com/users/maksimandrianov/repos","events_url":"https://api.github.com/users/maksimandrianov/events{/privacy}","received_events_url":"https://api.github.com/users/maksimandrianov/received_events","type":"User","site_admin":false},"body":"Что бы добавить новую секцию нужно:\r\n1) generator_tool --intermediate_data_path=path/to/intermediate_data --user_resource_path=path/to/omim/data --dump_wikipedia_urls=path/to/wiki_urls.txt - выгрузить список url.\r\n2) python omim/tools/python/descriptions_downloader.py --i=path/to/wiki_urls.txt path/to/pages_dir - скачать страницы wikipedia с помощью wikipedia api в pages_dir\r\n3) generator_tool --wikipedia_pages=path/to/wiki_urls.txt path/to/pages_dir --user_resource_path=path/to/omim/data --data_path=path/to/mwm --output=mwm_name  - добавить секцию в mwm_name\r\n\r\nили воспользоваться скриптом generate_planet.sh с новой опцией -d\r\n\r\nДля Артема:\r\nДля https://en.wikipedia.org/wiki/Fran%C3%A7ois_Capois\r\nПример запроса к api википедия: https://en.wikipedia.org/w/api.php?action=query&prop=extracts&titles=Fran%C3%A7ois_Capois&format=json&redirects=1\r\n[Результат](https://github.com/mapsme/omim/files/2595593/en.zip)\r\n\r\n[Небольшая статистика по размерам и языкам для Финляндии. ](https://github.com/mapsme/omim/files/2595777/finland_statistics.txt)\r\n\r\n\r\n\r\nРеквест поверх реквеста Даши. Можно смотреть только мой коммит.","created_at":"2018-11-09T15:25:05Z","updated_at":"2018-11-28T13:52:02Z","closed_at":null,"merged_at":null,"merge_commit_sha":"a0bb2d0a06380d7a1e83fea782d692ff2ffe6646","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/mapsme/omim/pulls/9811/commits","review_comments_url":"https://api.github.com/repos/mapsme/omim/pulls/9811/comments","review_comment_url":"https://api.github.com/repos/mapsme/omim/pulls/comments{/number}","comments_url":"https://api.github.com/repos/mapsme/omim/issues/9811/comments","statuses_url":"https://api.github.com/repos/mapsme/omim/statuses/9a94980c361ffca71c5952a1aa875825f6979516","head":{"label":"maksimandrianov:wiki","ref":"wiki","sha":"9a94980c361ffca71c5952a1aa875825f6979516","user":{"login":"maksimandrianov","id":12376740,"node_id":"MDQ6VXNlcjEyMzc2NzQw","avatar_url":"https://avatars3.githubusercontent.com/u/12376740?v=4","gravatar_id":"","url":"https://api.github.com/users/maksimandrianov","html_url":"https://github.com/maksimandrianov","followers_url":"https://api.github.com/users/maksimandrianov/followers","following_url":"https://api.github.com/users/maksimandrianov/following{/other_user}","gists_url":"https://api.github.com/users/maksimandrianov/gists{/gist_id}","starred_url":"https://api.github.com/users/maksimandrianov/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/maksimandrianov/subscriptions","organizations_url":"https://api.github.com/users/maksimandrianov/orgs","repos_url":"https://api.github.com/users/maksimandrianov/repos","events_url":"https://api.github.com/users/maksimandrianov/events{/privacy}","received_events_url":"https://api.github.com/users/maksimandrianov/received_events","type":"User","site_admin":false},"repo":{"id":133388788,"node_id":"MDEwOlJlcG9zaXRvcnkxMzMzODg3ODg=","name":"omim","full_name":"maksimandrianov/omim","private":false,"owner":{"login":"maksimandrianov","id":12376740,"node_id":"MDQ6VXNlcjEyMzc2NzQw","avatar_url":"https://avatars3.githubusercontent.com/u/12376740?v=4","gravatar_id":"","url":"https://api.github.com/users/maksimandrianov","html_url":"https://github.com/maksimandrianov","followers_url":"https://api.github.com/users/maksimandrianov/followers","following_url":"https://api.github.com/users/maksimandrianov/following{/other_user}","gists_url":"https://api.github.com/users/maksimandrianov/gists{/gist_id}","starred_url":"https://api.github.com/users/maksimandrianov/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/maksimandrianov/subscriptions","organizations_url":"https://api.github.com/users/maksimandrianov/orgs","repos_url":"https://api.github.com/users/maksimandrianov/repos","events_url":"https://api.github.com/users/maksimandrianov/events{/privacy}","received_events_url":"https://api.github.com/users/maksimandrianov/received_events","type":"User","site_admin":false},"html_url":"https://github.com/maksimandrianov/omim","description":"🗺️ MAPS.ME — Offline OpenStreetMap maps for iOS and Android","fork":true,"url":"https://api.github.com/repos/maksimandrianov/omim","forks_url":"https://api.github.com/repos/maksimandrianov/omim/forks","keys_url":"https://api.github.com/repos/maksimandrianov/omim/keys{/key_id}","collaborators_url":"https://api.github.com/repos/maksimandrianov/omim/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/maksimandrianov/omim/teams","hooks_url":"https://api.github.com/repos/maksimandrianov/omim/hooks","issue_events_url":"https://api.github.com/repos/maksimandrianov/omim/issues/events{/number}","events_url":"https://api.github.com/repos/maksimandrianov/omim/events","assignees_url":"https://api.github.com/repos/maksimandrianov/omim/assignees{/user}","branches_url":"https://api.github.com/repos/maksimandrianov/omim/branches{/branch}","tags_url":"https://api.github.com/repos/maksimandrianov/omim/tags","blobs_url":"https://api.github.com/repos/maksimandrianov/omim/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/maksimandrianov/omim/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/maksimandrianov/omim/git/refs{/sha}","trees_url":"https://api.github.com/repos/maksimandrianov/omim/git/trees{/sha}","statuses_url":"https://api.github.com/repos/maksimandrianov/omim/statuses/{sha}","languages_url":"https://api.github.com/repos/maksimandrianov/omim/languages","stargazers_url":"https://api.github.com/repos/maksimandrianov/omim/stargazers","contributors_url":"https://api.github.com/repos/maksimandrianov/omim/contributors","subscribers_url":"https://api.github.com/repos/maksimandrianov/omim/subscribers","subscription_url":"https://api.github.com/repos/maksimandrianov/omim/subscription","commits_url":"https://api.github.com/repos/maksimandrianov/omim/commits{/sha}","git_commits_url":"https://api.github.com/repos/maksimandrianov/omim/git/commits{/sha}","comments_url":"https://api.github.com/repos/maksimandrianov/omim/comments{/number}","issue_comment_url":"https://api.github.com/repos/maksimandrianov/omim/issues/comments{/number}","contents_url":"https://api.github.com/repos/maksimandrianov/omim/contents/{+path}","compare_url":"https://api.github.com/repos/maksimandrianov/omim/compare/{base}...{head}","merges_url":"https://api.github.com/repos/maksimandrianov/omim/merges","archive_url":"https://api.github.com/repos/maksimandrianov/omim/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/maksimandrianov/omim/downloads","issues_url":"https://api.github.com/repos/maksimandrianov/omim/issues{/number}","pulls_url":"https://api.github.com/repos/maksimandrianov/omim/pulls{/number}","milestones_url":"https://api.github.com/repos/maksimandrianov/omim/milestones{/number}","notifications_url":"https://api.github.com/repos/maksimandrianov/omim/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/maksimandrianov/omim/labels{/name}","releases_url":"https://api.github.com/repos/maksimandrianov/omim/releases{/id}","deployments_url":"https://api.github.com/repos/maksimandrianov/omim/deployments","created_at":"2018-05-14T16:11:26Z","updated_at":"2018-11-07T16:25:03Z","pushed_at":"2018-11-28T13:51:35Z","git_url":"git://github.com/maksimandrianov/omim.git","ssh_url":"git@github.com:maksimandrianov/omim.git","clone_url":"https://github.com/maksimandrianov/omim.git","svn_url":"https://github.com/maksimandrianov/omim","homepage":"","size":4185499,"stargazers_count":0,"watchers_count":0,"language":"C++","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":0,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0","node_id":"MDc6TGljZW5zZTI="},"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"mapsme:master","ref":"master","sha":"0e3c8613155401042fa0b6956f9532c2fb3376cb","user":{"login":"mapsme","id":14185141,"node_id":"MDEyOk9yZ2FuaXphdGlvbjE0MTg1MTQx","avatar_url":"https://avatars1.githubusercontent.com/u/14185141?v=4","gravatar_id":"","url":"https://api.github.com/users/mapsme","html_url":"https://github.com/mapsme","followers_url":"https://api.github.com/users/mapsme/followers","following_url":"https://api.github.com/users/mapsme/following{/other_user}","gists_url":"https://api.github.com/users/mapsme/gists{/gist_id}","starred_url":"https://api.github.com/users/mapsme/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mapsme/subscriptions","organizations_url":"https://api.github.com/users/mapsme/orgs","repos_url":"https://api.github.com/users/mapsme/repos","events_url":"https://api.github.com/users/mapsme/events{/privacy}","received_events_url":"https://api.github.com/users/mapsme/received_events","type":"Organization","site_admin":false},"repo":{"id":42798033,"node_id":"MDEwOlJlcG9zaXRvcnk0Mjc5ODAzMw==","name":"omim","full_name":"mapsme/omim","private":false,"owner":{"login":"mapsme","id":14185141,"node_id":"MDEyOk9yZ2FuaXphdGlvbjE0MTg1MTQx","avatar_url":"https://avatars1.githubusercontent.com/u/14185141?v=4","gravatar_id":"","url":"https://api.github.com/users/mapsme","html_url":"https://github.com/mapsme","followers_url":"https://api.github.com/users/mapsme/followers","following_url":"https://api.github.com/users/mapsme/following{/other_user}","gists_url":"https://api.github.com/users/mapsme/gists{/gist_id}","starred_url":"https://api.github.com/users/mapsme/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mapsme/subscriptions","organizations_url":"https://api.github.com/users/mapsme/orgs","repos_url":"https://api.github.com/users/mapsme/repos","events_url":"https://api.github.com/users/mapsme/events{/privacy}","received_events_url":"https://api.github.com/users/mapsme/received_events","type":"Organization","site_admin":false},"html_url":"https://github.com/mapsme/omim","description":"🗺️ MAPS.ME — Offline OpenStreetMap maps for iOS and Android","fork":false,"url":"https://api.github.com/repos/mapsme/omim","forks_url":"https://api.github.com/repos/mapsme/omim/forks","keys_url":"https://api.github.com/repos/mapsme/omim/keys{/key_id}","collaborators_url":"https://api.github.com/repos/mapsme/omim/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/mapsme/omim/teams","hooks_url":"https://api.github.com/repos/mapsme/omim/hooks","issue_events_url":"https://api.github.com/repos/mapsme/omim/issues/events{/number}","events_url":"https://api.github.com/repos/mapsme/omim/events","assignees_url":"https://api.github.com/repos/mapsme/omim/assignees{/user}","branches_url":"https://api.github.com/repos/mapsme/omim/branches{/branch}","tags_url":"https://api.github.com/repos/mapsme/omim/tags","blobs_url":"https://api.github.com/repos/mapsme/omim/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/mapsme/omim/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/mapsme/omim/git/refs{/sha}","trees_url":"https://api.github.com/repos/mapsme/omim/git/trees{/sha}","statuses_url":"https://api.github.com/repos/mapsme/omim/statuses/{sha}","languages_url":"https://api.github.com/repos/mapsme/omim/languages","stargazers_url":"https://api.github.com/repos/mapsme/omim/stargazers","contributors_url":"https://api.github.com/repos/mapsme/omim/contributors","subscribers_url":"https://api.github.com/repos/mapsme/omim/subscribers","subscription_url":"https://api.github.com/repos/mapsme/omim/subscription","commits_url":"https://api.github.com/repos/mapsme/omim/commits{/sha}","git_commits_url":"https://api.github.com/repos/mapsme/omim/git/commits{/sha}","comments_url":"https://api.github.com/repos/mapsme/omim/comments{/number}","issue_comment_url":"https://api.github.com/repos/mapsme/omim/issues/comments{/number}","contents_url":"https://api.github.com/repos/mapsme/omim/contents/{+path}","compare_url":"https://api.github.com/repos/mapsme/omim/compare/{base}...{head}","merges_url":"https://api.github.com/repos/mapsme/omim/merges","archive_url":"https://api.github.com/repos/mapsme/omim/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/mapsme/omim/downloads","issues_url":"https://api.github.com/repos/mapsme/omim/issues{/number}","pulls_url":"https://api.github.com/repos/mapsme/omim/pulls{/number}","milestones_url":"https://api.github.com/repos/mapsme/omim/milestones{/number}","notifications_url":"https://api.github.com/repos/mapsme/omim/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/mapsme/omim/labels{/name}","releases_url":"https://api.github.com/repos/mapsme/omim/releases{/id}","deployments_url":"https://api.github.com/repos/mapsme/omim/deployments","created_at":"2015-09-20T02:57:13Z","updated_at":"2018-11-28T13:26:28Z","pushed_at":"2018-11-28T13:51:40Z","git_url":"git://github.com/mapsme/omim.git","ssh_url":"git@github.com:mapsme/omim.git","clone_url":"https://github.com/mapsme/omim.git","svn_url":"https://github.com/mapsme/omim","homepage":"","size":4218462,"stargazers_count":2845,"watchers_count":2845,"language":"C++","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":619,"mirror_url":null,"archived":false,"open_issues_count":768,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0","node_id":"MDc6TGljZW5zZTI="},"forks":619,"open_issues":768,"watchers":2845,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/mapsme/omim/pulls/9811"},"html":{"href":"https://github.com/mapsme/omim/pull/9811"},"issue":{"href":"https://api.github.com/repos/mapsme/omim/issues/9811"},"comments":{"href":"https://api.github.com/repos/mapsme/omim/issues/9811/comments"},"review_comments":{"href":"https://api.github.com/repos/mapsme/omim/pulls/9811/comments"},"review_comment":{"href":"https://api.github.com/repos/mapsme/omim/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/mapsme/omim/pulls/9811/commits"},"statuses":{"href":"https://api.github.com/repos/mapsme/omim/statuses/9a94980c361ffca71c5952a1aa875825f6979516"}},"author_association":"CONTRIBUTOR"}}
{"actor":{"display_login":"maksimandrianov"}}