{"action":"created","comment":{"url":"https://api.github.com/repos/mapsme/omim/pulls/comments/235957348","pull_request_review_id":177947274,"id":235957348,"node_id":"MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDIzNTk1NzM0OA==","diff_hunk":"@@ -0,0 +1,104 @@\n+#include \"generator/descriptions_section_builder.hpp\"\n+\n+#include \"indexer/feature.hpp\"\n+#include \"indexer/feature_processor.hpp\"\n+\n+#include \"coding/file_container.hpp\"\n+#include \"coding/file_name_utils.hpp\"\n+#include \"coding/file_writer.hpp\"\n+#include \"coding/multilang_utf8_string.hpp\"\n+\n+#include \"descriptions/serdes.hpp\"\n+\n+#include \"base/logging.hpp\"\n+\n+#include \"defines.hpp\"\n+\n+#include <boost/algorithm/string.hpp>\n+#include <boost/filesystem/fstream.hpp>\n+#include <boost/filesystem/operations.hpp>\n+#include <boost/filesystem/path.hpp>\n+#include <boost/range/iterator_range.hpp>\n+\n+namespace generator\n+{\n+namespace fs = boost::filesystem;\n+\n+size_t FindPageAndFill(std::string const & wikipediaUrl, std::string const & wikipediaDir,\n+                       StringUtf8Multilang & string)\n+{\n+  size_t size = 0;\n+  auto tmp = wikipediaUrl;\n+  boost::trim(tmp);\n+  boost::replace_first(tmp, \"http://\", \"\");\n+  boost::replace_first(tmp, \"https://\", \"\");\n+  auto path = fs::path(wikipediaDir) / fs::path(tmp);\n+  if (!fs::exists(path) || !fs::is_directory(path))\n+    return size;\n+\n+  for (auto & entry : boost::make_iterator_range(fs::directory_iterator(path), {}))\n+  {\n+    auto const filename = entry.path().filename();\n+    auto const lang = filename.stem().string();\n+    auto const code = StringUtf8Multilang::GetLangIndex(lang);\n+    if (code == StringUtf8Multilang::kUnsupportedLanguageCode)\n+    {\n+      LOG(LWARNING, (lang, \"is unsupported language.\"));\n+      continue;\n+    }\n+\n+    auto const fullPath = path / filename;\n+    fs::fstream stream(fullPath);\n+    std::string content((std::istreambuf_iterator<char>(stream)), std::istreambuf_iterator<char>());\n+    size += content.size();\n+    string.AddString(code, std::move(content));\n+  }\n+\n+  return size;\n+}\n+\n+size_t FillDescriptions(std::string const & wikipediaDir, std::string const & mwmFile,\n+                        descriptions::DescriptionsCollection & descriptions)\n+{\n+  size_t size = 0;\n+  size_t counter = 0;\n+  feature::ForEachFromDat(mwmFile, [&](FeatureType & f, uint32_t featureId) {\n+    auto const wikiUrl = f.GetMetadata().GetWikiURL();\n+    if (wikiUrl.empty())\n+      return;\n+\n+    StringUtf8Multilang string;\n+    auto const ret = FindPageAndFill(wikiUrl, wikipediaDir, string);\n+    if (ret == 0)\n+      return;\n+\n+    size += ret;\n+    ++counter;\n+    auto description = descriptions::FeatureDescription(featureId, std::move(string));\n+    descriptions.emplace_back(std::move(description));\n+  });\n+\n+  LOG(LINFO, (\"Found\", counter ,\"pages for\", mwmFile));","path":"generator/descriptions_section_builder.cpp","position":103,"original_position":81,"commit_id":"4d9630e01aac071e471ce850a277c79369c2e09c","original_commit_id":"04d972da06f3309f4b61106c0ca03745928add9a","user":{"login":"maksimandrianov","id":12376740,"node_id":"MDQ6VXNlcjEyMzc2NzQw","avatar_url":"https://avatars3.githubusercontent.com/u/12376740?v=4","gravatar_id":"","url":"https://api.github.com/users/maksimandrianov","html_url":"https://github.com/maksimandrianov","followers_url":"https://api.github.com/users/maksimandrianov/followers","following_url":"https://api.github.com/users/maksimandrianov/following{/other_user}","gists_url":"https://api.github.com/users/maksimandrianov/gists{/gist_id}","starred_url":"https://api.github.com/users/maksimandrianov/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/maksimandrianov/subscriptions","organizations_url":"https://api.github.com/users/maksimandrianov/orgs","repos_url":"https://api.github.com/users/maksimandrianov/repos","events_url":"https://api.github.com/users/maksimandrianov/events{/privacy}","received_events_url":"https://api.github.com/users/maksimandrianov/received_events","type":"User","site_admin":false},"body":"видимо не запушил","created_at":"2018-11-23T14:31:47Z","updated_at":"2018-11-23T14:31:47Z","html_url":"https://github.com/mapsme/omim/pull/9811#discussion_r235957348","pull_request_url":"https://api.github.com/repos/mapsme/omim/pulls/9811","author_association":"CONTRIBUTOR","_links":{"self":{"href":"https://api.github.com/repos/mapsme/omim/pulls/comments/235957348"},"html":{"href":"https://github.com/mapsme/omim/pull/9811#discussion_r235957348"},"pull_request":{"href":"https://api.github.com/repos/mapsme/omim/pulls/9811"}},"in_reply_to_id":232323566},"pull_request":{"url":"https://api.github.com/repos/mapsme/omim/pulls/9811","id":229727697,"node_id":"MDExOlB1bGxSZXF1ZXN0MjI5NzI3Njk3","html_url":"https://github.com/mapsme/omim/pull/9811","diff_url":"https://github.com/mapsme/omim/pull/9811.diff","patch_url":"https://github.com/mapsme/omim/pull/9811.patch","issue_url":"https://api.github.com/repos/mapsme/omim/issues/9811","number":9811,"state":"open","locked":false,"title":"[DNM][generator] Added wikipedia descriptions generation.","user":{"login":"maksimandrianov","id":12376740,"node_id":"MDQ6VXNlcjEyMzc2NzQw","avatar_url":"https://avatars3.githubusercontent.com/u/12376740?v=4","gravatar_id":"","url":"https://api.github.com/users/maksimandrianov","html_url":"https://github.com/maksimandrianov","followers_url":"https://api.github.com/users/maksimandrianov/followers","following_url":"https://api.github.com/users/maksimandrianov/following{/other_user}","gists_url":"https://api.github.com/users/maksimandrianov/gists{/gist_id}","starred_url":"https://api.github.com/users/maksimandrianov/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/maksimandrianov/subscriptions","organizations_url":"https://api.github.com/users/maksimandrianov/orgs","repos_url":"https://api.github.com/users/maksimandrianov/repos","events_url":"https://api.github.com/users/maksimandrianov/events{/privacy}","received_events_url":"https://api.github.com/users/maksimandrianov/received_events","type":"User","site_admin":false},"body":"Что бы добавить новую секцию нужно:\r\n1) generator_tool --intermediate_data_path=path/to/intermediate_data --user_resource_path=path/to/omim/data --dump_wikipedia_urls=path/to/wiki_urls.txt - выгрузить список url.\r\n2) python omim/tools/python/descriptions_downloader.py --i=path/to/wiki_urls.txt path/to/pages_dir - скачать страницы wikipedia с помощью wikipedia api в pages_dir\r\n3) generator_tool --wikipedia_pages=path/to/wiki_urls.txt path/to/pages_dir --user_resource_path=path/to/omim/data --data_path=path/to/mwm --output=mwm_name  - добавить секцию в mwm_name\r\n\r\nили воспользоваться скриптом generate_planet.sh с новой опцией -d\r\n\r\nДля Артема:\r\nДля https://en.wikipedia.org/wiki/Fran%C3%A7ois_Capois\r\nПример запроса к api википедия: https://en.wikipedia.org/w/api.php?action=query&prop=extracts&titles=Fran%C3%A7ois_Capois&format=json&redirects=1\r\n[Результат](https://github.com/mapsme/omim/files/2595593/en.zip)\r\n\r\n[Небольшая статистика по размерам и языкам для Финляндии. ](https://github.com/mapsme/omim/files/2595777/finland_statistics.txt)\r\n\r\n\r\n\r\nРеквест поверх реквеста Даши. Можно смотреть только мой коммит.","created_at":"2018-11-09T15:25:05Z","updated_at":"2018-11-23T14:31:47Z","closed_at":null,"merged_at":null,"merge_commit_sha":"402650489a76ef7d8d01331204c66733d5c28313","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/mapsme/omim/pulls/9811/commits","review_comments_url":"https://api.github.com/repos/mapsme/omim/pulls/9811/comments","review_comment_url":"https://api.github.com/repos/mapsme/omim/pulls/comments{/number}","comments_url":"https://api.github.com/repos/mapsme/omim/issues/9811/comments","statuses_url":"https://api.github.com/repos/mapsme/omim/statuses/4d9630e01aac071e471ce850a277c79369c2e09c","head":{"label":"maksimandrianov:wiki","ref":"wiki","sha":"4d9630e01aac071e471ce850a277c79369c2e09c","user":{"login":"maksimandrianov","id":12376740,"node_id":"MDQ6VXNlcjEyMzc2NzQw","avatar_url":"https://avatars3.githubusercontent.com/u/12376740?v=4","gravatar_id":"","url":"https://api.github.com/users/maksimandrianov","html_url":"https://github.com/maksimandrianov","followers_url":"https://api.github.com/users/maksimandrianov/followers","following_url":"https://api.github.com/users/maksimandrianov/following{/other_user}","gists_url":"https://api.github.com/users/maksimandrianov/gists{/gist_id}","starred_url":"https://api.github.com/users/maksimandrianov/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/maksimandrianov/subscriptions","organizations_url":"https://api.github.com/users/maksimandrianov/orgs","repos_url":"https://api.github.com/users/maksimandrianov/repos","events_url":"https://api.github.com/users/maksimandrianov/events{/privacy}","received_events_url":"https://api.github.com/users/maksimandrianov/received_events","type":"User","site_admin":false},"repo":{"id":133388788,"node_id":"MDEwOlJlcG9zaXRvcnkxMzMzODg3ODg=","name":"omim","full_name":"maksimandrianov/omim","private":false,"owner":{"login":"maksimandrianov","id":12376740,"node_id":"MDQ6VXNlcjEyMzc2NzQw","avatar_url":"https://avatars3.githubusercontent.com/u/12376740?v=4","gravatar_id":"","url":"https://api.github.com/users/maksimandrianov","html_url":"https://github.com/maksimandrianov","followers_url":"https://api.github.com/users/maksimandrianov/followers","following_url":"https://api.github.com/users/maksimandrianov/following{/other_user}","gists_url":"https://api.github.com/users/maksimandrianov/gists{/gist_id}","starred_url":"https://api.github.com/users/maksimandrianov/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/maksimandrianov/subscriptions","organizations_url":"https://api.github.com/users/maksimandrianov/orgs","repos_url":"https://api.github.com/users/maksimandrianov/repos","events_url":"https://api.github.com/users/maksimandrianov/events{/privacy}","received_events_url":"https://api.github.com/users/maksimandrianov/received_events","type":"User","site_admin":false},"html_url":"https://github.com/maksimandrianov/omim","description":"🗺️ MAPS.ME — Offline OpenStreetMap maps for iOS and Android","fork":true,"url":"https://api.github.com/repos/maksimandrianov/omim","forks_url":"https://api.github.com/repos/maksimandrianov/omim/forks","keys_url":"https://api.github.com/repos/maksimandrianov/omim/keys{/key_id}","collaborators_url":"https://api.github.com/repos/maksimandrianov/omim/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/maksimandrianov/omim/teams","hooks_url":"https://api.github.com/repos/maksimandrianov/omim/hooks","issue_events_url":"https://api.github.com/repos/maksimandrianov/omim/issues/events{/number}","events_url":"https://api.github.com/repos/maksimandrianov/omim/events","assignees_url":"https://api.github.com/repos/maksimandrianov/omim/assignees{/user}","branches_url":"https://api.github.com/repos/maksimandrianov/omim/branches{/branch}","tags_url":"https://api.github.com/repos/maksimandrianov/omim/tags","blobs_url":"https://api.github.com/repos/maksimandrianov/omim/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/maksimandrianov/omim/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/maksimandrianov/omim/git/refs{/sha}","trees_url":"https://api.github.com/repos/maksimandrianov/omim/git/trees{/sha}","statuses_url":"https://api.github.com/repos/maksimandrianov/omim/statuses/{sha}","languages_url":"https://api.github.com/repos/maksimandrianov/omim/languages","stargazers_url":"https://api.github.com/repos/maksimandrianov/omim/stargazers","contributors_url":"https://api.github.com/repos/maksimandrianov/omim/contributors","subscribers_url":"https://api.github.com/repos/maksimandrianov/omim/subscribers","subscription_url":"https://api.github.com/repos/maksimandrianov/omim/subscription","commits_url":"https://api.github.com/repos/maksimandrianov/omim/commits{/sha}","git_commits_url":"https://api.github.com/repos/maksimandrianov/omim/git/commits{/sha}","comments_url":"https://api.github.com/repos/maksimandrianov/omim/comments{/number}","issue_comment_url":"https://api.github.com/repos/maksimandrianov/omim/issues/comments{/number}","contents_url":"https://api.github.com/repos/maksimandrianov/omim/contents/{+path}","compare_url":"https://api.github.com/repos/maksimandrianov/omim/compare/{base}...{head}","merges_url":"https://api.github.com/repos/maksimandrianov/omim/merges","archive_url":"https://api.github.com/repos/maksimandrianov/omim/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/maksimandrianov/omim/downloads","issues_url":"https://api.github.com/repos/maksimandrianov/omim/issues{/number}","pulls_url":"https://api.github.com/repos/maksimandrianov/omim/pulls{/number}","milestones_url":"https://api.github.com/repos/maksimandrianov/omim/milestones{/number}","notifications_url":"https://api.github.com/repos/maksimandrianov/omim/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/maksimandrianov/omim/labels{/name}","releases_url":"https://api.github.com/repos/maksimandrianov/omim/releases{/id}","deployments_url":"https://api.github.com/repos/maksimandrianov/omim/deployments","created_at":"2018-05-14T16:11:26Z","updated_at":"2018-11-07T16:25:03Z","pushed_at":"2018-11-23T14:00:17Z","git_url":"git://github.com/maksimandrianov/omim.git","ssh_url":"git@github.com:maksimandrianov/omim.git","clone_url":"https://github.com/maksimandrianov/omim.git","svn_url":"https://github.com/maksimandrianov/omim","homepage":"","size":4176350,"stargazers_count":0,"watchers_count":0,"language":"C++","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":0,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0","node_id":"MDc6TGljZW5zZTI="},"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"mapsme:master","ref":"master","sha":"5ecc13b454157050c9c0bb119beac2fdaf307deb","user":{"login":"mapsme","id":14185141,"node_id":"MDEyOk9yZ2FuaXphdGlvbjE0MTg1MTQx","avatar_url":"https://avatars1.githubusercontent.com/u/14185141?v=4","gravatar_id":"","url":"https://api.github.com/users/mapsme","html_url":"https://github.com/mapsme","followers_url":"https://api.github.com/users/mapsme/followers","following_url":"https://api.github.com/users/mapsme/following{/other_user}","gists_url":"https://api.github.com/users/mapsme/gists{/gist_id}","starred_url":"https://api.github.com/users/mapsme/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mapsme/subscriptions","organizations_url":"https://api.github.com/users/mapsme/orgs","repos_url":"https://api.github.com/users/mapsme/repos","events_url":"https://api.github.com/users/mapsme/events{/privacy}","received_events_url":"https://api.github.com/users/mapsme/received_events","type":"Organization","site_admin":false},"repo":{"id":42798033,"node_id":"MDEwOlJlcG9zaXRvcnk0Mjc5ODAzMw==","name":"omim","full_name":"mapsme/omim","private":false,"owner":{"login":"mapsme","id":14185141,"node_id":"MDEyOk9yZ2FuaXphdGlvbjE0MTg1MTQx","avatar_url":"https://avatars1.githubusercontent.com/u/14185141?v=4","gravatar_id":"","url":"https://api.github.com/users/mapsme","html_url":"https://github.com/mapsme","followers_url":"https://api.github.com/users/mapsme/followers","following_url":"https://api.github.com/users/mapsme/following{/other_user}","gists_url":"https://api.github.com/users/mapsme/gists{/gist_id}","starred_url":"https://api.github.com/users/mapsme/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mapsme/subscriptions","organizations_url":"https://api.github.com/users/mapsme/orgs","repos_url":"https://api.github.com/users/mapsme/repos","events_url":"https://api.github.com/users/mapsme/events{/privacy}","received_events_url":"https://api.github.com/users/mapsme/received_events","type":"Organization","site_admin":false},"html_url":"https://github.com/mapsme/omim","description":"🗺️ MAPS.ME — Offline OpenStreetMap maps for iOS and Android","fork":false,"url":"https://api.github.com/repos/mapsme/omim","forks_url":"https://api.github.com/repos/mapsme/omim/forks","keys_url":"https://api.github.com/repos/mapsme/omim/keys{/key_id}","collaborators_url":"https://api.github.com/repos/mapsme/omim/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/mapsme/omim/teams","hooks_url":"https://api.github.com/repos/mapsme/omim/hooks","issue_events_url":"https://api.github.com/repos/mapsme/omim/issues/events{/number}","events_url":"https://api.github.com/repos/mapsme/omim/events","assignees_url":"https://api.github.com/repos/mapsme/omim/assignees{/user}","branches_url":"https://api.github.com/repos/mapsme/omim/branches{/branch}","tags_url":"https://api.github.com/repos/mapsme/omim/tags","blobs_url":"https://api.github.com/repos/mapsme/omim/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/mapsme/omim/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/mapsme/omim/git/refs{/sha}","trees_url":"https://api.github.com/repos/mapsme/omim/git/trees{/sha}","statuses_url":"https://api.github.com/repos/mapsme/omim/statuses/{sha}","languages_url":"https://api.github.com/repos/mapsme/omim/languages","stargazers_url":"https://api.github.com/repos/mapsme/omim/stargazers","contributors_url":"https://api.github.com/repos/mapsme/omim/contributors","subscribers_url":"https://api.github.com/repos/mapsme/omim/subscribers","subscription_url":"https://api.github.com/repos/mapsme/omim/subscription","commits_url":"https://api.github.com/repos/mapsme/omim/commits{/sha}","git_commits_url":"https://api.github.com/repos/mapsme/omim/git/commits{/sha}","comments_url":"https://api.github.com/repos/mapsme/omim/comments{/number}","issue_comment_url":"https://api.github.com/repos/mapsme/omim/issues/comments{/number}","contents_url":"https://api.github.com/repos/mapsme/omim/contents/{+path}","compare_url":"https://api.github.com/repos/mapsme/omim/compare/{base}...{head}","merges_url":"https://api.github.com/repos/mapsme/omim/merges","archive_url":"https://api.github.com/repos/mapsme/omim/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/mapsme/omim/downloads","issues_url":"https://api.github.com/repos/mapsme/omim/issues{/number}","pulls_url":"https://api.github.com/repos/mapsme/omim/pulls{/number}","milestones_url":"https://api.github.com/repos/mapsme/omim/milestones{/number}","notifications_url":"https://api.github.com/repos/mapsme/omim/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/mapsme/omim/labels{/name}","releases_url":"https://api.github.com/repos/mapsme/omim/releases{/id}","deployments_url":"https://api.github.com/repos/mapsme/omim/deployments","created_at":"2015-09-20T02:57:13Z","updated_at":"2018-11-23T14:31:26Z","pushed_at":"2018-11-23T14:31:16Z","git_url":"git://github.com/mapsme/omim.git","ssh_url":"git@github.com:mapsme/omim.git","clone_url":"https://github.com/mapsme/omim.git","svn_url":"https://github.com/mapsme/omim","homepage":"","size":4216882,"stargazers_count":2842,"watchers_count":2842,"language":"C++","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":617,"mirror_url":null,"archived":false,"open_issues_count":762,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0","node_id":"MDc6TGljZW5zZTI="},"forks":617,"open_issues":762,"watchers":2842,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/mapsme/omim/pulls/9811"},"html":{"href":"https://github.com/mapsme/omim/pull/9811"},"issue":{"href":"https://api.github.com/repos/mapsme/omim/issues/9811"},"comments":{"href":"https://api.github.com/repos/mapsme/omim/issues/9811/comments"},"review_comments":{"href":"https://api.github.com/repos/mapsme/omim/pulls/9811/comments"},"review_comment":{"href":"https://api.github.com/repos/mapsme/omim/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/mapsme/omim/pulls/9811/commits"},"statuses":{"href":"https://api.github.com/repos/mapsme/omim/statuses/4d9630e01aac071e471ce850a277c79369c2e09c"}},"author_association":"CONTRIBUTOR"}}
{"actor":{"display_login":"maksimandrianov"}}