{"action":"created","comment":{"url":"https://api.github.com/repos/mapsme/omim/pulls/comments/204464914","pull_request_review_id":139541878,"id":204464914,"node_id":"MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDIwNDQ2NDkxNA==","diff_hunk":"@@ -5,61 +5,69 @@\n import re\n import time\n import sys\n+import json\n+import itertools\n import urllib.request\n import urllib.parse\n import urllib.error\n import logging\n from multiprocessing.pool import ThreadPool\n \n-\n-DIRECT_MAP_URL = 'http://direct.mapswithme.com/direct/'\n ROOT = os.path.dirname(os.path.realpath(__file__))\n-DEFAULT_DOWNLOAD_DIRECTORY = os.path.join(os.path.dirname(os.path.dirname(ROOT)), 'data')\n-MAP_REVISION_REGEXP = re.compile(r'(\\d{6})')\n+OMIM_ROOT = os.path.dirname(os.path.dirname(ROOT))\n+DEFAULT_DOWNLOAD_DIRECTORY = os.path.join(OMIM_ROOT, 'data')\n+COUNTRIES_TXT = os.path.join(OMIM_ROOT, 'data', 'countries.txt')\n+URL_PATTERN = 'http://{prefix}.mapswithme.com/direct/{version}/{name}.mwm'\n+MAP_SERVERS = ('maps-dl-ru1', 'maps-dl-ru2', 'maps-dl-ru3', 'maps-dl-ams1',\n+               'maps-dl-eu2', 'maps-dl-us1')\n MWM_NAME_REGEXP = re.compile(r'\"(\\S+\\.mwm)\"')\n \n logger = logging.getLogger(__name__)\n \n \n-def parse_html_by_regexp(url, regexp):\n-    def parse_line(line):\n-        return regexp.search(line.decode('utf-8'))\n-\n-    with urllib.request.urlopen(url) as http_response_lines:\n-        filtered_lines = filter(None, map(parse_line, http_response_lines))\n-        for line in filtered_lines:\n-            yield line.group(1)\n-\n-\n-def get_mwm_versions():\n-    return map(int, parse_html_by_regexp(DIRECT_MAP_URL, MAP_REVISION_REGEXP))\n+def country_names_generator(country_obj):\n+    if 'g' in country_obj:\n+        yield from country_names_generator(country_obj['g'])\n+    elif 's' in country_obj:\n+        yield country_obj['id']\n+    elif type(country_obj) is list:\n+        for country in country_obj:\n+            yield from country_names_generator(country)\n \n \n-def get_mwm_names(version_num):\n-    version_url = '{base}{version}'.format(base=DIRECT_MAP_URL, version=version_num)\n-    return map(urllib.parse.unquote, parse_html_by_regexp(version_url, MWM_NAME_REGEXP))\n-\n-\n-def download_file(arg_tuple, attempts=3):\n-    url, filename = arg_tuple\n+def download_file(url, filename, attempts=3):\n     try:\n         urllib.request.urlretrieve(url, filename=filename)\n     except urllib.error.HTTPError as err:\n         if 400 <= err.code < 500:\n-            logger.error('URL \\'{url}\\' is not found'.format(url=url), exc_info=True)\n+            logger.error('URL \\'{url}\\' is not found'.format(url=url))\n             return False\n     except PermissionError:\n-        logger.error('Can\\'t write file {filename}: permission denied'.format(filename=filename), exc_info=True)\n+        logger.error('Can\\'t write file {filename}: permission denied'.format(filename=filename))\n         return False\n     except urllib.error.URLError:\n         if attempts:","path":"tools/python/mwm_downloader.py","position":70,"original_position":70,"commit_id":"a6c17a73180e108b829260edf17a6bfca8060f72","original_commit_id":"a6c17a73180e108b829260edf17a6bfca8060f72","user":{"login":"therearesomewhocallmetim","id":14867234,"node_id":"MDQ6VXNlcjE0ODY3MjM0","avatar_url":"https://avatars1.githubusercontent.com/u/14867234?v=4","gravatar_id":"","url":"https://api.github.com/users/therearesomewhocallmetim","html_url":"https://github.com/therearesomewhocallmetim","followers_url":"https://api.github.com/users/therearesomewhocallmetim/followers","following_url":"https://api.github.com/users/therearesomewhocallmetim/following{/other_user}","gists_url":"https://api.github.com/users/therearesomewhocallmetim/gists{/gist_id}","starred_url":"https://api.github.com/users/therearesomewhocallmetim/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/therearesomewhocallmetim/subscriptions","organizations_url":"https://api.github.com/users/therearesomewhocallmetim/orgs","repos_url":"https://api.github.com/users/therearesomewhocallmetim/repos","events_url":"https://api.github.com/users/therearesomewhocallmetim/events{/privacy}","received_events_url":"https://api.github.com/users/therearesomewhocallmetim/received_events","type":"User","site_admin":false},"body":"Вот на всякий случай я бы поставил `if attempts > 0`. Потому что `bool(-1) == True`","created_at":"2018-07-23T16:12:17Z","updated_at":"2018-07-23T16:12:34Z","html_url":"https://github.com/mapsme/omim/pull/9169#discussion_r204464914","pull_request_url":"https://api.github.com/repos/mapsme/omim/pulls/9169","author_association":"CONTRIBUTOR","_links":{"self":{"href":"https://api.github.com/repos/mapsme/omim/pulls/comments/204464914"},"html":{"href":"https://github.com/mapsme/omim/pull/9169#discussion_r204464914"},"pull_request":{"href":"https://api.github.com/repos/mapsme/omim/pulls/9169"}}},"pull_request":{"url":"https://api.github.com/repos/mapsme/omim/pulls/9169","id":203245340,"node_id":"MDExOlB1bGxSZXF1ZXN0MjAzMjQ1MzQw","html_url":"https://github.com/mapsme/omim/pull/9169","diff_url":"https://github.com/mapsme/omim/pull/9169.diff","patch_url":"https://github.com/mapsme/omim/pull/9169.patch","issue_url":"https://api.github.com/repos/mapsme/omim/issues/9169","number":9169,"state":"open","locked":false,"title":"[python] Update python mwm_downloader","user":{"login":"greshilov","id":814614,"node_id":"MDQ6VXNlcjgxNDYxNA==","avatar_url":"https://avatars1.githubusercontent.com/u/814614?v=4","gravatar_id":"","url":"https://api.github.com/users/greshilov","html_url":"https://github.com/greshilov","followers_url":"https://api.github.com/users/greshilov/followers","following_url":"https://api.github.com/users/greshilov/following{/other_user}","gists_url":"https://api.github.com/users/greshilov/gists{/gist_id}","starred_url":"https://api.github.com/users/greshilov/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/greshilov/subscriptions","organizations_url":"https://api.github.com/users/greshilov/orgs","repos_url":"https://api.github.com/users/greshilov/repos","events_url":"https://api.github.com/users/greshilov/events{/privacy}","received_events_url":"https://api.github.com/users/greshilov/received_events","type":"User","site_admin":false},"body":"`country_names_generator` based on `check_item` from `cdncheck` repo. ","created_at":"2018-07-23T14:52:16Z","updated_at":"2018-07-23T16:12:34Z","closed_at":null,"merged_at":null,"merge_commit_sha":"dc0d1c17efc861184ecefc8afab066d4ef9c6fe3","assignee":null,"assignees":[],"requested_reviewers":[{"login":"burivuh","id":2204417,"node_id":"MDQ6VXNlcjIyMDQ0MTc=","avatar_url":"https://avatars3.githubusercontent.com/u/2204417?v=4","gravatar_id":"","url":"https://api.github.com/users/burivuh","html_url":"https://github.com/burivuh","followers_url":"https://api.github.com/users/burivuh/followers","following_url":"https://api.github.com/users/burivuh/following{/other_user}","gists_url":"https://api.github.com/users/burivuh/gists{/gist_id}","starred_url":"https://api.github.com/users/burivuh/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/burivuh/subscriptions","organizations_url":"https://api.github.com/users/burivuh/orgs","repos_url":"https://api.github.com/users/burivuh/repos","events_url":"https://api.github.com/users/burivuh/events{/privacy}","received_events_url":"https://api.github.com/users/burivuh/received_events","type":"User","site_admin":false}],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/mapsme/omim/pulls/9169/commits","review_comments_url":"https://api.github.com/repos/mapsme/omim/pulls/9169/comments","review_comment_url":"https://api.github.com/repos/mapsme/omim/pulls/comments{/number}","comments_url":"https://api.github.com/repos/mapsme/omim/issues/9169/comments","statuses_url":"https://api.github.com/repos/mapsme/omim/statuses/a6c17a73180e108b829260edf17a6bfca8060f72","head":{"label":"greshilov:mwm-downloader-upgrade","ref":"mwm-downloader-upgrade","sha":"a6c17a73180e108b829260edf17a6bfca8060f72","user":{"login":"greshilov","id":814614,"node_id":"MDQ6VXNlcjgxNDYxNA==","avatar_url":"https://avatars1.githubusercontent.com/u/814614?v=4","gravatar_id":"","url":"https://api.github.com/users/greshilov","html_url":"https://github.com/greshilov","followers_url":"https://api.github.com/users/greshilov/followers","following_url":"https://api.github.com/users/greshilov/following{/other_user}","gists_url":"https://api.github.com/users/greshilov/gists{/gist_id}","starred_url":"https://api.github.com/users/greshilov/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/greshilov/subscriptions","organizations_url":"https://api.github.com/users/greshilov/orgs","repos_url":"https://api.github.com/users/greshilov/repos","events_url":"https://api.github.com/users/greshilov/events{/privacy}","received_events_url":"https://api.github.com/users/greshilov/received_events","type":"User","site_admin":false},"repo":{"id":108246161,"node_id":"MDEwOlJlcG9zaXRvcnkxMDgyNDYxNjE=","name":"omim","full_name":"greshilov/omim","owner":{"login":"greshilov","id":814614,"node_id":"MDQ6VXNlcjgxNDYxNA==","avatar_url":"https://avatars1.githubusercontent.com/u/814614?v=4","gravatar_id":"","url":"https://api.github.com/users/greshilov","html_url":"https://github.com/greshilov","followers_url":"https://api.github.com/users/greshilov/followers","following_url":"https://api.github.com/users/greshilov/following{/other_user}","gists_url":"https://api.github.com/users/greshilov/gists{/gist_id}","starred_url":"https://api.github.com/users/greshilov/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/greshilov/subscriptions","organizations_url":"https://api.github.com/users/greshilov/orgs","repos_url":"https://api.github.com/users/greshilov/repos","events_url":"https://api.github.com/users/greshilov/events{/privacy}","received_events_url":"https://api.github.com/users/greshilov/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/greshilov/omim","description":"🗺️ MAPS.ME — Offline OpenStreetMap maps for iOS and Android","fork":true,"url":"https://api.github.com/repos/greshilov/omim","forks_url":"https://api.github.com/repos/greshilov/omim/forks","keys_url":"https://api.github.com/repos/greshilov/omim/keys{/key_id}","collaborators_url":"https://api.github.com/repos/greshilov/omim/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/greshilov/omim/teams","hooks_url":"https://api.github.com/repos/greshilov/omim/hooks","issue_events_url":"https://api.github.com/repos/greshilov/omim/issues/events{/number}","events_url":"https://api.github.com/repos/greshilov/omim/events","assignees_url":"https://api.github.com/repos/greshilov/omim/assignees{/user}","branches_url":"https://api.github.com/repos/greshilov/omim/branches{/branch}","tags_url":"https://api.github.com/repos/greshilov/omim/tags","blobs_url":"https://api.github.com/repos/greshilov/omim/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/greshilov/omim/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/greshilov/omim/git/refs{/sha}","trees_url":"https://api.github.com/repos/greshilov/omim/git/trees{/sha}","statuses_url":"https://api.github.com/repos/greshilov/omim/statuses/{sha}","languages_url":"https://api.github.com/repos/greshilov/omim/languages","stargazers_url":"https://api.github.com/repos/greshilov/omim/stargazers","contributors_url":"https://api.github.com/repos/greshilov/omim/contributors","subscribers_url":"https://api.github.com/repos/greshilov/omim/subscribers","subscription_url":"https://api.github.com/repos/greshilov/omim/subscription","commits_url":"https://api.github.com/repos/greshilov/omim/commits{/sha}","git_commits_url":"https://api.github.com/repos/greshilov/omim/git/commits{/sha}","comments_url":"https://api.github.com/repos/greshilov/omim/comments{/number}","issue_comment_url":"https://api.github.com/repos/greshilov/omim/issues/comments{/number}","contents_url":"https://api.github.com/repos/greshilov/omim/contents/{+path}","compare_url":"https://api.github.com/repos/greshilov/omim/compare/{base}...{head}","merges_url":"https://api.github.com/repos/greshilov/omim/merges","archive_url":"https://api.github.com/repos/greshilov/omim/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/greshilov/omim/downloads","issues_url":"https://api.github.com/repos/greshilov/omim/issues{/number}","pulls_url":"https://api.github.com/repos/greshilov/omim/pulls{/number}","milestones_url":"https://api.github.com/repos/greshilov/omim/milestones{/number}","notifications_url":"https://api.github.com/repos/greshilov/omim/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/greshilov/omim/labels{/name}","releases_url":"https://api.github.com/repos/greshilov/omim/releases{/id}","deployments_url":"https://api.github.com/repos/greshilov/omim/deployments","created_at":"2017-10-25T09:07:40Z","updated_at":"2018-07-23T16:02:00Z","pushed_at":"2018-07-23T16:01:59Z","git_url":"git://github.com/greshilov/omim.git","ssh_url":"git@github.com:greshilov/omim.git","clone_url":"https://github.com/greshilov/omim.git","svn_url":"https://github.com/greshilov/omim","homepage":"","size":3992026,"stargazers_count":0,"watchers_count":0,"language":"C++","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":0,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0","node_id":"MDc6TGljZW5zZTI="},"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"mapsme:master","ref":"master","sha":"08f38bd4fba8623603ec406307d715993aab4eed","user":{"login":"mapsme","id":14185141,"node_id":"MDEyOk9yZ2FuaXphdGlvbjE0MTg1MTQx","avatar_url":"https://avatars1.githubusercontent.com/u/14185141?v=4","gravatar_id":"","url":"https://api.github.com/users/mapsme","html_url":"https://github.com/mapsme","followers_url":"https://api.github.com/users/mapsme/followers","following_url":"https://api.github.com/users/mapsme/following{/other_user}","gists_url":"https://api.github.com/users/mapsme/gists{/gist_id}","starred_url":"https://api.github.com/users/mapsme/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mapsme/subscriptions","organizations_url":"https://api.github.com/users/mapsme/orgs","repos_url":"https://api.github.com/users/mapsme/repos","events_url":"https://api.github.com/users/mapsme/events{/privacy}","received_events_url":"https://api.github.com/users/mapsme/received_events","type":"Organization","site_admin":false},"repo":{"id":42798033,"node_id":"MDEwOlJlcG9zaXRvcnk0Mjc5ODAzMw==","name":"omim","full_name":"mapsme/omim","owner":{"login":"mapsme","id":14185141,"node_id":"MDEyOk9yZ2FuaXphdGlvbjE0MTg1MTQx","avatar_url":"https://avatars1.githubusercontent.com/u/14185141?v=4","gravatar_id":"","url":"https://api.github.com/users/mapsme","html_url":"https://github.com/mapsme","followers_url":"https://api.github.com/users/mapsme/followers","following_url":"https://api.github.com/users/mapsme/following{/other_user}","gists_url":"https://api.github.com/users/mapsme/gists{/gist_id}","starred_url":"https://api.github.com/users/mapsme/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mapsme/subscriptions","organizations_url":"https://api.github.com/users/mapsme/orgs","repos_url":"https://api.github.com/users/mapsme/repos","events_url":"https://api.github.com/users/mapsme/events{/privacy}","received_events_url":"https://api.github.com/users/mapsme/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/mapsme/omim","description":"🗺️ MAPS.ME — Offline OpenStreetMap maps for iOS and Android","fork":false,"url":"https://api.github.com/repos/mapsme/omim","forks_url":"https://api.github.com/repos/mapsme/omim/forks","keys_url":"https://api.github.com/repos/mapsme/omim/keys{/key_id}","collaborators_url":"https://api.github.com/repos/mapsme/omim/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/mapsme/omim/teams","hooks_url":"https://api.github.com/repos/mapsme/omim/hooks","issue_events_url":"https://api.github.com/repos/mapsme/omim/issues/events{/number}","events_url":"https://api.github.com/repos/mapsme/omim/events","assignees_url":"https://api.github.com/repos/mapsme/omim/assignees{/user}","branches_url":"https://api.github.com/repos/mapsme/omim/branches{/branch}","tags_url":"https://api.github.com/repos/mapsme/omim/tags","blobs_url":"https://api.github.com/repos/mapsme/omim/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/mapsme/omim/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/mapsme/omim/git/refs{/sha}","trees_url":"https://api.github.com/repos/mapsme/omim/git/trees{/sha}","statuses_url":"https://api.github.com/repos/mapsme/omim/statuses/{sha}","languages_url":"https://api.github.com/repos/mapsme/omim/languages","stargazers_url":"https://api.github.com/repos/mapsme/omim/stargazers","contributors_url":"https://api.github.com/repos/mapsme/omim/contributors","subscribers_url":"https://api.github.com/repos/mapsme/omim/subscribers","subscription_url":"https://api.github.com/repos/mapsme/omim/subscription","commits_url":"https://api.github.com/repos/mapsme/omim/commits{/sha}","git_commits_url":"https://api.github.com/repos/mapsme/omim/git/commits{/sha}","comments_url":"https://api.github.com/repos/mapsme/omim/comments{/number}","issue_comment_url":"https://api.github.com/repos/mapsme/omim/issues/comments{/number}","contents_url":"https://api.github.com/repos/mapsme/omim/contents/{+path}","compare_url":"https://api.github.com/repos/mapsme/omim/compare/{base}...{head}","merges_url":"https://api.github.com/repos/mapsme/omim/merges","archive_url":"https://api.github.com/repos/mapsme/omim/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/mapsme/omim/downloads","issues_url":"https://api.github.com/repos/mapsme/omim/issues{/number}","pulls_url":"https://api.github.com/repos/mapsme/omim/pulls{/number}","milestones_url":"https://api.github.com/repos/mapsme/omim/milestones{/number}","notifications_url":"https://api.github.com/repos/mapsme/omim/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/mapsme/omim/labels{/name}","releases_url":"https://api.github.com/repos/mapsme/omim/releases{/id}","deployments_url":"https://api.github.com/repos/mapsme/omim/deployments","created_at":"2015-09-20T02:57:13Z","updated_at":"2018-07-23T15:55:41Z","pushed_at":"2018-07-23T16:04:50Z","git_url":"git://github.com/mapsme/omim.git","ssh_url":"git@github.com:mapsme/omim.git","clone_url":"https://github.com/mapsme/omim.git","svn_url":"https://github.com/mapsme/omim","homepage":"","size":3999228,"stargazers_count":2718,"watchers_count":2718,"language":"C++","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":592,"mirror_url":null,"archived":false,"open_issues_count":719,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0","node_id":"MDc6TGljZW5zZTI="},"forks":592,"open_issues":719,"watchers":2718,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/mapsme/omim/pulls/9169"},"html":{"href":"https://github.com/mapsme/omim/pull/9169"},"issue":{"href":"https://api.github.com/repos/mapsme/omim/issues/9169"},"comments":{"href":"https://api.github.com/repos/mapsme/omim/issues/9169/comments"},"review_comments":{"href":"https://api.github.com/repos/mapsme/omim/pulls/9169/comments"},"review_comment":{"href":"https://api.github.com/repos/mapsme/omim/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/mapsme/omim/pulls/9169/commits"},"statuses":{"href":"https://api.github.com/repos/mapsme/omim/statuses/a6c17a73180e108b829260edf17a6bfca8060f72"}},"author_association":"CONTRIBUTOR"}}
{"actor":{"display_login":"therearesomewhocallmetim"}}