{"action":"created","comment":{"url":"https://api.github.com/repos/mapsme/omim/pulls/comments/175046156","pull_request_review_id":104513225,"id":175046156,"diff_hunk":"@@ -0,0 +1,135 @@\n+#!/usr/bin/env python3\n+# -*- coding: utf-8 -*-\n+\n+import argparse\n+import os\n+import re\n+import time\n+import sys\n+import urllib.request\n+import logging\n+\n+from concurrent.futures import ThreadPoolExecutor, as_completed\n+\n+\n+DIRECT_MAP_URL = 'http://direct.mapswithme.com/direct/'\n+ROOT = os.path.dirname(os.path.realpath(__file__))\n+DEFAULT_DOWNLOAD_DIRECTORY = os.path.join(os.path.dirname(os.path.dirname(ROOT)), 'data')\n+MAP_REVISION_REGEXP = re.compile(r'(\\d{6})')\n+MWM_NAME_REGEXP = re.compile(r'\"(\\S+).mwm\"')\n+\n+logger = logging.getLogger(__name__)\n+\n+\n+def parse_html_by_regexp(url, regexp):\n+    parse_result = []\n+    with urllib.request.urlopen(url) as f:\n+        lines = f.read().decode('utf-8').split('\\n')\n+        for line in lines:\n+            match = regexp.search(line)\n+            if match:\n+                parse_result.append(match.group(1))\n+    return parse_result\n+\n+\n+def get_possible_mwm_revisions():\n+    return [int(revision) for revision in parse_html_by_regexp(DIRECT_MAP_URL, MAP_REVISION_REGEXP)]\n+\n+\n+def get_possible_mwm_names(revision_num):\n+    revision_url = '{base}{revision}'.format(base=DIRECT_MAP_URL, revision=revision_num)\n+    return [name.replace('%20', ' ') for name in parse_html_by_regexp(revision_url, MWM_NAME_REGEXP)]\n+\n+\n+def download_file(url, filename, attempts=3):\n+    try:\n+        urllib.request.urlretrieve(url, filename=filename)\n+    except:\n+        logger.error('Map with name {name} not loaded'.format(name=os.path.basename(filename)), exc_info=True)\n+        if attempts:\n+            download_file(url, filename, attempts-1)\n+        return False\n+    return True\n+\n+\n+def progress_bar(total, progress):\n+    \"\"\"\n+    Displays or updates a console progress bar.\n+\n+    Original source: https://stackoverflow.com/a/15860757/1391441\n+    \"\"\"\n+    initial_progress = progress\n+    bar_length = 20\n+    progress = float(progress) / float(total)\n+    block = int(round(bar_length * progress))\n+    text = '\\r[{}] {:.0f}% {}/{}'.format(\n+        '#' * block + '-' * (bar_length - block), round(progress * 100, 0), initial_progress, total)","path":"tools/python/mwm_downloader.py","position":66,"original_position":66,"commit_id":"78ef55f4601f82682d5a2b57110fc7261dd0695d","original_commit_id":"78ef55f4601f82682d5a2b57110fc7261dd0695d","user":{"login":"burivuh","id":2204417,"avatar_url":"https://avatars3.githubusercontent.com/u/2204417?v=4","gravatar_id":"","url":"https://api.github.com/users/burivuh","html_url":"https://github.com/burivuh","followers_url":"https://api.github.com/users/burivuh/followers","following_url":"https://api.github.com/users/burivuh/following{/other_user}","gists_url":"https://api.github.com/users/burivuh/gists{/gist_id}","starred_url":"https://api.github.com/users/burivuh/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/burivuh/subscriptions","organizations_url":"https://api.github.com/users/burivuh/orgs","repos_url":"https://api.github.com/users/burivuh/repos","events_url":"https://api.github.com/users/burivuh/events{/privacy}","received_events_url":"https://api.github.com/users/burivuh/received_events","type":"User","site_admin":false},"body":"add another line for a better formatting please","created_at":"2018-03-16T10:17:05Z","updated_at":"2018-03-16T10:17:05Z","html_url":"https://github.com/mapsme/omim/pull/8319#discussion_r175046156","pull_request_url":"https://api.github.com/repos/mapsme/omim/pulls/8319","author_association":"CONTRIBUTOR","_links":{"self":{"href":"https://api.github.com/repos/mapsme/omim/pulls/comments/175046156"},"html":{"href":"https://github.com/mapsme/omim/pull/8319#discussion_r175046156"},"pull_request":{"href":"https://api.github.com/repos/mapsme/omim/pulls/8319"}}},"pull_request":{"url":"https://api.github.com/repos/mapsme/omim/pulls/8319","id":175472923,"html_url":"https://github.com/mapsme/omim/pull/8319","diff_url":"https://github.com/mapsme/omim/pull/8319.diff","patch_url":"https://github.com/mapsme/omim/pull/8319.patch","issue_url":"https://api.github.com/repos/mapsme/omim/issues/8319","number":8319,"state":"open","locked":false,"title":"[python] Mwm downloader script","user":{"login":"greshilov","id":814614,"avatar_url":"https://avatars1.githubusercontent.com/u/814614?v=4","gravatar_id":"","url":"https://api.github.com/users/greshilov","html_url":"https://github.com/greshilov","followers_url":"https://api.github.com/users/greshilov/followers","following_url":"https://api.github.com/users/greshilov/following{/other_user}","gists_url":"https://api.github.com/users/greshilov/gists{/gist_id}","starred_url":"https://api.github.com/users/greshilov/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/greshilov/subscriptions","organizations_url":"https://api.github.com/users/greshilov/orgs","repos_url":"https://api.github.com/users/greshilov/repos","events_url":"https://api.github.com/users/greshilov/events{/privacy}","received_events_url":"https://api.github.com/users/greshilov/received_events","type":"User","site_admin":false},"body":"Little python 3 script to download mwm files from `direct.mapswithme.com`.\r\nNeeded for my CI tasks. For example to prepare environment before `routing_integration_tests`.\r\n**Usage:**\r\nDownload all latest maps in `data/{maps_version}`\r\n```\r\ntools/python/mwm_downloader.py \r\n\r\nStart downloading maps within 8 threads\r\n[--------------------] 0% 4/1152\r\n```\r\nDownload 180126 version of Armenia.mwm and Albania.mwm in directory `foo` within 2 threads.\r\n```\r\ntools/python/mwm_downloader.py -r 180126 -m Armenia Albania -d foo -t 2\r\n\r\nStart downloading maps within 2 threads\r\n[####################] 100% 2/2\r\nFinished in 2 s.\r\n```","created_at":"2018-03-16T08:57:54Z","updated_at":"2018-03-16T10:17:05Z","closed_at":null,"merged_at":null,"merge_commit_sha":"243248e4e8e196d7ab8d218c96dddc9442f637af","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/mapsme/omim/pulls/8319/commits","review_comments_url":"https://api.github.com/repos/mapsme/omim/pulls/8319/comments","review_comment_url":"https://api.github.com/repos/mapsme/omim/pulls/comments{/number}","comments_url":"https://api.github.com/repos/mapsme/omim/issues/8319/comments","statuses_url":"https://api.github.com/repos/mapsme/omim/statuses/78ef55f4601f82682d5a2b57110fc7261dd0695d","head":{"label":"greshilov:python-mwm-downloader","ref":"python-mwm-downloader","sha":"78ef55f4601f82682d5a2b57110fc7261dd0695d","user":{"login":"greshilov","id":814614,"avatar_url":"https://avatars1.githubusercontent.com/u/814614?v=4","gravatar_id":"","url":"https://api.github.com/users/greshilov","html_url":"https://github.com/greshilov","followers_url":"https://api.github.com/users/greshilov/followers","following_url":"https://api.github.com/users/greshilov/following{/other_user}","gists_url":"https://api.github.com/users/greshilov/gists{/gist_id}","starred_url":"https://api.github.com/users/greshilov/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/greshilov/subscriptions","organizations_url":"https://api.github.com/users/greshilov/orgs","repos_url":"https://api.github.com/users/greshilov/repos","events_url":"https://api.github.com/users/greshilov/events{/privacy}","received_events_url":"https://api.github.com/users/greshilov/received_events","type":"User","site_admin":false},"repo":{"id":108246161,"name":"omim","full_name":"greshilov/omim","owner":{"login":"greshilov","id":814614,"avatar_url":"https://avatars1.githubusercontent.com/u/814614?v=4","gravatar_id":"","url":"https://api.github.com/users/greshilov","html_url":"https://github.com/greshilov","followers_url":"https://api.github.com/users/greshilov/followers","following_url":"https://api.github.com/users/greshilov/following{/other_user}","gists_url":"https://api.github.com/users/greshilov/gists{/gist_id}","starred_url":"https://api.github.com/users/greshilov/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/greshilov/subscriptions","organizations_url":"https://api.github.com/users/greshilov/orgs","repos_url":"https://api.github.com/users/greshilov/repos","events_url":"https://api.github.com/users/greshilov/events{/privacy}","received_events_url":"https://api.github.com/users/greshilov/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/greshilov/omim","description":"🗺️ MAPS.ME — Offline OpenStreetMap maps for iOS and Android","fork":true,"url":"https://api.github.com/repos/greshilov/omim","forks_url":"https://api.github.com/repos/greshilov/omim/forks","keys_url":"https://api.github.com/repos/greshilov/omim/keys{/key_id}","collaborators_url":"https://api.github.com/repos/greshilov/omim/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/greshilov/omim/teams","hooks_url":"https://api.github.com/repos/greshilov/omim/hooks","issue_events_url":"https://api.github.com/repos/greshilov/omim/issues/events{/number}","events_url":"https://api.github.com/repos/greshilov/omim/events","assignees_url":"https://api.github.com/repos/greshilov/omim/assignees{/user}","branches_url":"https://api.github.com/repos/greshilov/omim/branches{/branch}","tags_url":"https://api.github.com/repos/greshilov/omim/tags","blobs_url":"https://api.github.com/repos/greshilov/omim/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/greshilov/omim/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/greshilov/omim/git/refs{/sha}","trees_url":"https://api.github.com/repos/greshilov/omim/git/trees{/sha}","statuses_url":"https://api.github.com/repos/greshilov/omim/statuses/{sha}","languages_url":"https://api.github.com/repos/greshilov/omim/languages","stargazers_url":"https://api.github.com/repos/greshilov/omim/stargazers","contributors_url":"https://api.github.com/repos/greshilov/omim/contributors","subscribers_url":"https://api.github.com/repos/greshilov/omim/subscribers","subscription_url":"https://api.github.com/repos/greshilov/omim/subscription","commits_url":"https://api.github.com/repos/greshilov/omim/commits{/sha}","git_commits_url":"https://api.github.com/repos/greshilov/omim/git/commits{/sha}","comments_url":"https://api.github.com/repos/greshilov/omim/comments{/number}","issue_comment_url":"https://api.github.com/repos/greshilov/omim/issues/comments{/number}","contents_url":"https://api.github.com/repos/greshilov/omim/contents/{+path}","compare_url":"https://api.github.com/repos/greshilov/omim/compare/{base}...{head}","merges_url":"https://api.github.com/repos/greshilov/omim/merges","archive_url":"https://api.github.com/repos/greshilov/omim/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/greshilov/omim/downloads","issues_url":"https://api.github.com/repos/greshilov/omim/issues{/number}","pulls_url":"https://api.github.com/repos/greshilov/omim/pulls{/number}","milestones_url":"https://api.github.com/repos/greshilov/omim/milestones{/number}","notifications_url":"https://api.github.com/repos/greshilov/omim/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/greshilov/omim/labels{/name}","releases_url":"https://api.github.com/repos/greshilov/omim/releases{/id}","deployments_url":"https://api.github.com/repos/greshilov/omim/deployments","created_at":"2017-10-25T09:07:40Z","updated_at":"2018-03-16T08:41:21Z","pushed_at":"2018-03-16T09:17:46Z","git_url":"git://github.com/greshilov/omim.git","ssh_url":"git@github.com:greshilov/omim.git","clone_url":"https://github.com/greshilov/omim.git","svn_url":"https://github.com/greshilov/omim","homepage":"","size":3641218,"stargazers_count":0,"watchers_count":0,"language":"C++","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":0,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0"},"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"mapsme:master","ref":"master","sha":"224ac2641b9015d06d0bcd062bf200122c3434f8","user":{"login":"mapsme","id":14185141,"avatar_url":"https://avatars1.githubusercontent.com/u/14185141?v=4","gravatar_id":"","url":"https://api.github.com/users/mapsme","html_url":"https://github.com/mapsme","followers_url":"https://api.github.com/users/mapsme/followers","following_url":"https://api.github.com/users/mapsme/following{/other_user}","gists_url":"https://api.github.com/users/mapsme/gists{/gist_id}","starred_url":"https://api.github.com/users/mapsme/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mapsme/subscriptions","organizations_url":"https://api.github.com/users/mapsme/orgs","repos_url":"https://api.github.com/users/mapsme/repos","events_url":"https://api.github.com/users/mapsme/events{/privacy}","received_events_url":"https://api.github.com/users/mapsme/received_events","type":"Organization","site_admin":false},"repo":{"id":42798033,"name":"omim","full_name":"mapsme/omim","owner":{"login":"mapsme","id":14185141,"avatar_url":"https://avatars1.githubusercontent.com/u/14185141?v=4","gravatar_id":"","url":"https://api.github.com/users/mapsme","html_url":"https://github.com/mapsme","followers_url":"https://api.github.com/users/mapsme/followers","following_url":"https://api.github.com/users/mapsme/following{/other_user}","gists_url":"https://api.github.com/users/mapsme/gists{/gist_id}","starred_url":"https://api.github.com/users/mapsme/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mapsme/subscriptions","organizations_url":"https://api.github.com/users/mapsme/orgs","repos_url":"https://api.github.com/users/mapsme/repos","events_url":"https://api.github.com/users/mapsme/events{/privacy}","received_events_url":"https://api.github.com/users/mapsme/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/mapsme/omim","description":"🗺️ MAPS.ME — Offline OpenStreetMap maps for iOS and Android","fork":false,"url":"https://api.github.com/repos/mapsme/omim","forks_url":"https://api.github.com/repos/mapsme/omim/forks","keys_url":"https://api.github.com/repos/mapsme/omim/keys{/key_id}","collaborators_url":"https://api.github.com/repos/mapsme/omim/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/mapsme/omim/teams","hooks_url":"https://api.github.com/repos/mapsme/omim/hooks","issue_events_url":"https://api.github.com/repos/mapsme/omim/issues/events{/number}","events_url":"https://api.github.com/repos/mapsme/omim/events","assignees_url":"https://api.github.com/repos/mapsme/omim/assignees{/user}","branches_url":"https://api.github.com/repos/mapsme/omim/branches{/branch}","tags_url":"https://api.github.com/repos/mapsme/omim/tags","blobs_url":"https://api.github.com/repos/mapsme/omim/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/mapsme/omim/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/mapsme/omim/git/refs{/sha}","trees_url":"https://api.github.com/repos/mapsme/omim/git/trees{/sha}","statuses_url":"https://api.github.com/repos/mapsme/omim/statuses/{sha}","languages_url":"https://api.github.com/repos/mapsme/omim/languages","stargazers_url":"https://api.github.com/repos/mapsme/omim/stargazers","contributors_url":"https://api.github.com/repos/mapsme/omim/contributors","subscribers_url":"https://api.github.com/repos/mapsme/omim/subscribers","subscription_url":"https://api.github.com/repos/mapsme/omim/subscription","commits_url":"https://api.github.com/repos/mapsme/omim/commits{/sha}","git_commits_url":"https://api.github.com/repos/mapsme/omim/git/commits{/sha}","comments_url":"https://api.github.com/repos/mapsme/omim/comments{/number}","issue_comment_url":"https://api.github.com/repos/mapsme/omim/issues/comments{/number}","contents_url":"https://api.github.com/repos/mapsme/omim/contents/{+path}","compare_url":"https://api.github.com/repos/mapsme/omim/compare/{base}...{head}","merges_url":"https://api.github.com/repos/mapsme/omim/merges","archive_url":"https://api.github.com/repos/mapsme/omim/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/mapsme/omim/downloads","issues_url":"https://api.github.com/repos/mapsme/omim/issues{/number}","pulls_url":"https://api.github.com/repos/mapsme/omim/pulls{/number}","milestones_url":"https://api.github.com/repos/mapsme/omim/milestones{/number}","notifications_url":"https://api.github.com/repos/mapsme/omim/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/mapsme/omim/labels{/name}","releases_url":"https://api.github.com/repos/mapsme/omim/releases{/id}","deployments_url":"https://api.github.com/repos/mapsme/omim/deployments","created_at":"2015-09-20T02:57:13Z","updated_at":"2018-03-16T10:12:02Z","pushed_at":"2018-03-16T10:14:00Z","git_url":"git://github.com/mapsme/omim.git","ssh_url":"git@github.com:mapsme/omim.git","clone_url":"https://github.com/mapsme/omim.git","svn_url":"https://github.com/mapsme/omim","homepage":"","size":3658574,"stargazers_count":2606,"watchers_count":2606,"language":"C++","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":564,"mirror_url":null,"archived":false,"open_issues_count":659,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0"},"forks":564,"open_issues":659,"watchers":2606,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/mapsme/omim/pulls/8319"},"html":{"href":"https://github.com/mapsme/omim/pull/8319"},"issue":{"href":"https://api.github.com/repos/mapsme/omim/issues/8319"},"comments":{"href":"https://api.github.com/repos/mapsme/omim/issues/8319/comments"},"review_comments":{"href":"https://api.github.com/repos/mapsme/omim/pulls/8319/comments"},"review_comment":{"href":"https://api.github.com/repos/mapsme/omim/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/mapsme/omim/pulls/8319/commits"},"statuses":{"href":"https://api.github.com/repos/mapsme/omim/statuses/78ef55f4601f82682d5a2b57110fc7261dd0695d"}},"author_association":"CONTRIBUTOR"}}
{"actor":{"display_login":"burivuh"}}