{"action":"created","comment":{"url":"https://api.github.com/repos/ros-infrastructure/rosinstall_generator/pulls/comments/74862394","id":74862394,"diff_hunk":"@@ -132,26 +133,38 @@ def _generate_rosinstall_for_package(distro, pkg_name, flat=False, tar=False):\n \n \n def _generate_rosinstall(local_name, url, release_tag, tar=False, vcs_type=None):\n+    logger = logging.getLogger('rosinstall_generator.generate')\n+\n     if tar:\n-        # the repository name might be different than repo.name coming from rosdistro\n-        repo_name = os.path.basename(url[:-4])\n-        suffix = '.git'\n-        if not url.endswith(suffix):\n-            raise RuntimeError(\"The repository '%s' must end with '%s': %s\" % (repo_name, suffix, url))\n-        url = url[:-len(suffix)] + '/archive/{0}.tar.gz'.format(release_tag)\n-        data = [{\n-            'tar': {\n-                'local-name': local_name,\n-                'uri': url,\n-                'version': '{0}-{1}'.format(repo_name, release_tag.replace('/', '-'))\n-            }\n-        }]\n-    else:\n-        data = [{\n-            vcs_type or 'git': {\n-                'local-name': local_name,\n-                'uri': url,\n-                'version': release_tag\n+        # Github tarball:    https://github.com/ros/ros_comm/archive/1.11.20.tar.gz\n+        # Bitbucket tarball: https://bitbucket.org/osrf/gazebo/get/gazebo7_7.3.1.tar.gz\n+        # Gitlab tarball:    https://gitlab.com/gitlab-org/gitlab-ce/repository/archive.tar.gz?ref=master\n+        match = re.match('(?:https?:\\/\\/|ssh:\\/\\/|git:\\/\\/|git@)([\\w.-]+)[:/]([\\w/-]*)(?:\\.git)?$', url)\n+\n+        if match:\n+            server, repo_path = match.groups()\n+            url_templates = {\n+                'github': 'https://{0}/{1}/archive/{2}.tar.gz',\n+                'bitbucket': 'https://{0}/{1}/get/{2}.tar.gz',\n+                'gitlab': 'https://{0}/{1}/repository/archive.tar.gz?ref={2}'\n             }\n-        }]\n-    return data\n+            for server_key, tarball_url_template in url_templates.items():\n+                if server_key in server:\n+                    return [{ 'tar': {\n+                        'local-name': local_name,\n+                        'uri': tarball_url_template.format(server, repo_path, release_tag),\n+                        'version': '{0}-{1}'.format(os.path.basename(repo_path), release_tag.replace('/', '-'))\n+                    }}]\n+            logger.log(logging.WARN, 'Tarball requested for repo {0}, but git server {1} is unrecognized. '.format(\n+                local_name, server))\n+        else:\n+            logger.log(logging.WARN, 'Tarball requested for repo {0}, but I can\\'t parse git URL {1}. '.format(\n+                local_name, url))\n+\n+        logger.log(logging.WARN, 'Falling back on git clone for repo {0}'.format(local_name))","path":"src/rosinstall_generator/distro.py","position":61,"original_position":61,"commit_id":"6cc76fc2d6cfa74c65b4234e06421e1e43598519","original_commit_id":"6cc76fc2d6cfa74c65b4234e06421e1e43598519","user":{"login":"mikepurvis","id":944666,"avatar_url":"https://avatars.githubusercontent.com/u/944666?v=3","gravatar_id":"","url":"https://api.github.com/users/mikepurvis","html_url":"https://github.com/mikepurvis","followers_url":"https://api.github.com/users/mikepurvis/followers","following_url":"https://api.github.com/users/mikepurvis/following{/other_user}","gists_url":"https://api.github.com/users/mikepurvis/gists{/gist_id}","starred_url":"https://api.github.com/users/mikepurvis/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mikepurvis/subscriptions","organizations_url":"https://api.github.com/users/mikepurvis/orgs","repos_url":"https://api.github.com/users/mikepurvis/repos","events_url":"https://api.github.com/users/mikepurvis/events{/privacy}","received_events_url":"https://api.github.com/users/mikepurvis/received_events","type":"User","site_admin":false},"body":"Downloading tarballs is an optimization over cloning repos, so it makes sense to me that the fallback behaviour should be to just clone, but I can change it to error out instead.","created_at":"2016-08-16T01:00:27Z","updated_at":"2016-08-16T01:00:27Z","html_url":"https://github.com/ros-infrastructure/rosinstall_generator/pull/37#discussion_r74862394","pull_request_url":"https://api.github.com/repos/ros-infrastructure/rosinstall_generator/pulls/37","_links":{"self":{"href":"https://api.github.com/repos/ros-infrastructure/rosinstall_generator/pulls/comments/74862394"},"html":{"href":"https://github.com/ros-infrastructure/rosinstall_generator/pull/37#discussion_r74862394"},"pull_request":{"href":"https://api.github.com/repos/ros-infrastructure/rosinstall_generator/pulls/37"}}},"pull_request":{"url":"https://api.github.com/repos/ros-infrastructure/rosinstall_generator/pulls/37","id":78739839,"html_url":"https://github.com/ros-infrastructure/rosinstall_generator/pull/37","diff_url":"https://github.com/ros-infrastructure/rosinstall_generator/pull/37.diff","patch_url":"https://github.com/ros-infrastructure/rosinstall_generator/pull/37.patch","issue_url":"https://api.github.com/repos/ros-infrastructure/rosinstall_generator/issues/37","number":37,"state":"open","locked":false,"title":"Change the tarball generator to work with bitbucket and gitlab URLs.","user":{"login":"mikepurvis","id":944666,"avatar_url":"https://avatars.githubusercontent.com/u/944666?v=3","gravatar_id":"","url":"https://api.github.com/users/mikepurvis","html_url":"https://github.com/mikepurvis","followers_url":"https://api.github.com/users/mikepurvis/followers","following_url":"https://api.github.com/users/mikepurvis/following{/other_user}","gists_url":"https://api.github.com/users/mikepurvis/gists{/gist_id}","starred_url":"https://api.github.com/users/mikepurvis/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mikepurvis/subscriptions","organizations_url":"https://api.github.com/users/mikepurvis/orgs","repos_url":"https://api.github.com/users/mikepurvis/repos","events_url":"https://api.github.com/users/mikepurvis/events{/privacy}","received_events_url":"https://api.github.com/users/mikepurvis/received_events","type":"User","site_admin":false},"body":"Since there's no standard scheme for getting a tarball URL from a git repo, this at least generalizes  the github-centric logic to also cover bitbucket and gitlab. Simple tests from our in-house rosdistro:\r\n\r\n```\r\n$ rosinstall_generator actionlib --tar --rosdistro indigo\r\n- tar:\r\n    local-name: actionlib\r\n    uri: https://gitlab.clearpathrobotics.com/gbp/actionlib-gbp/repository/archive.tar.gz?ref=release/indigo/actionlib/1.11.5-1\r\n    version: actionlib-gbp-release-indigo-actionlib-1.11.5-1\r\n$ rosinstall_generator actionlib --tar --rosdistro indigo --upstream-development\r\n- tar:\r\n    local-name: actionlib\r\n    uri: https://github.com/clearpathrobotics/actionlib/archive/indigo-devel.tar.gz\r\n    version: actionlib-indigo-devel\r\n$ rosinstall_generator cpr_navigation --tar --rosdistro indigo\r\n- tar:\r\n    local-name: cpr_navigation/cpr_navigation\r\n    uri: https://bitbucket.org/clearpathrobotics/cpr_navigation-gbp/get/release/indigo/cpr_navigation/2.0.6-0.tar.gz\r\n    version: cpr_navigation-gbp-release-indigo-cpr_navigation-2.0.6-0\r\n```\r\n\r\nTo confirm the failure path, I manually mutated a local copy of my indigo cache, and got:\r\n\r\n```\r\n$ rosinstall_generator actionlib --tar --rosdistro indigo > foo.txt\r\nTarball requested for repo actionlib, but git server gatlab.clearpathrobotics.com is unrecognized.\r\nFalling back on git clone for repo actionlib\r\n$ cat foo.txt\r\n- git:\r\n    local-name: actionlib\r\n    uri: git@gatlab.clearpathrobotics.com:gbp/actionlib-gbp.git\r\n    version: release/indigo/actionlib/1.11.5-1\r\n```\r\n\r\nOnce https://github.com/ros-infrastructure/rosdistro/pull/79 eventually merges, this could be refactored to take advantage of `RepositorySpecification.VCS_REGEX`, but it's only a small savings, so I think I'd rather get this in now than wait for that.","created_at":"2016-07-25T19:45:00Z","updated_at":"2016-08-16T01:00:28Z","closed_at":null,"merged_at":null,"merge_commit_sha":"5fc76de171ee2347a63dd9f28d61f1eeba77fabd","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/ros-infrastructure/rosinstall_generator/pulls/37/commits","review_comments_url":"https://api.github.com/repos/ros-infrastructure/rosinstall_generator/pulls/37/comments","review_comment_url":"https://api.github.com/repos/ros-infrastructure/rosinstall_generator/pulls/comments{/number}","comments_url":"https://api.github.com/repos/ros-infrastructure/rosinstall_generator/issues/37/comments","statuses_url":"https://api.github.com/repos/ros-infrastructure/rosinstall_generator/statuses/6cc76fc2d6cfa74c65b4234e06421e1e43598519","head":{"label":"mikepurvis:support-bitbucket-gitlab","ref":"support-bitbucket-gitlab","sha":"6cc76fc2d6cfa74c65b4234e06421e1e43598519","user":{"login":"mikepurvis","id":944666,"avatar_url":"https://avatars.githubusercontent.com/u/944666?v=3","gravatar_id":"","url":"https://api.github.com/users/mikepurvis","html_url":"https://github.com/mikepurvis","followers_url":"https://api.github.com/users/mikepurvis/followers","following_url":"https://api.github.com/users/mikepurvis/following{/other_user}","gists_url":"https://api.github.com/users/mikepurvis/gists{/gist_id}","starred_url":"https://api.github.com/users/mikepurvis/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mikepurvis/subscriptions","organizations_url":"https://api.github.com/users/mikepurvis/orgs","repos_url":"https://api.github.com/users/mikepurvis/repos","events_url":"https://api.github.com/users/mikepurvis/events{/privacy}","received_events_url":"https://api.github.com/users/mikepurvis/received_events","type":"User","site_admin":false},"repo":{"id":48804925,"name":"rosinstall_generator","full_name":"mikepurvis/rosinstall_generator","owner":{"login":"mikepurvis","id":944666,"avatar_url":"https://avatars.githubusercontent.com/u/944666?v=3","gravatar_id":"","url":"https://api.github.com/users/mikepurvis","html_url":"https://github.com/mikepurvis","followers_url":"https://api.github.com/users/mikepurvis/followers","following_url":"https://api.github.com/users/mikepurvis/following{/other_user}","gists_url":"https://api.github.com/users/mikepurvis/gists{/gist_id}","starred_url":"https://api.github.com/users/mikepurvis/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mikepurvis/subscriptions","organizations_url":"https://api.github.com/users/mikepurvis/orgs","repos_url":"https://api.github.com/users/mikepurvis/repos","events_url":"https://api.github.com/users/mikepurvis/events{/privacy}","received_events_url":"https://api.github.com/users/mikepurvis/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/mikepurvis/rosinstall_generator","description":"","fork":true,"url":"https://api.github.com/repos/mikepurvis/rosinstall_generator","forks_url":"https://api.github.com/repos/mikepurvis/rosinstall_generator/forks","keys_url":"https://api.github.com/repos/mikepurvis/rosinstall_generator/keys{/key_id}","collaborators_url":"https://api.github.com/repos/mikepurvis/rosinstall_generator/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/mikepurvis/rosinstall_generator/teams","hooks_url":"https://api.github.com/repos/mikepurvis/rosinstall_generator/hooks","issue_events_url":"https://api.github.com/repos/mikepurvis/rosinstall_generator/issues/events{/number}","events_url":"https://api.github.com/repos/mikepurvis/rosinstall_generator/events","assignees_url":"https://api.github.com/repos/mikepurvis/rosinstall_generator/assignees{/user}","branches_url":"https://api.github.com/repos/mikepurvis/rosinstall_generator/branches{/branch}","tags_url":"https://api.github.com/repos/mikepurvis/rosinstall_generator/tags","blobs_url":"https://api.github.com/repos/mikepurvis/rosinstall_generator/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/mikepurvis/rosinstall_generator/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/mikepurvis/rosinstall_generator/git/refs{/sha}","trees_url":"https://api.github.com/repos/mikepurvis/rosinstall_generator/git/trees{/sha}","statuses_url":"https://api.github.com/repos/mikepurvis/rosinstall_generator/statuses/{sha}","languages_url":"https://api.github.com/repos/mikepurvis/rosinstall_generator/languages","stargazers_url":"https://api.github.com/repos/mikepurvis/rosinstall_generator/stargazers","contributors_url":"https://api.github.com/repos/mikepurvis/rosinstall_generator/contributors","subscribers_url":"https://api.github.com/repos/mikepurvis/rosinstall_generator/subscribers","subscription_url":"https://api.github.com/repos/mikepurvis/rosinstall_generator/subscription","commits_url":"https://api.github.com/repos/mikepurvis/rosinstall_generator/commits{/sha}","git_commits_url":"https://api.github.com/repos/mikepurvis/rosinstall_generator/git/commits{/sha}","comments_url":"https://api.github.com/repos/mikepurvis/rosinstall_generator/comments{/number}","issue_comment_url":"https://api.github.com/repos/mikepurvis/rosinstall_generator/issues/comments{/number}","contents_url":"https://api.github.com/repos/mikepurvis/rosinstall_generator/contents/{+path}","compare_url":"https://api.github.com/repos/mikepurvis/rosinstall_generator/compare/{base}...{head}","merges_url":"https://api.github.com/repos/mikepurvis/rosinstall_generator/merges","archive_url":"https://api.github.com/repos/mikepurvis/rosinstall_generator/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/mikepurvis/rosinstall_generator/downloads","issues_url":"https://api.github.com/repos/mikepurvis/rosinstall_generator/issues{/number}","pulls_url":"https://api.github.com/repos/mikepurvis/rosinstall_generator/pulls{/number}","milestones_url":"https://api.github.com/repos/mikepurvis/rosinstall_generator/milestones{/number}","notifications_url":"https://api.github.com/repos/mikepurvis/rosinstall_generator/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/mikepurvis/rosinstall_generator/labels{/name}","releases_url":"https://api.github.com/repos/mikepurvis/rosinstall_generator/releases{/id}","deployments_url":"https://api.github.com/repos/mikepurvis/rosinstall_generator/deployments","created_at":"2015-12-30T14:45:10Z","updated_at":"2015-12-30T14:45:12Z","pushed_at":"2016-08-03T02:53:23Z","git_url":"git://github.com/mikepurvis/rosinstall_generator.git","ssh_url":"git@github.com:mikepurvis/rosinstall_generator.git","clone_url":"https://github.com/mikepurvis/rosinstall_generator.git","svn_url":"https://github.com/mikepurvis/rosinstall_generator","homepage":null,"size":79,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"ros-infrastructure:master","ref":"master","sha":"8dce72299628b479891c6bfba6a016aeef899a98","user":{"login":"ros-infrastructure","id":2328632,"avatar_url":"https://avatars.githubusercontent.com/u/2328632?v=3","gravatar_id":"","url":"https://api.github.com/users/ros-infrastructure","html_url":"https://github.com/ros-infrastructure","followers_url":"https://api.github.com/users/ros-infrastructure/followers","following_url":"https://api.github.com/users/ros-infrastructure/following{/other_user}","gists_url":"https://api.github.com/users/ros-infrastructure/gists{/gist_id}","starred_url":"https://api.github.com/users/ros-infrastructure/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ros-infrastructure/subscriptions","organizations_url":"https://api.github.com/users/ros-infrastructure/orgs","repos_url":"https://api.github.com/users/ros-infrastructure/repos","events_url":"https://api.github.com/users/ros-infrastructure/events{/privacy}","received_events_url":"https://api.github.com/users/ros-infrastructure/received_events","type":"Organization","site_admin":false},"repo":{"id":10672706,"name":"rosinstall_generator","full_name":"ros-infrastructure/rosinstall_generator","owner":{"login":"ros-infrastructure","id":2328632,"avatar_url":"https://avatars.githubusercontent.com/u/2328632?v=3","gravatar_id":"","url":"https://api.github.com/users/ros-infrastructure","html_url":"https://github.com/ros-infrastructure","followers_url":"https://api.github.com/users/ros-infrastructure/followers","following_url":"https://api.github.com/users/ros-infrastructure/following{/other_user}","gists_url":"https://api.github.com/users/ros-infrastructure/gists{/gist_id}","starred_url":"https://api.github.com/users/ros-infrastructure/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ros-infrastructure/subscriptions","organizations_url":"https://api.github.com/users/ros-infrastructure/orgs","repos_url":"https://api.github.com/users/ros-infrastructure/repos","events_url":"https://api.github.com/users/ros-infrastructure/events{/privacy}","received_events_url":"https://api.github.com/users/ros-infrastructure/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/ros-infrastructure/rosinstall_generator","description":"","fork":false,"url":"https://api.github.com/repos/ros-infrastructure/rosinstall_generator","forks_url":"https://api.github.com/repos/ros-infrastructure/rosinstall_generator/forks","keys_url":"https://api.github.com/repos/ros-infrastructure/rosinstall_generator/keys{/key_id}","collaborators_url":"https://api.github.com/repos/ros-infrastructure/rosinstall_generator/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/ros-infrastructure/rosinstall_generator/teams","hooks_url":"https://api.github.com/repos/ros-infrastructure/rosinstall_generator/hooks","issue_events_url":"https://api.github.com/repos/ros-infrastructure/rosinstall_generator/issues/events{/number}","events_url":"https://api.github.com/repos/ros-infrastructure/rosinstall_generator/events","assignees_url":"https://api.github.com/repos/ros-infrastructure/rosinstall_generator/assignees{/user}","branches_url":"https://api.github.com/repos/ros-infrastructure/rosinstall_generator/branches{/branch}","tags_url":"https://api.github.com/repos/ros-infrastructure/rosinstall_generator/tags","blobs_url":"https://api.github.com/repos/ros-infrastructure/rosinstall_generator/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/ros-infrastructure/rosinstall_generator/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/ros-infrastructure/rosinstall_generator/git/refs{/sha}","trees_url":"https://api.github.com/repos/ros-infrastructure/rosinstall_generator/git/trees{/sha}","statuses_url":"https://api.github.com/repos/ros-infrastructure/rosinstall_generator/statuses/{sha}","languages_url":"https://api.github.com/repos/ros-infrastructure/rosinstall_generator/languages","stargazers_url":"https://api.github.com/repos/ros-infrastructure/rosinstall_generator/stargazers","contributors_url":"https://api.github.com/repos/ros-infrastructure/rosinstall_generator/contributors","subscribers_url":"https://api.github.com/repos/ros-infrastructure/rosinstall_generator/subscribers","subscription_url":"https://api.github.com/repos/ros-infrastructure/rosinstall_generator/subscription","commits_url":"https://api.github.com/repos/ros-infrastructure/rosinstall_generator/commits{/sha}","git_commits_url":"https://api.github.com/repos/ros-infrastructure/rosinstall_generator/git/commits{/sha}","comments_url":"https://api.github.com/repos/ros-infrastructure/rosinstall_generator/comments{/number}","issue_comment_url":"https://api.github.com/repos/ros-infrastructure/rosinstall_generator/issues/comments{/number}","contents_url":"https://api.github.com/repos/ros-infrastructure/rosinstall_generator/contents/{+path}","compare_url":"https://api.github.com/repos/ros-infrastructure/rosinstall_generator/compare/{base}...{head}","merges_url":"https://api.github.com/repos/ros-infrastructure/rosinstall_generator/merges","archive_url":"https://api.github.com/repos/ros-infrastructure/rosinstall_generator/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/ros-infrastructure/rosinstall_generator/downloads","issues_url":"https://api.github.com/repos/ros-infrastructure/rosinstall_generator/issues{/number}","pulls_url":"https://api.github.com/repos/ros-infrastructure/rosinstall_generator/pulls{/number}","milestones_url":"https://api.github.com/repos/ros-infrastructure/rosinstall_generator/milestones{/number}","notifications_url":"https://api.github.com/repos/ros-infrastructure/rosinstall_generator/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/ros-infrastructure/rosinstall_generator/labels{/name}","releases_url":"https://api.github.com/repos/ros-infrastructure/rosinstall_generator/releases{/id}","deployments_url":"https://api.github.com/repos/ros-infrastructure/rosinstall_generator/deployments","created_at":"2013-06-13T18:22:27Z","updated_at":"2016-04-25T15:46:00Z","pushed_at":"2016-08-03T02:53:23Z","git_url":"git://github.com/ros-infrastructure/rosinstall_generator.git","ssh_url":"git@github.com:ros-infrastructure/rosinstall_generator.git","clone_url":"https://github.com/ros-infrastructure/rosinstall_generator.git","svn_url":"https://github.com/ros-infrastructure/rosinstall_generator","homepage":null,"size":120,"stargazers_count":3,"watchers_count":3,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":4,"mirror_url":null,"open_issues_count":7,"forks":4,"open_issues":7,"watchers":3,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/ros-infrastructure/rosinstall_generator/pulls/37"},"html":{"href":"https://github.com/ros-infrastructure/rosinstall_generator/pull/37"},"issue":{"href":"https://api.github.com/repos/ros-infrastructure/rosinstall_generator/issues/37"},"comments":{"href":"https://api.github.com/repos/ros-infrastructure/rosinstall_generator/issues/37/comments"},"review_comments":{"href":"https://api.github.com/repos/ros-infrastructure/rosinstall_generator/pulls/37/comments"},"review_comment":{"href":"https://api.github.com/repos/ros-infrastructure/rosinstall_generator/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/ros-infrastructure/rosinstall_generator/pulls/37/commits"},"statuses":{"href":"https://api.github.com/repos/ros-infrastructure/rosinstall_generator/statuses/6cc76fc2d6cfa74c65b4234e06421e1e43598519"}}}}
{"actor":{"display_login":"mikepurvis"}}