{"action":"created","comment":{"url":"https://api.github.com/repos/scrapy/scrapy/pulls/comments/88214511","pull_request_review_id":8793208,"id":88214511,"diff_hunk":"@@ -34,10 +37,13 @@ def __iter__(self):\n                 yield d\n \n \n-def sitemap_urls_from_robots(robots_text):\n+def sitemap_urls_from_robots(robots_text, base_url=None):\n     \"\"\"Return an iterator over all sitemap urls contained in the given\n     robots.txt file\n     \"\"\"\n     for line in robots_text.splitlines():\n         if line.lstrip().lower().startswith('sitemap:'):\n-            yield line.split(':', 1)[1].strip()\n+            url = line.split(':', 1)[1].strip()\n+            if not re.match('https?://', url):","path":"scrapy/utils/sitemap.py","position":24,"original_position":24,"commit_id":"9e7694c039c26a54d5608f0934e59c57dfcd87f5","original_commit_id":"9e7694c039c26a54d5608f0934e59c57dfcd87f5","user":{"login":"redapple","id":886296,"avatar_url":"https://avatars.githubusercontent.com/u/886296?v=3","gravatar_id":"","url":"https://api.github.com/users/redapple","html_url":"https://github.com/redapple","followers_url":"https://api.github.com/users/redapple/followers","following_url":"https://api.github.com/users/redapple/following{/other_user}","gists_url":"https://api.github.com/users/redapple/gists{/gist_id}","starred_url":"https://api.github.com/users/redapple/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/redapple/subscriptions","organizations_url":"https://api.github.com/users/redapple/orgs","repos_url":"https://api.github.com/users/redapple/repos","events_url":"https://api.github.com/users/redapple/events{/privacy}","received_events_url":"https://api.github.com/users/redapple/received_events","type":"User","site_admin":false},"body":"One may want to use case-insensitive regex here (it should not happen too often though and urljoin handles it also)","created_at":"2016-11-16T11:04:30Z","updated_at":"2016-11-16T11:04:30Z","html_url":"https://github.com/scrapy/scrapy/pull/2395#discussion_r88214511","pull_request_url":"https://api.github.com/repos/scrapy/scrapy/pulls/2395","_links":{"self":{"href":"https://api.github.com/repos/scrapy/scrapy/pulls/comments/88214511"},"html":{"href":"https://github.com/scrapy/scrapy/pull/2395#discussion_r88214511"},"pull_request":{"href":"https://api.github.com/repos/scrapy/scrapy/pulls/2395"}}},"pull_request":{"url":"https://api.github.com/repos/scrapy/scrapy/pulls/2395","id":93780230,"html_url":"https://github.com/scrapy/scrapy/pull/2395","diff_url":"https://github.com/scrapy/scrapy/pull/2395.diff","patch_url":"https://github.com/scrapy/scrapy/pull/2395.patch","issue_url":"https://api.github.com/repos/scrapy/scrapy/issues/2395","number":2395,"state":"open","locked":false,"title":"[MRG+1] Handle relative sitemap URLs in robots.txt","user":{"login":"elacuesta","id":1731933,"avatar_url":"https://avatars.githubusercontent.com/u/1731933?v=3","gravatar_id":"","url":"https://api.github.com/users/elacuesta","html_url":"https://github.com/elacuesta","followers_url":"https://api.github.com/users/elacuesta/followers","following_url":"https://api.github.com/users/elacuesta/following{/other_user}","gists_url":"https://api.github.com/users/elacuesta/gists{/gist_id}","starred_url":"https://api.github.com/users/elacuesta/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/elacuesta/subscriptions","organizations_url":"https://api.github.com/users/elacuesta/orgs","repos_url":"https://api.github.com/users/elacuesta/repos","events_url":"https://api.github.com/users/elacuesta/events{/privacy}","received_events_url":"https://api.github.com/users/elacuesta/received_events","type":"User","site_admin":false},"body":"Fixes #2390","created_at":"2016-11-15T14:55:38Z","updated_at":"2016-11-16T11:04:30Z","closed_at":null,"merged_at":null,"merge_commit_sha":"2f2d3e6c6ae70aaec1efea425ca56c2f2f25e5ce","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/scrapy/scrapy/pulls/2395/commits","review_comments_url":"https://api.github.com/repos/scrapy/scrapy/pulls/2395/comments","review_comment_url":"https://api.github.com/repos/scrapy/scrapy/pulls/comments{/number}","comments_url":"https://api.github.com/repos/scrapy/scrapy/issues/2395/comments","statuses_url":"https://api.github.com/repos/scrapy/scrapy/statuses/9e7694c039c26a54d5608f0934e59c57dfcd87f5","head":{"label":"elacuesta:robots_txt_relative_sitemap_url","ref":"robots_txt_relative_sitemap_url","sha":"9e7694c039c26a54d5608f0934e59c57dfcd87f5","user":{"login":"elacuesta","id":1731933,"avatar_url":"https://avatars.githubusercontent.com/u/1731933?v=3","gravatar_id":"","url":"https://api.github.com/users/elacuesta","html_url":"https://github.com/elacuesta","followers_url":"https://api.github.com/users/elacuesta/followers","following_url":"https://api.github.com/users/elacuesta/following{/other_user}","gists_url":"https://api.github.com/users/elacuesta/gists{/gist_id}","starred_url":"https://api.github.com/users/elacuesta/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/elacuesta/subscriptions","organizations_url":"https://api.github.com/users/elacuesta/orgs","repos_url":"https://api.github.com/users/elacuesta/repos","events_url":"https://api.github.com/users/elacuesta/events{/privacy}","received_events_url":"https://api.github.com/users/elacuesta/received_events","type":"User","site_admin":false},"repo":{"id":61134385,"name":"scrapy","full_name":"elacuesta/scrapy","owner":{"login":"elacuesta","id":1731933,"avatar_url":"https://avatars.githubusercontent.com/u/1731933?v=3","gravatar_id":"","url":"https://api.github.com/users/elacuesta","html_url":"https://github.com/elacuesta","followers_url":"https://api.github.com/users/elacuesta/followers","following_url":"https://api.github.com/users/elacuesta/following{/other_user}","gists_url":"https://api.github.com/users/elacuesta/gists{/gist_id}","starred_url":"https://api.github.com/users/elacuesta/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/elacuesta/subscriptions","organizations_url":"https://api.github.com/users/elacuesta/orgs","repos_url":"https://api.github.com/users/elacuesta/repos","events_url":"https://api.github.com/users/elacuesta/events{/privacy}","received_events_url":"https://api.github.com/users/elacuesta/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/elacuesta/scrapy","description":"Scrapy, a fast high-level web crawling & scraping framework for Python.","fork":true,"url":"https://api.github.com/repos/elacuesta/scrapy","forks_url":"https://api.github.com/repos/elacuesta/scrapy/forks","keys_url":"https://api.github.com/repos/elacuesta/scrapy/keys{/key_id}","collaborators_url":"https://api.github.com/repos/elacuesta/scrapy/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/elacuesta/scrapy/teams","hooks_url":"https://api.github.com/repos/elacuesta/scrapy/hooks","issue_events_url":"https://api.github.com/repos/elacuesta/scrapy/issues/events{/number}","events_url":"https://api.github.com/repos/elacuesta/scrapy/events","assignees_url":"https://api.github.com/repos/elacuesta/scrapy/assignees{/user}","branches_url":"https://api.github.com/repos/elacuesta/scrapy/branches{/branch}","tags_url":"https://api.github.com/repos/elacuesta/scrapy/tags","blobs_url":"https://api.github.com/repos/elacuesta/scrapy/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/elacuesta/scrapy/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/elacuesta/scrapy/git/refs{/sha}","trees_url":"https://api.github.com/repos/elacuesta/scrapy/git/trees{/sha}","statuses_url":"https://api.github.com/repos/elacuesta/scrapy/statuses/{sha}","languages_url":"https://api.github.com/repos/elacuesta/scrapy/languages","stargazers_url":"https://api.github.com/repos/elacuesta/scrapy/stargazers","contributors_url":"https://api.github.com/repos/elacuesta/scrapy/contributors","subscribers_url":"https://api.github.com/repos/elacuesta/scrapy/subscribers","subscription_url":"https://api.github.com/repos/elacuesta/scrapy/subscription","commits_url":"https://api.github.com/repos/elacuesta/scrapy/commits{/sha}","git_commits_url":"https://api.github.com/repos/elacuesta/scrapy/git/commits{/sha}","comments_url":"https://api.github.com/repos/elacuesta/scrapy/comments{/number}","issue_comment_url":"https://api.github.com/repos/elacuesta/scrapy/issues/comments{/number}","contents_url":"https://api.github.com/repos/elacuesta/scrapy/contents/{+path}","compare_url":"https://api.github.com/repos/elacuesta/scrapy/compare/{base}...{head}","merges_url":"https://api.github.com/repos/elacuesta/scrapy/merges","archive_url":"https://api.github.com/repos/elacuesta/scrapy/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/elacuesta/scrapy/downloads","issues_url":"https://api.github.com/repos/elacuesta/scrapy/issues{/number}","pulls_url":"https://api.github.com/repos/elacuesta/scrapy/pulls{/number}","milestones_url":"https://api.github.com/repos/elacuesta/scrapy/milestones{/number}","notifications_url":"https://api.github.com/repos/elacuesta/scrapy/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/elacuesta/scrapy/labels{/name}","releases_url":"https://api.github.com/repos/elacuesta/scrapy/releases{/id}","deployments_url":"https://api.github.com/repos/elacuesta/scrapy/deployments","created_at":"2016-06-14T15:30:23Z","updated_at":"2016-06-14T15:30:26Z","pushed_at":"2016-11-15T16:01:57Z","git_url":"git://github.com/elacuesta/scrapy.git","ssh_url":"git@github.com:elacuesta/scrapy.git","clone_url":"https://github.com/elacuesta/scrapy.git","svn_url":"https://github.com/elacuesta/scrapy","homepage":"http://scrapy.org","size":14992,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_downloads":false,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"scrapy:master","ref":"master","sha":"1cbc57faa5439515fb937a8adce64dce01498e2a","user":{"login":"scrapy","id":733635,"avatar_url":"https://avatars.githubusercontent.com/u/733635?v=3","gravatar_id":"","url":"https://api.github.com/users/scrapy","html_url":"https://github.com/scrapy","followers_url":"https://api.github.com/users/scrapy/followers","following_url":"https://api.github.com/users/scrapy/following{/other_user}","gists_url":"https://api.github.com/users/scrapy/gists{/gist_id}","starred_url":"https://api.github.com/users/scrapy/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/scrapy/subscriptions","organizations_url":"https://api.github.com/users/scrapy/orgs","repos_url":"https://api.github.com/users/scrapy/repos","events_url":"https://api.github.com/users/scrapy/events{/privacy}","received_events_url":"https://api.github.com/users/scrapy/received_events","type":"Organization","site_admin":false},"repo":{"id":529502,"name":"scrapy","full_name":"scrapy/scrapy","owner":{"login":"scrapy","id":733635,"avatar_url":"https://avatars.githubusercontent.com/u/733635?v=3","gravatar_id":"","url":"https://api.github.com/users/scrapy","html_url":"https://github.com/scrapy","followers_url":"https://api.github.com/users/scrapy/followers","following_url":"https://api.github.com/users/scrapy/following{/other_user}","gists_url":"https://api.github.com/users/scrapy/gists{/gist_id}","starred_url":"https://api.github.com/users/scrapy/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/scrapy/subscriptions","organizations_url":"https://api.github.com/users/scrapy/orgs","repos_url":"https://api.github.com/users/scrapy/repos","events_url":"https://api.github.com/users/scrapy/events{/privacy}","received_events_url":"https://api.github.com/users/scrapy/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/scrapy/scrapy","description":"Scrapy, a fast high-level web crawling & scraping framework for Python.","fork":false,"url":"https://api.github.com/repos/scrapy/scrapy","forks_url":"https://api.github.com/repos/scrapy/scrapy/forks","keys_url":"https://api.github.com/repos/scrapy/scrapy/keys{/key_id}","collaborators_url":"https://api.github.com/repos/scrapy/scrapy/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/scrapy/scrapy/teams","hooks_url":"https://api.github.com/repos/scrapy/scrapy/hooks","issue_events_url":"https://api.github.com/repos/scrapy/scrapy/issues/events{/number}","events_url":"https://api.github.com/repos/scrapy/scrapy/events","assignees_url":"https://api.github.com/repos/scrapy/scrapy/assignees{/user}","branches_url":"https://api.github.com/repos/scrapy/scrapy/branches{/branch}","tags_url":"https://api.github.com/repos/scrapy/scrapy/tags","blobs_url":"https://api.github.com/repos/scrapy/scrapy/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/scrapy/scrapy/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/scrapy/scrapy/git/refs{/sha}","trees_url":"https://api.github.com/repos/scrapy/scrapy/git/trees{/sha}","statuses_url":"https://api.github.com/repos/scrapy/scrapy/statuses/{sha}","languages_url":"https://api.github.com/repos/scrapy/scrapy/languages","stargazers_url":"https://api.github.com/repos/scrapy/scrapy/stargazers","contributors_url":"https://api.github.com/repos/scrapy/scrapy/contributors","subscribers_url":"https://api.github.com/repos/scrapy/scrapy/subscribers","subscription_url":"https://api.github.com/repos/scrapy/scrapy/subscription","commits_url":"https://api.github.com/repos/scrapy/scrapy/commits{/sha}","git_commits_url":"https://api.github.com/repos/scrapy/scrapy/git/commits{/sha}","comments_url":"https://api.github.com/repos/scrapy/scrapy/comments{/number}","issue_comment_url":"https://api.github.com/repos/scrapy/scrapy/issues/comments{/number}","contents_url":"https://api.github.com/repos/scrapy/scrapy/contents/{+path}","compare_url":"https://api.github.com/repos/scrapy/scrapy/compare/{base}...{head}","merges_url":"https://api.github.com/repos/scrapy/scrapy/merges","archive_url":"https://api.github.com/repos/scrapy/scrapy/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/scrapy/scrapy/downloads","issues_url":"https://api.github.com/repos/scrapy/scrapy/issues{/number}","pulls_url":"https://api.github.com/repos/scrapy/scrapy/pulls{/number}","milestones_url":"https://api.github.com/repos/scrapy/scrapy/milestones{/number}","notifications_url":"https://api.github.com/repos/scrapy/scrapy/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/scrapy/scrapy/labels{/name}","releases_url":"https://api.github.com/repos/scrapy/scrapy/releases{/id}","deployments_url":"https://api.github.com/repos/scrapy/scrapy/deployments","created_at":"2010-02-22T02:01:14Z","updated_at":"2016-11-16T10:28:16Z","pushed_at":"2016-11-16T10:59:38Z","git_url":"git://github.com/scrapy/scrapy.git","ssh_url":"git@github.com:scrapy/scrapy.git","clone_url":"https://github.com/scrapy/scrapy.git","svn_url":"https://github.com/scrapy/scrapy","homepage":"http://scrapy.org","size":51923,"stargazers_count":16896,"watchers_count":16896,"language":"Python","has_issues":true,"has_downloads":false,"has_wiki":true,"has_pages":false,"forks_count":4662,"mirror_url":null,"open_issues_count":454,"forks":4662,"open_issues":454,"watchers":16896,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/scrapy/scrapy/pulls/2395"},"html":{"href":"https://github.com/scrapy/scrapy/pull/2395"},"issue":{"href":"https://api.github.com/repos/scrapy/scrapy/issues/2395"},"comments":{"href":"https://api.github.com/repos/scrapy/scrapy/issues/2395/comments"},"review_comments":{"href":"https://api.github.com/repos/scrapy/scrapy/pulls/2395/comments"},"review_comment":{"href":"https://api.github.com/repos/scrapy/scrapy/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/scrapy/scrapy/pulls/2395/commits"},"statuses":{"href":"https://api.github.com/repos/scrapy/scrapy/statuses/9e7694c039c26a54d5608f0934e59c57dfcd87f5"}}}}
{"actor":{"display_login":"redapple"}}