{"action":"created","comment":{"url":"https://api.github.com/repos/public-people/scrape-news/pulls/comments/202517953","pull_request_review_id":137241158,"id":202517953,"node_id":"MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDIwMjUxNzk1Mw==","diff_hunk":"@@ -0,0 +1,79 @@\n+# -*- coding: utf-8 -*-\n+\n+from scrapy.spiders import CrawlSpider, Rule\n+from scrapy.linkextractors import LinkExtractor\n+from scrapenews.items import ScrapenewsItem\n+from datetime import datetime\n+import pytz\n+\n+SAST = pytz.timezone('Africa/Johannesburg')\n+\n+\n+class sabcSpider(CrawlSpider):\n+    name = 'sabc'\n+    allowed_domains = ['sabcnews.com']\n+\n+    start_urls = ['http://www.sabcnews.com/sabcnews/']\n+\n+    link_extractor = LinkExtractor(\n+        allow=('http://www.sabcnews.com/sabcnews/', ),\n+        deny=(\n+            '/?post_type=',\n+            '/?p=',\n+            '/author/',\n+            '/feed/',\n+            '/rss-feeds/',\n+            '/tag/',\n+        )\n+    )\n+\n+    rules = (\n+        Rule(link_extractor, process_links='filter_links', callback='parse_item', follow=True),\n+    )\n+\n+    publication_name = 'SABC News'\n+\n+\n+    def parse_item(self, response):\n+\n+        canonical_url = response.xpath('//link[@rel=\"canonical\"]/@href').extract_first()\n+        title = response.xpath('//h1/text()').extract_first()\n+        self.logger.info('%s %s', response.url, title)\n+        # should we be using canonical_url instead of response.url for the above?\n+        og_type = response.xpath('//meta[@property=\"og:type\"]/@content').extract_first()\n+\n+        if og_type == 'article':\n+            article_body = response.css('div.post-content')\n+            body_html = \" \".join(article_body.css('::text').extract())\n+            byline = response.css('span.author::text').extract_first().strip()\n+            publication_date_str = response.css('span.create::text').extract_first().strip()\n+            publication_date = datetime.strptime(publication_date_str, '%d %B %Y, %I:%M %p')\n+            publication_date = SAST.localize(publication_date)\n+\n+            if body_html:\n+                item = ScrapenewsItem()\n+                item['body_html'] = body_html\n+                item['title'] = title.strip()\n+                item['byline'] = byline\n+                item['published_at'] = publication_date.isoformat()\n+                item['retrieved_at'] = datetime.utcnow().isoformat()\n+                item['url'] = canonical_url\n+                item['file_name'] = response.url.split('/')[-2]\n+                # should we be using canonical_url instead of response.url for the above?\n+                item['spider_name'] = self.name\n+                item['publication_name'] = self.publication_name\n+\n+                yield item\n+\n+            else:\n+                self.logger.info(\"No body found for %s\", response.url)\n+                # should we be using canonical_url instead of response.url for the above?","path":"scrapenews/spiders/sabc.py","position":70,"original_position":70,"commit_id":"721a28b861dfef83251f7f802f51994f245a50fc","original_commit_id":"721a28b861dfef83251f7f802f51994f245a50fc","user":{"login":"jbothma","id":235801,"node_id":"MDQ6VXNlcjIzNTgwMQ==","avatar_url":"https://avatars0.githubusercontent.com/u/235801?v=4","gravatar_id":"","url":"https://api.github.com/users/jbothma","html_url":"https://github.com/jbothma","followers_url":"https://api.github.com/users/jbothma/followers","following_url":"https://api.github.com/users/jbothma/following{/other_user}","gists_url":"https://api.github.com/users/jbothma/gists{/gist_id}","starred_url":"https://api.github.com/users/jbothma/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jbothma/subscriptions","organizations_url":"https://api.github.com/users/jbothma/orgs","repos_url":"https://api.github.com/users/jbothma/repos","events_url":"https://api.github.com/users/jbothma/events{/privacy}","received_events_url":"https://api.github.com/users/jbothma/received_events","type":"User","site_admin":false},"body":"yep","created_at":"2018-07-14T15:33:04Z","updated_at":"2018-07-14T15:33:05Z","html_url":"https://github.com/public-people/scrape-news/pull/10#discussion_r202517953","pull_request_url":"https://api.github.com/repos/public-people/scrape-news/pulls/10","author_association":"CONTRIBUTOR","_links":{"self":{"href":"https://api.github.com/repos/public-people/scrape-news/pulls/comments/202517953"},"html":{"href":"https://github.com/public-people/scrape-news/pull/10#discussion_r202517953"},"pull_request":{"href":"https://api.github.com/repos/public-people/scrape-news/pulls/10"}}},"pull_request":{"url":"https://api.github.com/repos/public-people/scrape-news/pulls/10","id":197419417,"node_id":"MDExOlB1bGxSZXF1ZXN0MTk3NDE5NDE3","html_url":"https://github.com/public-people/scrape-news/pull/10","diff_url":"https://github.com/public-people/scrape-news/pull/10.diff","patch_url":"https://github.com/public-people/scrape-news/pull/10.patch","issue_url":"https://api.github.com/repos/public-people/scrape-news/issues/10","number":10,"state":"open","locked":false,"title":"add sabc crawling spider","user":{"login":"goose-life","id":32566441,"node_id":"MDQ6VXNlcjMyNTY2NDQx","avatar_url":"https://avatars1.githubusercontent.com/u/32566441?v=4","gravatar_id":"","url":"https://api.github.com/users/goose-life","html_url":"https://github.com/goose-life","followers_url":"https://api.github.com/users/goose-life/followers","following_url":"https://api.github.com/users/goose-life/following{/other_user}","gists_url":"https://api.github.com/users/goose-life/gists{/gist_id}","starred_url":"https://api.github.com/users/goose-life/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/goose-life/subscriptions","organizations_url":"https://api.github.com/users/goose-life/orgs","repos_url":"https://api.github.com/users/goose-life/repos","events_url":"https://api.github.com/users/goose-life/events{/privacy}","received_events_url":"https://api.github.com/users/goose-life/received_events","type":"User","site_admin":false},"body":"","created_at":"2018-06-26T13:18:44Z","updated_at":"2018-07-14T15:33:05Z","closed_at":null,"merged_at":null,"merge_commit_sha":"029338d4e5ad399b1657c6b6f6ad6e71c37d3d9e","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/public-people/scrape-news/pulls/10/commits","review_comments_url":"https://api.github.com/repos/public-people/scrape-news/pulls/10/comments","review_comment_url":"https://api.github.com/repos/public-people/scrape-news/pulls/comments{/number}","comments_url":"https://api.github.com/repos/public-people/scrape-news/issues/10/comments","statuses_url":"https://api.github.com/repos/public-people/scrape-news/statuses/721a28b861dfef83251f7f802f51994f245a50fc","head":{"label":"public-people:sabc","ref":"sabc","sha":"721a28b861dfef83251f7f802f51994f245a50fc","user":{"login":"public-people","id":36810691,"node_id":"MDEyOk9yZ2FuaXphdGlvbjM2ODEwNjkx","avatar_url":"https://avatars2.githubusercontent.com/u/36810691?v=4","gravatar_id":"","url":"https://api.github.com/users/public-people","html_url":"https://github.com/public-people","followers_url":"https://api.github.com/users/public-people/followers","following_url":"https://api.github.com/users/public-people/following{/other_user}","gists_url":"https://api.github.com/users/public-people/gists{/gist_id}","starred_url":"https://api.github.com/users/public-people/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/public-people/subscriptions","organizations_url":"https://api.github.com/users/public-people/orgs","repos_url":"https://api.github.com/users/public-people/repos","events_url":"https://api.github.com/users/public-people/events{/privacy}","received_events_url":"https://api.github.com/users/public-people/received_events","type":"Organization","site_admin":false},"repo":{"id":123028777,"node_id":"MDEwOlJlcG9zaXRvcnkxMjMwMjg3Nzc=","name":"scrape-news","full_name":"public-people/scrape-news","owner":{"login":"public-people","id":36810691,"node_id":"MDEyOk9yZ2FuaXphdGlvbjM2ODEwNjkx","avatar_url":"https://avatars2.githubusercontent.com/u/36810691?v=4","gravatar_id":"","url":"https://api.github.com/users/public-people","html_url":"https://github.com/public-people","followers_url":"https://api.github.com/users/public-people/followers","following_url":"https://api.github.com/users/public-people/following{/other_user}","gists_url":"https://api.github.com/users/public-people/gists{/gist_id}","starred_url":"https://api.github.com/users/public-people/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/public-people/subscriptions","organizations_url":"https://api.github.com/users/public-people/orgs","repos_url":"https://api.github.com/users/public-people/repos","events_url":"https://api.github.com/users/public-people/events{/privacy}","received_events_url":"https://api.github.com/users/public-people/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/public-people/scrape-news","description":"Scrape South African news","fork":false,"url":"https://api.github.com/repos/public-people/scrape-news","forks_url":"https://api.github.com/repos/public-people/scrape-news/forks","keys_url":"https://api.github.com/repos/public-people/scrape-news/keys{/key_id}","collaborators_url":"https://api.github.com/repos/public-people/scrape-news/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/public-people/scrape-news/teams","hooks_url":"https://api.github.com/repos/public-people/scrape-news/hooks","issue_events_url":"https://api.github.com/repos/public-people/scrape-news/issues/events{/number}","events_url":"https://api.github.com/repos/public-people/scrape-news/events","assignees_url":"https://api.github.com/repos/public-people/scrape-news/assignees{/user}","branches_url":"https://api.github.com/repos/public-people/scrape-news/branches{/branch}","tags_url":"https://api.github.com/repos/public-people/scrape-news/tags","blobs_url":"https://api.github.com/repos/public-people/scrape-news/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/public-people/scrape-news/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/public-people/scrape-news/git/refs{/sha}","trees_url":"https://api.github.com/repos/public-people/scrape-news/git/trees{/sha}","statuses_url":"https://api.github.com/repos/public-people/scrape-news/statuses/{sha}","languages_url":"https://api.github.com/repos/public-people/scrape-news/languages","stargazers_url":"https://api.github.com/repos/public-people/scrape-news/stargazers","contributors_url":"https://api.github.com/repos/public-people/scrape-news/contributors","subscribers_url":"https://api.github.com/repos/public-people/scrape-news/subscribers","subscription_url":"https://api.github.com/repos/public-people/scrape-news/subscription","commits_url":"https://api.github.com/repos/public-people/scrape-news/commits{/sha}","git_commits_url":"https://api.github.com/repos/public-people/scrape-news/git/commits{/sha}","comments_url":"https://api.github.com/repos/public-people/scrape-news/comments{/number}","issue_comment_url":"https://api.github.com/repos/public-people/scrape-news/issues/comments{/number}","contents_url":"https://api.github.com/repos/public-people/scrape-news/contents/{+path}","compare_url":"https://api.github.com/repos/public-people/scrape-news/compare/{base}...{head}","merges_url":"https://api.github.com/repos/public-people/scrape-news/merges","archive_url":"https://api.github.com/repos/public-people/scrape-news/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/public-people/scrape-news/downloads","issues_url":"https://api.github.com/repos/public-people/scrape-news/issues{/number}","pulls_url":"https://api.github.com/repos/public-people/scrape-news/pulls{/number}","milestones_url":"https://api.github.com/repos/public-people/scrape-news/milestones{/number}","notifications_url":"https://api.github.com/repos/public-people/scrape-news/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/public-people/scrape-news/labels{/name}","releases_url":"https://api.github.com/repos/public-people/scrape-news/releases{/id}","deployments_url":"https://api.github.com/repos/public-people/scrape-news/deployments","created_at":"2018-02-26T20:50:29Z","updated_at":"2018-07-14T15:31:23Z","pushed_at":"2018-07-14T15:31:25Z","git_url":"git://github.com/public-people/scrape-news.git","ssh_url":"git@github.com:public-people/scrape-news.git","clone_url":"https://github.com/public-people/scrape-news.git","svn_url":"https://github.com/public-people/scrape-news","homepage":null,"size":61,"stargazers_count":2,"watchers_count":2,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":4,"mirror_url":null,"archived":false,"open_issues_count":4,"license":{"key":"mit","name":"MIT License","spdx_id":"MIT","url":"https://api.github.com/licenses/mit","node_id":"MDc6TGljZW5zZTEz"},"forks":4,"open_issues":4,"watchers":2,"default_branch":"master"}},"base":{"label":"public-people:master","ref":"master","sha":"b94507059331f4e3b9417389b986b8ea6417ec95","user":{"login":"public-people","id":36810691,"node_id":"MDEyOk9yZ2FuaXphdGlvbjM2ODEwNjkx","avatar_url":"https://avatars2.githubusercontent.com/u/36810691?v=4","gravatar_id":"","url":"https://api.github.com/users/public-people","html_url":"https://github.com/public-people","followers_url":"https://api.github.com/users/public-people/followers","following_url":"https://api.github.com/users/public-people/following{/other_user}","gists_url":"https://api.github.com/users/public-people/gists{/gist_id}","starred_url":"https://api.github.com/users/public-people/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/public-people/subscriptions","organizations_url":"https://api.github.com/users/public-people/orgs","repos_url":"https://api.github.com/users/public-people/repos","events_url":"https://api.github.com/users/public-people/events{/privacy}","received_events_url":"https://api.github.com/users/public-people/received_events","type":"Organization","site_admin":false},"repo":{"id":123028777,"node_id":"MDEwOlJlcG9zaXRvcnkxMjMwMjg3Nzc=","name":"scrape-news","full_name":"public-people/scrape-news","owner":{"login":"public-people","id":36810691,"node_id":"MDEyOk9yZ2FuaXphdGlvbjM2ODEwNjkx","avatar_url":"https://avatars2.githubusercontent.com/u/36810691?v=4","gravatar_id":"","url":"https://api.github.com/users/public-people","html_url":"https://github.com/public-people","followers_url":"https://api.github.com/users/public-people/followers","following_url":"https://api.github.com/users/public-people/following{/other_user}","gists_url":"https://api.github.com/users/public-people/gists{/gist_id}","starred_url":"https://api.github.com/users/public-people/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/public-people/subscriptions","organizations_url":"https://api.github.com/users/public-people/orgs","repos_url":"https://api.github.com/users/public-people/repos","events_url":"https://api.github.com/users/public-people/events{/privacy}","received_events_url":"https://api.github.com/users/public-people/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/public-people/scrape-news","description":"Scrape South African news","fork":false,"url":"https://api.github.com/repos/public-people/scrape-news","forks_url":"https://api.github.com/repos/public-people/scrape-news/forks","keys_url":"https://api.github.com/repos/public-people/scrape-news/keys{/key_id}","collaborators_url":"https://api.github.com/repos/public-people/scrape-news/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/public-people/scrape-news/teams","hooks_url":"https://api.github.com/repos/public-people/scrape-news/hooks","issue_events_url":"https://api.github.com/repos/public-people/scrape-news/issues/events{/number}","events_url":"https://api.github.com/repos/public-people/scrape-news/events","assignees_url":"https://api.github.com/repos/public-people/scrape-news/assignees{/user}","branches_url":"https://api.github.com/repos/public-people/scrape-news/branches{/branch}","tags_url":"https://api.github.com/repos/public-people/scrape-news/tags","blobs_url":"https://api.github.com/repos/public-people/scrape-news/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/public-people/scrape-news/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/public-people/scrape-news/git/refs{/sha}","trees_url":"https://api.github.com/repos/public-people/scrape-news/git/trees{/sha}","statuses_url":"https://api.github.com/repos/public-people/scrape-news/statuses/{sha}","languages_url":"https://api.github.com/repos/public-people/scrape-news/languages","stargazers_url":"https://api.github.com/repos/public-people/scrape-news/stargazers","contributors_url":"https://api.github.com/repos/public-people/scrape-news/contributors","subscribers_url":"https://api.github.com/repos/public-people/scrape-news/subscribers","subscription_url":"https://api.github.com/repos/public-people/scrape-news/subscription","commits_url":"https://api.github.com/repos/public-people/scrape-news/commits{/sha}","git_commits_url":"https://api.github.com/repos/public-people/scrape-news/git/commits{/sha}","comments_url":"https://api.github.com/repos/public-people/scrape-news/comments{/number}","issue_comment_url":"https://api.github.com/repos/public-people/scrape-news/issues/comments{/number}","contents_url":"https://api.github.com/repos/public-people/scrape-news/contents/{+path}","compare_url":"https://api.github.com/repos/public-people/scrape-news/compare/{base}...{head}","merges_url":"https://api.github.com/repos/public-people/scrape-news/merges","archive_url":"https://api.github.com/repos/public-people/scrape-news/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/public-people/scrape-news/downloads","issues_url":"https://api.github.com/repos/public-people/scrape-news/issues{/number}","pulls_url":"https://api.github.com/repos/public-people/scrape-news/pulls{/number}","milestones_url":"https://api.github.com/repos/public-people/scrape-news/milestones{/number}","notifications_url":"https://api.github.com/repos/public-people/scrape-news/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/public-people/scrape-news/labels{/name}","releases_url":"https://api.github.com/repos/public-people/scrape-news/releases{/id}","deployments_url":"https://api.github.com/repos/public-people/scrape-news/deployments","created_at":"2018-02-26T20:50:29Z","updated_at":"2018-07-14T15:31:23Z","pushed_at":"2018-07-14T15:31:25Z","git_url":"git://github.com/public-people/scrape-news.git","ssh_url":"git@github.com:public-people/scrape-news.git","clone_url":"https://github.com/public-people/scrape-news.git","svn_url":"https://github.com/public-people/scrape-news","homepage":null,"size":61,"stargazers_count":2,"watchers_count":2,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":4,"mirror_url":null,"archived":false,"open_issues_count":4,"license":{"key":"mit","name":"MIT License","spdx_id":"MIT","url":"https://api.github.com/licenses/mit","node_id":"MDc6TGljZW5zZTEz"},"forks":4,"open_issues":4,"watchers":2,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/public-people/scrape-news/pulls/10"},"html":{"href":"https://github.com/public-people/scrape-news/pull/10"},"issue":{"href":"https://api.github.com/repos/public-people/scrape-news/issues/10"},"comments":{"href":"https://api.github.com/repos/public-people/scrape-news/issues/10/comments"},"review_comments":{"href":"https://api.github.com/repos/public-people/scrape-news/pulls/10/comments"},"review_comment":{"href":"https://api.github.com/repos/public-people/scrape-news/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/public-people/scrape-news/pulls/10/commits"},"statuses":{"href":"https://api.github.com/repos/public-people/scrape-news/statuses/721a28b861dfef83251f7f802f51994f245a50fc"}},"author_association":"COLLABORATOR"}}
{"actor":{"display_login":"jbothma"}}