{"action":"created","comment":{"url":"https://api.github.com/repos/okfn-brasil/diario-oficial/pulls/comments/191091593","pull_request_review_id":123599120,"id":191091593,"diff_hunk":"@@ -0,0 +1,71 @@\n+import datetime as dt\n+\n+import scrapy\n+from scrapy.http import FormRequest\n+\n+from gazette.items import Gazette\n+\n+\n+class PrFozDoIguacuSpider(scrapy.Spider):\n+    MUNICIPALITY_ID = '4108304'\n+    name = 'pr_foz_do_iguacu'\n+    allowed_domains = ['pmfi.pr.gov.br']\n+    start_urls = ['http://www.pmfi.pr.gov.br/utilidades/diario/index.xhtml']\n+    AVAILABLE_FROM = dt.date(2015, 1, 1)  # actually September/2007\n+\n+    def parse(self, response):\n+        \"\"\"\n+        @url http://www.pmfi.pr.gov.br/utilidades/diario/index.xhtml\n+        @returns requests 1 1\n+        \"\"\"\n+        selector = '(//span[@class=\"ui-paginator-current\"])[1]/text()'\n+        paginator_text = response.xpath(selector).extract_first()\n+        quantity_of_documents = int(paginator_text[1:-1].split()[-1]) + 1\n+\n+        data = {\n+            'formDiarioOficialView': 'formDiarioOficialView',\n+            'formDiarioOficialView:dtdiario': 'formDiarioOficialView:dtdiario',\n+            'formDiarioOficialView:dtdiario_encodeFeature': 'true',\n+            'formDiarioOficialView:dtdiario_first': '0',\n+            'formDiarioOficialView:dtdiario_pagination': 'false',\n+            'formDiarioOficialView:dtdiario_rows': str(quantity_of_documents),\n+        }\n+        return FormRequest(response.url, formdata=data, callback=self.parse_items)\n+\n+    def parse_items(self, response):\n+        \"\"\"\n+        @url http://www.pmfi.pr.gov.br/utilidades/diario/index.xhtml\n+        @returns items 10 10\n+        \"\"\"\n+        base_url = 'http://www.pmfi.pr.gov.br{}'\n+        lines = response.xpath('//tr[@role=\"row\"]')\n+        items = []\n+        for line in lines:\n+            date, url, is_extra_edition = self.get_gazette_data(line)\n+\n+            if date >= self.AVAILABLE_FROM:\n+                items.append(","path":"processing/data_collection/gazette/spiders/pr_foz_do_iguacu.py","position":47,"original_position":47,"commit_id":"4692320d7ba4528d27ae3978244bba5ccaf6761b","original_commit_id":"4692320d7ba4528d27ae3978244bba5ccaf6761b","user":{"login":"rennerocha","id":382887,"avatar_url":"https://avatars3.githubusercontent.com/u/382887?v=4","gravatar_id":"","url":"https://api.github.com/users/rennerocha","html_url":"https://github.com/rennerocha","followers_url":"https://api.github.com/users/rennerocha/followers","following_url":"https://api.github.com/users/rennerocha/following{/other_user}","gists_url":"https://api.github.com/users/rennerocha/gists{/gist_id}","starred_url":"https://api.github.com/users/rennerocha/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/rennerocha/subscriptions","organizations_url":"https://api.github.com/users/rennerocha/orgs","repos_url":"https://api.github.com/users/rennerocha/repos","events_url":"https://api.github.com/users/rennerocha/events{/privacy}","received_events_url":"https://api.github.com/users/rennerocha/received_events","type":"User","site_admin":false},"body":"There is non need to include each item in a list and then return the whole list. You can simply yield a `Gazette` as soon you scrape it. Doing that, this item will be downloaded even if the other gazettes are not scraped yet as the item will join the pipelines.","created_at":"2018-05-27T23:43:41Z","updated_at":"2018-05-27T23:43:56Z","html_url":"https://github.com/okfn-brasil/diario-oficial/pull/27#discussion_r191091593","pull_request_url":"https://api.github.com/repos/okfn-brasil/diario-oficial/pulls/27","author_association":"CONTRIBUTOR","_links":{"self":{"href":"https://api.github.com/repos/okfn-brasil/diario-oficial/pulls/comments/191091593"},"html":{"href":"https://github.com/okfn-brasil/diario-oficial/pull/27#discussion_r191091593"},"pull_request":{"href":"https://api.github.com/repos/okfn-brasil/diario-oficial/pulls/27"}}},"pull_request":{"url":"https://api.github.com/repos/okfn-brasil/diario-oficial/pulls/27","id":184197789,"html_url":"https://github.com/okfn-brasil/diario-oficial/pull/27","diff_url":"https://github.com/okfn-brasil/diario-oficial/pull/27.diff","patch_url":"https://github.com/okfn-brasil/diario-oficial/pull/27.patch","issue_url":"https://api.github.com/repos/okfn-brasil/diario-oficial/issues/27","number":27,"state":"open","locked":false,"title":"Spider for Foz do Iguaçu/PR","user":{"login":"giovanisleite","id":11703235,"avatar_url":"https://avatars2.githubusercontent.com/u/11703235?v=4","gravatar_id":"","url":"https://api.github.com/users/giovanisleite","html_url":"https://github.com/giovanisleite","followers_url":"https://api.github.com/users/giovanisleite/followers","following_url":"https://api.github.com/users/giovanisleite/following{/other_user}","gists_url":"https://api.github.com/users/giovanisleite/gists{/gist_id}","starred_url":"https://api.github.com/users/giovanisleite/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/giovanisleite/subscriptions","organizations_url":"https://api.github.com/users/giovanisleite/orgs","repos_url":"https://api.github.com/users/giovanisleite/repos","events_url":"https://api.github.com/users/giovanisleite/events{/privacy}","received_events_url":"https://api.github.com/users/giovanisleite/received_events","type":"User","site_admin":false},"body":"Foz do Iguaçu is the 97th largest GDP, but 107th largest population (sorry, I wasn't sure about the criterion)\r\n\r\n- [Proof of legislature power in these documents](http://www.pmfi.pr.gov.br/ArquivosDB;jsessionid=9be543a88bd7e52927e641fe690e?idMidia=104275)\r\n- [Special Edition Example](http://www.pmfi.pr.gov.br/ArquivosDB?idMidia=61569)\r\n\r\n- [x]  Filter the documents by period of interest #23 ","created_at":"2018-04-26T00:34:54Z","updated_at":"2018-05-27T23:43:56Z","closed_at":null,"merged_at":null,"merge_commit_sha":"627de1bdd17a2506ab58ba7c3a16228796c2c1c8","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/okfn-brasil/diario-oficial/pulls/27/commits","review_comments_url":"https://api.github.com/repos/okfn-brasil/diario-oficial/pulls/27/comments","review_comment_url":"https://api.github.com/repos/okfn-brasil/diario-oficial/pulls/comments{/number}","comments_url":"https://api.github.com/repos/okfn-brasil/diario-oficial/issues/27/comments","statuses_url":"https://api.github.com/repos/okfn-brasil/diario-oficial/statuses/4692320d7ba4528d27ae3978244bba5ccaf6761b","head":{"label":"giovanisleite:giovani-foz-do-iguacu","ref":"giovani-foz-do-iguacu","sha":"4692320d7ba4528d27ae3978244bba5ccaf6761b","user":{"login":"giovanisleite","id":11703235,"avatar_url":"https://avatars2.githubusercontent.com/u/11703235?v=4","gravatar_id":"","url":"https://api.github.com/users/giovanisleite","html_url":"https://github.com/giovanisleite","followers_url":"https://api.github.com/users/giovanisleite/followers","following_url":"https://api.github.com/users/giovanisleite/following{/other_user}","gists_url":"https://api.github.com/users/giovanisleite/gists{/gist_id}","starred_url":"https://api.github.com/users/giovanisleite/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/giovanisleite/subscriptions","organizations_url":"https://api.github.com/users/giovanisleite/orgs","repos_url":"https://api.github.com/users/giovanisleite/repos","events_url":"https://api.github.com/users/giovanisleite/events{/privacy}","received_events_url":"https://api.github.com/users/giovanisleite/received_events","type":"User","site_admin":false},"repo":{"id":129796666,"name":"diario-oficial","full_name":"giovanisleite/diario-oficial","owner":{"login":"giovanisleite","id":11703235,"avatar_url":"https://avatars2.githubusercontent.com/u/11703235?v=4","gravatar_id":"","url":"https://api.github.com/users/giovanisleite","html_url":"https://github.com/giovanisleite","followers_url":"https://api.github.com/users/giovanisleite/followers","following_url":"https://api.github.com/users/giovanisleite/following{/other_user}","gists_url":"https://api.github.com/users/giovanisleite/gists{/gist_id}","starred_url":"https://api.github.com/users/giovanisleite/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/giovanisleite/subscriptions","organizations_url":"https://api.github.com/users/giovanisleite/orgs","repos_url":"https://api.github.com/users/giovanisleite/repos","events_url":"https://api.github.com/users/giovanisleite/events{/privacy}","received_events_url":"https://api.github.com/users/giovanisleite/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/giovanisleite/diario-oficial","description":null,"fork":true,"url":"https://api.github.com/repos/giovanisleite/diario-oficial","forks_url":"https://api.github.com/repos/giovanisleite/diario-oficial/forks","keys_url":"https://api.github.com/repos/giovanisleite/diario-oficial/keys{/key_id}","collaborators_url":"https://api.github.com/repos/giovanisleite/diario-oficial/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/giovanisleite/diario-oficial/teams","hooks_url":"https://api.github.com/repos/giovanisleite/diario-oficial/hooks","issue_events_url":"https://api.github.com/repos/giovanisleite/diario-oficial/issues/events{/number}","events_url":"https://api.github.com/repos/giovanisleite/diario-oficial/events","assignees_url":"https://api.github.com/repos/giovanisleite/diario-oficial/assignees{/user}","branches_url":"https://api.github.com/repos/giovanisleite/diario-oficial/branches{/branch}","tags_url":"https://api.github.com/repos/giovanisleite/diario-oficial/tags","blobs_url":"https://api.github.com/repos/giovanisleite/diario-oficial/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/giovanisleite/diario-oficial/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/giovanisleite/diario-oficial/git/refs{/sha}","trees_url":"https://api.github.com/repos/giovanisleite/diario-oficial/git/trees{/sha}","statuses_url":"https://api.github.com/repos/giovanisleite/diario-oficial/statuses/{sha}","languages_url":"https://api.github.com/repos/giovanisleite/diario-oficial/languages","stargazers_url":"https://api.github.com/repos/giovanisleite/diario-oficial/stargazers","contributors_url":"https://api.github.com/repos/giovanisleite/diario-oficial/contributors","subscribers_url":"https://api.github.com/repos/giovanisleite/diario-oficial/subscribers","subscription_url":"https://api.github.com/repos/giovanisleite/diario-oficial/subscription","commits_url":"https://api.github.com/repos/giovanisleite/diario-oficial/commits{/sha}","git_commits_url":"https://api.github.com/repos/giovanisleite/diario-oficial/git/commits{/sha}","comments_url":"https://api.github.com/repos/giovanisleite/diario-oficial/comments{/number}","issue_comment_url":"https://api.github.com/repos/giovanisleite/diario-oficial/issues/comments{/number}","contents_url":"https://api.github.com/repos/giovanisleite/diario-oficial/contents/{+path}","compare_url":"https://api.github.com/repos/giovanisleite/diario-oficial/compare/{base}...{head}","merges_url":"https://api.github.com/repos/giovanisleite/diario-oficial/merges","archive_url":"https://api.github.com/repos/giovanisleite/diario-oficial/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/giovanisleite/diario-oficial/downloads","issues_url":"https://api.github.com/repos/giovanisleite/diario-oficial/issues{/number}","pulls_url":"https://api.github.com/repos/giovanisleite/diario-oficial/pulls{/number}","milestones_url":"https://api.github.com/repos/giovanisleite/diario-oficial/milestones{/number}","notifications_url":"https://api.github.com/repos/giovanisleite/diario-oficial/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/giovanisleite/diario-oficial/labels{/name}","releases_url":"https://api.github.com/repos/giovanisleite/diario-oficial/releases{/id}","deployments_url":"https://api.github.com/repos/giovanisleite/diario-oficial/deployments","created_at":"2018-04-16T19:44:20Z","updated_at":"2018-05-03T16:54:54Z","pushed_at":"2018-05-23T03:49:51Z","git_url":"git://github.com/giovanisleite/diario-oficial.git","ssh_url":"git@github.com:giovanisleite/diario-oficial.git","clone_url":"https://github.com/giovanisleite/diario-oficial.git","svn_url":"https://github.com/giovanisleite/diario-oficial","homepage":"https://diario.serenata.ai/","size":11516,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":0,"license":{"key":"mit","name":"MIT License","spdx_id":"MIT","url":"https://api.github.com/licenses/mit"},"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"okfn-brasil:master","ref":"master","sha":"08fbeb7b32809b43d3d8f5195839dc73a86b585f","user":{"login":"okfn-brasil","id":1666382,"avatar_url":"https://avatars1.githubusercontent.com/u/1666382?v=4","gravatar_id":"","url":"https://api.github.com/users/okfn-brasil","html_url":"https://github.com/okfn-brasil","followers_url":"https://api.github.com/users/okfn-brasil/followers","following_url":"https://api.github.com/users/okfn-brasil/following{/other_user}","gists_url":"https://api.github.com/users/okfn-brasil/gists{/gist_id}","starred_url":"https://api.github.com/users/okfn-brasil/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/okfn-brasil/subscriptions","organizations_url":"https://api.github.com/users/okfn-brasil/orgs","repos_url":"https://api.github.com/users/okfn-brasil/repos","events_url":"https://api.github.com/users/okfn-brasil/events{/privacy}","received_events_url":"https://api.github.com/users/okfn-brasil/received_events","type":"Organization","site_admin":false},"repo":{"id":127598755,"name":"diario-oficial","full_name":"okfn-brasil/diario-oficial","owner":{"login":"okfn-brasil","id":1666382,"avatar_url":"https://avatars1.githubusercontent.com/u/1666382?v=4","gravatar_id":"","url":"https://api.github.com/users/okfn-brasil","html_url":"https://github.com/okfn-brasil","followers_url":"https://api.github.com/users/okfn-brasil/followers","following_url":"https://api.github.com/users/okfn-brasil/following{/other_user}","gists_url":"https://api.github.com/users/okfn-brasil/gists{/gist_id}","starred_url":"https://api.github.com/users/okfn-brasil/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/okfn-brasil/subscriptions","organizations_url":"https://api.github.com/users/okfn-brasil/orgs","repos_url":"https://api.github.com/users/okfn-brasil/repos","events_url":"https://api.github.com/users/okfn-brasil/events{/privacy}","received_events_url":"https://api.github.com/users/okfn-brasil/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/okfn-brasil/diario-oficial","description":"📰 Brazilian government gazettes, accessible to everyone.","fork":false,"url":"https://api.github.com/repos/okfn-brasil/diario-oficial","forks_url":"https://api.github.com/repos/okfn-brasil/diario-oficial/forks","keys_url":"https://api.github.com/repos/okfn-brasil/diario-oficial/keys{/key_id}","collaborators_url":"https://api.github.com/repos/okfn-brasil/diario-oficial/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/okfn-brasil/diario-oficial/teams","hooks_url":"https://api.github.com/repos/okfn-brasil/diario-oficial/hooks","issue_events_url":"https://api.github.com/repos/okfn-brasil/diario-oficial/issues/events{/number}","events_url":"https://api.github.com/repos/okfn-brasil/diario-oficial/events","assignees_url":"https://api.github.com/repos/okfn-brasil/diario-oficial/assignees{/user}","branches_url":"https://api.github.com/repos/okfn-brasil/diario-oficial/branches{/branch}","tags_url":"https://api.github.com/repos/okfn-brasil/diario-oficial/tags","blobs_url":"https://api.github.com/repos/okfn-brasil/diario-oficial/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/okfn-brasil/diario-oficial/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/okfn-brasil/diario-oficial/git/refs{/sha}","trees_url":"https://api.github.com/repos/okfn-brasil/diario-oficial/git/trees{/sha}","statuses_url":"https://api.github.com/repos/okfn-brasil/diario-oficial/statuses/{sha}","languages_url":"https://api.github.com/repos/okfn-brasil/diario-oficial/languages","stargazers_url":"https://api.github.com/repos/okfn-brasil/diario-oficial/stargazers","contributors_url":"https://api.github.com/repos/okfn-brasil/diario-oficial/contributors","subscribers_url":"https://api.github.com/repos/okfn-brasil/diario-oficial/subscribers","subscription_url":"https://api.github.com/repos/okfn-brasil/diario-oficial/subscription","commits_url":"https://api.github.com/repos/okfn-brasil/diario-oficial/commits{/sha}","git_commits_url":"https://api.github.com/repos/okfn-brasil/diario-oficial/git/commits{/sha}","comments_url":"https://api.github.com/repos/okfn-brasil/diario-oficial/comments{/number}","issue_comment_url":"https://api.github.com/repos/okfn-brasil/diario-oficial/issues/comments{/number}","contents_url":"https://api.github.com/repos/okfn-brasil/diario-oficial/contents/{+path}","compare_url":"https://api.github.com/repos/okfn-brasil/diario-oficial/compare/{base}...{head}","merges_url":"https://api.github.com/repos/okfn-brasil/diario-oficial/merges","archive_url":"https://api.github.com/repos/okfn-brasil/diario-oficial/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/okfn-brasil/diario-oficial/downloads","issues_url":"https://api.github.com/repos/okfn-brasil/diario-oficial/issues{/number}","pulls_url":"https://api.github.com/repos/okfn-brasil/diario-oficial/pulls{/number}","milestones_url":"https://api.github.com/repos/okfn-brasil/diario-oficial/milestones{/number}","notifications_url":"https://api.github.com/repos/okfn-brasil/diario-oficial/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/okfn-brasil/diario-oficial/labels{/name}","releases_url":"https://api.github.com/repos/okfn-brasil/diario-oficial/releases{/id}","deployments_url":"https://api.github.com/repos/okfn-brasil/diario-oficial/deployments","created_at":"2018-04-01T05:01:21Z","updated_at":"2018-05-27T20:29:19Z","pushed_at":"2018-05-27T23:40:50Z","git_url":"git://github.com/okfn-brasil/diario-oficial.git","ssh_url":"git@github.com:okfn-brasil/diario-oficial.git","clone_url":"https://github.com/okfn-brasil/diario-oficial.git","svn_url":"https://github.com/okfn-brasil/diario-oficial","homepage":"https://diario.serenata.ai/","size":11664,"stargazers_count":156,"watchers_count":156,"language":"Python","has_issues":true,"has_projects":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":49,"mirror_url":null,"archived":false,"open_issues_count":28,"license":{"key":"mit","name":"MIT License","spdx_id":"MIT","url":"https://api.github.com/licenses/mit"},"forks":49,"open_issues":28,"watchers":156,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/okfn-brasil/diario-oficial/pulls/27"},"html":{"href":"https://github.com/okfn-brasil/diario-oficial/pull/27"},"issue":{"href":"https://api.github.com/repos/okfn-brasil/diario-oficial/issues/27"},"comments":{"href":"https://api.github.com/repos/okfn-brasil/diario-oficial/issues/27/comments"},"review_comments":{"href":"https://api.github.com/repos/okfn-brasil/diario-oficial/pulls/27/comments"},"review_comment":{"href":"https://api.github.com/repos/okfn-brasil/diario-oficial/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/okfn-brasil/diario-oficial/pulls/27/commits"},"statuses":{"href":"https://api.github.com/repos/okfn-brasil/diario-oficial/statuses/4692320d7ba4528d27ae3978244bba5ccaf6761b"}},"author_association":"CONTRIBUTOR"}}
{"actor":{"display_login":"rennerocha"}}