{"action":"created","comment":{"url":"https://api.github.com/repos/scrapinghub/frontera/pulls/comments/78899908","id":78899908,"diff_hunk":"@@ -0,0 +1,78 @@\n+from __future__ import absolute_import\n+import unittest\n+\n+from frontera.contrib.backends.remote.messagebus import MessageBusBackend\n+from frontera.settings import Settings\n+from frontera.core.models import Request, Response\n+\n+\n+r1 = Request('http://www.example.com/', meta={b'domain': {b'fingerprint': b'1'}})\n+r2 = Request('http://www.scrapy.org/', meta={b'domain': {b'fingerprint': b'2'}})\n+r3 = Request('http://www.test.com/some/page', meta={b'domain': {b'fingerprint': b'3'}})\n+\n+\n+class TestMessageBusBackend(unittest.TestCase):\n+\n+    def mbb_setup(self, settings=None):\n+        manager = type('manager', (object,), {})\n+        settings = settings or Settings()\n+        settings.MESSAGE_BUS = 'tests.mocks.message_bus.FakeMessageBus'\n+        settings.STORE_CONTENT = True\n+        manager.settings = settings\n+        manager.request_model = Request\n+        manager.response_model = Response\n+        return MessageBusBackend(manager)\n+\n+    def test_feed_partitions_less_than_equal_partion_id_and_partion_id_less_than_zero(self):\n+        settings = Settings()\n+        # test partition_id > feed_partitions\n+        settings.SPIDER_PARTITION_ID = 2\n+        settings.SPIDER_FEED_PARTITIONS = 1\n+        self.assertRaises(ValueError, self.mbb_setup, settings)\n+\n+        # test partition_id = feed_partitions\n+        settings.SPIDER_PARTITION_ID = 1\n+        self.assertRaises(ValueError, self.mbb_setup, settings)\n+\n+        # test partition_id < 0\n+        settings.SPIDER_PARTITION_ID = -1\n+        self.assertRaises(ValueError, self.mbb_setup, settings)\n+\n+    def test_add_seeds(self):\n+        mbb = self.mbb_setup()\n+        mbb.add_seeds([r1, r2, r3])\n+        seeds = [mbb._decoder.decode(m)[1][0] for m in mbb.spider_log_producer.messages]\n+        self.assertEqual(set([seed.url for seed in seeds]), set([r1.url, r2.url, r3.url]))\n+\n+    def test_page_crawled(self):\n+        mbb = self.mbb_setup()\n+        resp = Response(r1.url, body='body', request=r1)\n+        mbb.page_crawled(resp)\n+        page = mbb._decoder.decode(mbb.spider_log_producer.messages[0])[1]\n+        self.assertEqual(page.request.url, resp.request.url)\n+\n+    def test_links_extracted(self):\n+        mbb = self.mbb_setup()\n+        mbb.links_extracted(r1, [r2, r3])\n+        links = [mbb._decoder.decode(m)[2][0] for m in mbb.spider_log_producer.messages]\n+        self.assertEqual(set([link.url for link in links]), set([r2.url, r3.url]))","path":"tests/test_message_bus_backend.py","position":58,"original_position":58,"commit_id":"36c12e8ae8c7e809912153e463d16b2cec60a6f1","original_commit_id":"36c12e8ae8c7e809912153e463d16b2cec60a6f1","user":{"login":"sibiryakov","id":9044124,"avatar_url":"https://avatars.githubusercontent.com/u/9044124?v=3","gravatar_id":"","url":"https://api.github.com/users/sibiryakov","html_url":"https://github.com/sibiryakov","followers_url":"https://api.github.com/users/sibiryakov/followers","following_url":"https://api.github.com/users/sibiryakov/following{/other_user}","gists_url":"https://api.github.com/users/sibiryakov/gists{/gist_id}","starred_url":"https://api.github.com/users/sibiryakov/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/sibiryakov/subscriptions","organizations_url":"https://api.github.com/users/sibiryakov/orgs","repos_url":"https://api.github.com/users/sibiryakov/repos","events_url":"https://api.github.com/users/sibiryakov/events{/privacy}","received_events_url":"https://api.github.com/users/sibiryakov/received_events","type":"User","site_admin":false},"body":"The request check (r1) is missing.","created_at":"2016-09-15T06:57:35Z","updated_at":"2016-09-15T07:03:17Z","html_url":"https://github.com/scrapinghub/frontera/pull/205#discussion_r78899908","pull_request_url":"https://api.github.com/repos/scrapinghub/frontera/pulls/205","_links":{"self":{"href":"https://api.github.com/repos/scrapinghub/frontera/pulls/comments/78899908"},"html":{"href":"https://github.com/scrapinghub/frontera/pull/205#discussion_r78899908"},"pull_request":{"href":"https://api.github.com/repos/scrapinghub/frontera/pulls/205"}}},"pull_request":{"url":"https://api.github.com/repos/scrapinghub/frontera/pulls/205","id":85038983,"html_url":"https://github.com/scrapinghub/frontera/pull/205","diff_url":"https://github.com/scrapinghub/frontera/pull/205.diff","patch_url":"https://github.com/scrapinghub/frontera/pull/205.patch","issue_url":"https://api.github.com/repos/scrapinghub/frontera/issues/205","number":205,"state":"open","locked":false,"title":"Added tests for message_bus_backend","user":{"login":"voith","id":8171193,"avatar_url":"https://avatars.githubusercontent.com/u/8171193?v=3","gravatar_id":"","url":"https://api.github.com/users/voith","html_url":"https://github.com/voith","followers_url":"https://api.github.com/users/voith/followers","following_url":"https://api.github.com/users/voith/following{/other_user}","gists_url":"https://api.github.com/users/voith/gists{/gist_id}","starred_url":"https://api.github.com/users/voith/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/voith/subscriptions","organizations_url":"https://api.github.com/users/voith/orgs","repos_url":"https://api.github.com/users/voith/repos","events_url":"https://api.github.com/users/voith/events{/privacy}","received_events_url":"https://api.github.com/users/voith/received_events","type":"User","site_admin":false},"body":"","created_at":"2016-09-13T12:30:32Z","updated_at":"2016-09-15T07:03:17Z","closed_at":null,"merged_at":null,"merge_commit_sha":"0388465bf56172c5d42c1652d38947ff1dfa59dc","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/scrapinghub/frontera/pulls/205/commits","review_comments_url":"https://api.github.com/repos/scrapinghub/frontera/pulls/205/comments","review_comment_url":"https://api.github.com/repos/scrapinghub/frontera/pulls/comments{/number}","comments_url":"https://api.github.com/repos/scrapinghub/frontera/issues/205/comments","statuses_url":"https://api.github.com/repos/scrapinghub/frontera/statuses/36c12e8ae8c7e809912153e463d16b2cec60a6f1","head":{"label":"voith:test-messagebackend","ref":"test-messagebackend","sha":"36c12e8ae8c7e809912153e463d16b2cec60a6f1","user":{"login":"voith","id":8171193,"avatar_url":"https://avatars.githubusercontent.com/u/8171193?v=3","gravatar_id":"","url":"https://api.github.com/users/voith","html_url":"https://github.com/voith","followers_url":"https://api.github.com/users/voith/followers","following_url":"https://api.github.com/users/voith/following{/other_user}","gists_url":"https://api.github.com/users/voith/gists{/gist_id}","starred_url":"https://api.github.com/users/voith/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/voith/subscriptions","organizations_url":"https://api.github.com/users/voith/orgs","repos_url":"https://api.github.com/users/voith/repos","events_url":"https://api.github.com/users/voith/events{/privacy}","received_events_url":"https://api.github.com/users/voith/received_events","type":"User","site_admin":false},"repo":{"id":51388183,"name":"frontera","full_name":"voith/frontera","owner":{"login":"voith","id":8171193,"avatar_url":"https://avatars.githubusercontent.com/u/8171193?v=3","gravatar_id":"","url":"https://api.github.com/users/voith","html_url":"https://github.com/voith","followers_url":"https://api.github.com/users/voith/followers","following_url":"https://api.github.com/users/voith/following{/other_user}","gists_url":"https://api.github.com/users/voith/gists{/gist_id}","starred_url":"https://api.github.com/users/voith/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/voith/subscriptions","organizations_url":"https://api.github.com/users/voith/orgs","repos_url":"https://api.github.com/users/voith/repos","events_url":"https://api.github.com/users/voith/events{/privacy}","received_events_url":"https://api.github.com/users/voith/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/voith/frontera","description":"A flexible frontier for web crawlers","fork":true,"url":"https://api.github.com/repos/voith/frontera","forks_url":"https://api.github.com/repos/voith/frontera/forks","keys_url":"https://api.github.com/repos/voith/frontera/keys{/key_id}","collaborators_url":"https://api.github.com/repos/voith/frontera/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/voith/frontera/teams","hooks_url":"https://api.github.com/repos/voith/frontera/hooks","issue_events_url":"https://api.github.com/repos/voith/frontera/issues/events{/number}","events_url":"https://api.github.com/repos/voith/frontera/events","assignees_url":"https://api.github.com/repos/voith/frontera/assignees{/user}","branches_url":"https://api.github.com/repos/voith/frontera/branches{/branch}","tags_url":"https://api.github.com/repos/voith/frontera/tags","blobs_url":"https://api.github.com/repos/voith/frontera/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/voith/frontera/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/voith/frontera/git/refs{/sha}","trees_url":"https://api.github.com/repos/voith/frontera/git/trees{/sha}","statuses_url":"https://api.github.com/repos/voith/frontera/statuses/{sha}","languages_url":"https://api.github.com/repos/voith/frontera/languages","stargazers_url":"https://api.github.com/repos/voith/frontera/stargazers","contributors_url":"https://api.github.com/repos/voith/frontera/contributors","subscribers_url":"https://api.github.com/repos/voith/frontera/subscribers","subscription_url":"https://api.github.com/repos/voith/frontera/subscription","commits_url":"https://api.github.com/repos/voith/frontera/commits{/sha}","git_commits_url":"https://api.github.com/repos/voith/frontera/git/commits{/sha}","comments_url":"https://api.github.com/repos/voith/frontera/comments{/number}","issue_comment_url":"https://api.github.com/repos/voith/frontera/issues/comments{/number}","contents_url":"https://api.github.com/repos/voith/frontera/contents/{+path}","compare_url":"https://api.github.com/repos/voith/frontera/compare/{base}...{head}","merges_url":"https://api.github.com/repos/voith/frontera/merges","archive_url":"https://api.github.com/repos/voith/frontera/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/voith/frontera/downloads","issues_url":"https://api.github.com/repos/voith/frontera/issues{/number}","pulls_url":"https://api.github.com/repos/voith/frontera/pulls{/number}","milestones_url":"https://api.github.com/repos/voith/frontera/milestones{/number}","notifications_url":"https://api.github.com/repos/voith/frontera/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/voith/frontera/labels{/name}","releases_url":"https://api.github.com/repos/voith/frontera/releases{/id}","deployments_url":"https://api.github.com/repos/voith/frontera/deployments","created_at":"2016-02-09T18:21:11Z","updated_at":"2016-02-09T18:21:12Z","pushed_at":"2016-09-13T12:28:08Z","git_url":"git://github.com/voith/frontera.git","ssh_url":"git@github.com:voith/frontera.git","clone_url":"https://github.com/voith/frontera.git","svn_url":"https://github.com/voith/frontera","homepage":null,"size":4078,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"scrapinghub:master","ref":"master","sha":"9ff6e76548277d25a31f861331044d4900f4c7dc","user":{"login":"scrapinghub","id":699596,"avatar_url":"https://avatars.githubusercontent.com/u/699596?v=3","gravatar_id":"","url":"https://api.github.com/users/scrapinghub","html_url":"https://github.com/scrapinghub","followers_url":"https://api.github.com/users/scrapinghub/followers","following_url":"https://api.github.com/users/scrapinghub/following{/other_user}","gists_url":"https://api.github.com/users/scrapinghub/gists{/gist_id}","starred_url":"https://api.github.com/users/scrapinghub/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/scrapinghub/subscriptions","organizations_url":"https://api.github.com/users/scrapinghub/orgs","repos_url":"https://api.github.com/users/scrapinghub/repos","events_url":"https://api.github.com/users/scrapinghub/events{/privacy}","received_events_url":"https://api.github.com/users/scrapinghub/received_events","type":"Organization","site_admin":false},"repo":{"id":27003377,"name":"frontera","full_name":"scrapinghub/frontera","owner":{"login":"scrapinghub","id":699596,"avatar_url":"https://avatars.githubusercontent.com/u/699596?v=3","gravatar_id":"","url":"https://api.github.com/users/scrapinghub","html_url":"https://github.com/scrapinghub","followers_url":"https://api.github.com/users/scrapinghub/followers","following_url":"https://api.github.com/users/scrapinghub/following{/other_user}","gists_url":"https://api.github.com/users/scrapinghub/gists{/gist_id}","starred_url":"https://api.github.com/users/scrapinghub/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/scrapinghub/subscriptions","organizations_url":"https://api.github.com/users/scrapinghub/orgs","repos_url":"https://api.github.com/users/scrapinghub/repos","events_url":"https://api.github.com/users/scrapinghub/events{/privacy}","received_events_url":"https://api.github.com/users/scrapinghub/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/scrapinghub/frontera","description":"A scalable frontier for web crawlers","fork":false,"url":"https://api.github.com/repos/scrapinghub/frontera","forks_url":"https://api.github.com/repos/scrapinghub/frontera/forks","keys_url":"https://api.github.com/repos/scrapinghub/frontera/keys{/key_id}","collaborators_url":"https://api.github.com/repos/scrapinghub/frontera/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/scrapinghub/frontera/teams","hooks_url":"https://api.github.com/repos/scrapinghub/frontera/hooks","issue_events_url":"https://api.github.com/repos/scrapinghub/frontera/issues/events{/number}","events_url":"https://api.github.com/repos/scrapinghub/frontera/events","assignees_url":"https://api.github.com/repos/scrapinghub/frontera/assignees{/user}","branches_url":"https://api.github.com/repos/scrapinghub/frontera/branches{/branch}","tags_url":"https://api.github.com/repos/scrapinghub/frontera/tags","blobs_url":"https://api.github.com/repos/scrapinghub/frontera/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/scrapinghub/frontera/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/scrapinghub/frontera/git/refs{/sha}","trees_url":"https://api.github.com/repos/scrapinghub/frontera/git/trees{/sha}","statuses_url":"https://api.github.com/repos/scrapinghub/frontera/statuses/{sha}","languages_url":"https://api.github.com/repos/scrapinghub/frontera/languages","stargazers_url":"https://api.github.com/repos/scrapinghub/frontera/stargazers","contributors_url":"https://api.github.com/repos/scrapinghub/frontera/contributors","subscribers_url":"https://api.github.com/repos/scrapinghub/frontera/subscribers","subscription_url":"https://api.github.com/repos/scrapinghub/frontera/subscription","commits_url":"https://api.github.com/repos/scrapinghub/frontera/commits{/sha}","git_commits_url":"https://api.github.com/repos/scrapinghub/frontera/git/commits{/sha}","comments_url":"https://api.github.com/repos/scrapinghub/frontera/comments{/number}","issue_comment_url":"https://api.github.com/repos/scrapinghub/frontera/issues/comments{/number}","contents_url":"https://api.github.com/repos/scrapinghub/frontera/contents/{+path}","compare_url":"https://api.github.com/repos/scrapinghub/frontera/compare/{base}...{head}","merges_url":"https://api.github.com/repos/scrapinghub/frontera/merges","archive_url":"https://api.github.com/repos/scrapinghub/frontera/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/scrapinghub/frontera/downloads","issues_url":"https://api.github.com/repos/scrapinghub/frontera/issues{/number}","pulls_url":"https://api.github.com/repos/scrapinghub/frontera/pulls{/number}","milestones_url":"https://api.github.com/repos/scrapinghub/frontera/milestones{/number}","notifications_url":"https://api.github.com/repos/scrapinghub/frontera/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/scrapinghub/frontera/labels{/name}","releases_url":"https://api.github.com/repos/scrapinghub/frontera/releases{/id}","deployments_url":"https://api.github.com/repos/scrapinghub/frontera/deployments","created_at":"2014-11-22T15:42:50Z","updated_at":"2016-09-12T05:22:15Z","pushed_at":"2016-09-13T12:30:33Z","git_url":"git://github.com/scrapinghub/frontera.git","ssh_url":"git@github.com:scrapinghub/frontera.git","clone_url":"https://github.com/scrapinghub/frontera.git","svn_url":"https://github.com/scrapinghub/frontera","homepage":"","size":4867,"stargazers_count":265,"watchers_count":265,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":73,"mirror_url":null,"open_issues_count":46,"forks":73,"open_issues":46,"watchers":265,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/scrapinghub/frontera/pulls/205"},"html":{"href":"https://github.com/scrapinghub/frontera/pull/205"},"issue":{"href":"https://api.github.com/repos/scrapinghub/frontera/issues/205"},"comments":{"href":"https://api.github.com/repos/scrapinghub/frontera/issues/205/comments"},"review_comments":{"href":"https://api.github.com/repos/scrapinghub/frontera/pulls/205/comments"},"review_comment":{"href":"https://api.github.com/repos/scrapinghub/frontera/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/scrapinghub/frontera/pulls/205/commits"},"statuses":{"href":"https://api.github.com/repos/scrapinghub/frontera/statuses/36c12e8ae8c7e809912153e463d16b2cec60a6f1"}}}}
{"actor":{"display_login":"sibiryakov"}}