{"action":"closed","number":34,"pull_request":{"url":"https://api.github.com/repos/ViDA-NYU/ache/pulls/34","id":63167092,"html_url":"https://github.com/ViDA-NYU/ache/pull/34","diff_url":"https://github.com/ViDA-NYU/ache/pull/34.diff","patch_url":"https://github.com/ViDA-NYU/ache/pull/34.patch","issue_url":"https://api.github.com/repos/ViDA-NYU/ache/issues/34","number":34,"state":"closed","locked":false,"title":"Rewrite of CrawlerManager using a single threaded manager","user":{"login":"aecio","id":150570,"avatar_url":"https://avatars.githubusercontent.com/u/150570?v=3","gravatar_id":"","url":"https://api.github.com/users/aecio","html_url":"https://github.com/aecio","followers_url":"https://api.github.com/users/aecio/followers","following_url":"https://api.github.com/users/aecio/following{/other_user}","gists_url":"https://api.github.com/users/aecio/gists{/gist_id}","starred_url":"https://api.github.com/users/aecio/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/aecio/subscriptions","organizations_url":"https://api.github.com/users/aecio/orgs","repos_url":"https://api.github.com/users/aecio/repos","events_url":"https://api.github.com/users/aecio/events{/privacy}","received_events_url":"https://api.github.com/users/aecio/received_events","type":"User","site_admin":false},"body":"This is a rewrite of the CrawlerManager module using a single thread to manage a pool of downloading threads. This solves previous problems with multiple threads being created indefinitely. Code is simpler and unit tested. Also already include fixes for the following issues:\r\n- Fixes #21\r\n- Fixes #26 \r\n\r\nAlso include new features:\r\n- Minimum access interval to websites of the same domain\r\n- Allow configure valid mime types that should be downloaded (issue #26). The fetcher will cancel the download if the mime type is not one those mime types specified.","created_at":"2016-03-17T03:00:38Z","updated_at":"2016-04-08T23:49:54Z","closed_at":"2016-04-08T23:49:54Z","merged_at":"2016-04-08T23:49:54Z","merge_commit_sha":"ff3efe6d158fdec85db90421963c7c79b5208bd5","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/ViDA-NYU/ache/pulls/34/commits","review_comments_url":"https://api.github.com/repos/ViDA-NYU/ache/pulls/34/comments","review_comment_url":"https://api.github.com/repos/ViDA-NYU/ache/pulls/comments{/number}","comments_url":"https://api.github.com/repos/ViDA-NYU/ache/issues/34/comments","statuses_url":"https://api.github.com/repos/ViDA-NYU/ache/statuses/ff3efe6d158fdec85db90421963c7c79b5208bd5","head":{"label":"ViDA-NYU:aecio/async-crawler","ref":"aecio/async-crawler","sha":"ff3efe6d158fdec85db90421963c7c79b5208bd5","user":{"login":"ViDA-NYU","id":8114708,"avatar_url":"https://avatars.githubusercontent.com/u/8114708?v=3","gravatar_id":"","url":"https://api.github.com/users/ViDA-NYU","html_url":"https://github.com/ViDA-NYU","followers_url":"https://api.github.com/users/ViDA-NYU/followers","following_url":"https://api.github.com/users/ViDA-NYU/following{/other_user}","gists_url":"https://api.github.com/users/ViDA-NYU/gists{/gist_id}","starred_url":"https://api.github.com/users/ViDA-NYU/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ViDA-NYU/subscriptions","organizations_url":"https://api.github.com/users/ViDA-NYU/orgs","repos_url":"https://api.github.com/users/ViDA-NYU/repos","events_url":"https://api.github.com/users/ViDA-NYU/events{/privacy}","received_events_url":"https://api.github.com/users/ViDA-NYU/received_events","type":"Organization","site_admin":false},"repo":{"id":28800030,"name":"ache","full_name":"ViDA-NYU/ache","owner":{"login":"ViDA-NYU","id":8114708,"avatar_url":"https://avatars.githubusercontent.com/u/8114708?v=3","gravatar_id":"","url":"https://api.github.com/users/ViDA-NYU","html_url":"https://github.com/ViDA-NYU","followers_url":"https://api.github.com/users/ViDA-NYU/followers","following_url":"https://api.github.com/users/ViDA-NYU/following{/other_user}","gists_url":"https://api.github.com/users/ViDA-NYU/gists{/gist_id}","starred_url":"https://api.github.com/users/ViDA-NYU/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ViDA-NYU/subscriptions","organizations_url":"https://api.github.com/users/ViDA-NYU/orgs","repos_url":"https://api.github.com/users/ViDA-NYU/repos","events_url":"https://api.github.com/users/ViDA-NYU/events{/privacy}","received_events_url":"https://api.github.com/users/ViDA-NYU/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/ViDA-NYU/ache","description":"ACHE is a web crawler for domain-specific search","fork":false,"url":"https://api.github.com/repos/ViDA-NYU/ache","forks_url":"https://api.github.com/repos/ViDA-NYU/ache/forks","keys_url":"https://api.github.com/repos/ViDA-NYU/ache/keys{/key_id}","collaborators_url":"https://api.github.com/repos/ViDA-NYU/ache/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/ViDA-NYU/ache/teams","hooks_url":"https://api.github.com/repos/ViDA-NYU/ache/hooks","issue_events_url":"https://api.github.com/repos/ViDA-NYU/ache/issues/events{/number}","events_url":"https://api.github.com/repos/ViDA-NYU/ache/events","assignees_url":"https://api.github.com/repos/ViDA-NYU/ache/assignees{/user}","branches_url":"https://api.github.com/repos/ViDA-NYU/ache/branches{/branch}","tags_url":"https://api.github.com/repos/ViDA-NYU/ache/tags","blobs_url":"https://api.github.com/repos/ViDA-NYU/ache/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/ViDA-NYU/ache/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/ViDA-NYU/ache/git/refs{/sha}","trees_url":"https://api.github.com/repos/ViDA-NYU/ache/git/trees{/sha}","statuses_url":"https://api.github.com/repos/ViDA-NYU/ache/statuses/{sha}","languages_url":"https://api.github.com/repos/ViDA-NYU/ache/languages","stargazers_url":"https://api.github.com/repos/ViDA-NYU/ache/stargazers","contributors_url":"https://api.github.com/repos/ViDA-NYU/ache/contributors","subscribers_url":"https://api.github.com/repos/ViDA-NYU/ache/subscribers","subscription_url":"https://api.github.com/repos/ViDA-NYU/ache/subscription","commits_url":"https://api.github.com/repos/ViDA-NYU/ache/commits{/sha}","git_commits_url":"https://api.github.com/repos/ViDA-NYU/ache/git/commits{/sha}","comments_url":"https://api.github.com/repos/ViDA-NYU/ache/comments{/number}","issue_comment_url":"https://api.github.com/repos/ViDA-NYU/ache/issues/comments{/number}","contents_url":"https://api.github.com/repos/ViDA-NYU/ache/contents/{+path}","compare_url":"https://api.github.com/repos/ViDA-NYU/ache/compare/{base}...{head}","merges_url":"https://api.github.com/repos/ViDA-NYU/ache/merges","archive_url":"https://api.github.com/repos/ViDA-NYU/ache/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/ViDA-NYU/ache/downloads","issues_url":"https://api.github.com/repos/ViDA-NYU/ache/issues{/number}","pulls_url":"https://api.github.com/repos/ViDA-NYU/ache/pulls{/number}","milestones_url":"https://api.github.com/repos/ViDA-NYU/ache/milestones{/number}","notifications_url":"https://api.github.com/repos/ViDA-NYU/ache/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/ViDA-NYU/ache/labels{/name}","releases_url":"https://api.github.com/repos/ViDA-NYU/ache/releases{/id}","deployments_url":"https://api.github.com/repos/ViDA-NYU/ache/deployments","created_at":"2015-01-05T06:11:43Z","updated_at":"2016-04-05T08:28:00Z","pushed_at":"2016-04-08T23:49:54Z","git_url":"git://github.com/ViDA-NYU/ache.git","ssh_url":"git@github.com:ViDA-NYU/ache.git","clone_url":"https://github.com/ViDA-NYU/ache.git","svn_url":"https://github.com/ViDA-NYU/ache","homepage":"","size":64485,"stargazers_count":28,"watchers_count":28,"language":"HTML","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":16,"mirror_url":null,"open_issues_count":4,"forks":16,"open_issues":4,"watchers":28,"default_branch":"master"}},"base":{"label":"ViDA-NYU:master","ref":"master","sha":"44fd61b27945ecb726c323c9a71d1a0ebbaa0bb0","user":{"login":"ViDA-NYU","id":8114708,"avatar_url":"https://avatars.githubusercontent.com/u/8114708?v=3","gravatar_id":"","url":"https://api.github.com/users/ViDA-NYU","html_url":"https://github.com/ViDA-NYU","followers_url":"https://api.github.com/users/ViDA-NYU/followers","following_url":"https://api.github.com/users/ViDA-NYU/following{/other_user}","gists_url":"https://api.github.com/users/ViDA-NYU/gists{/gist_id}","starred_url":"https://api.github.com/users/ViDA-NYU/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ViDA-NYU/subscriptions","organizations_url":"https://api.github.com/users/ViDA-NYU/orgs","repos_url":"https://api.github.com/users/ViDA-NYU/repos","events_url":"https://api.github.com/users/ViDA-NYU/events{/privacy}","received_events_url":"https://api.github.com/users/ViDA-NYU/received_events","type":"Organization","site_admin":false},"repo":{"id":28800030,"name":"ache","full_name":"ViDA-NYU/ache","owner":{"login":"ViDA-NYU","id":8114708,"avatar_url":"https://avatars.githubusercontent.com/u/8114708?v=3","gravatar_id":"","url":"https://api.github.com/users/ViDA-NYU","html_url":"https://github.com/ViDA-NYU","followers_url":"https://api.github.com/users/ViDA-NYU/followers","following_url":"https://api.github.com/users/ViDA-NYU/following{/other_user}","gists_url":"https://api.github.com/users/ViDA-NYU/gists{/gist_id}","starred_url":"https://api.github.com/users/ViDA-NYU/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ViDA-NYU/subscriptions","organizations_url":"https://api.github.com/users/ViDA-NYU/orgs","repos_url":"https://api.github.com/users/ViDA-NYU/repos","events_url":"https://api.github.com/users/ViDA-NYU/events{/privacy}","received_events_url":"https://api.github.com/users/ViDA-NYU/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/ViDA-NYU/ache","description":"ACHE is a web crawler for domain-specific search","fork":false,"url":"https://api.github.com/repos/ViDA-NYU/ache","forks_url":"https://api.github.com/repos/ViDA-NYU/ache/forks","keys_url":"https://api.github.com/repos/ViDA-NYU/ache/keys{/key_id}","collaborators_url":"https://api.github.com/repos/ViDA-NYU/ache/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/ViDA-NYU/ache/teams","hooks_url":"https://api.github.com/repos/ViDA-NYU/ache/hooks","issue_events_url":"https://api.github.com/repos/ViDA-NYU/ache/issues/events{/number}","events_url":"https://api.github.com/repos/ViDA-NYU/ache/events","assignees_url":"https://api.github.com/repos/ViDA-NYU/ache/assignees{/user}","branches_url":"https://api.github.com/repos/ViDA-NYU/ache/branches{/branch}","tags_url":"https://api.github.com/repos/ViDA-NYU/ache/tags","blobs_url":"https://api.github.com/repos/ViDA-NYU/ache/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/ViDA-NYU/ache/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/ViDA-NYU/ache/git/refs{/sha}","trees_url":"https://api.github.com/repos/ViDA-NYU/ache/git/trees{/sha}","statuses_url":"https://api.github.com/repos/ViDA-NYU/ache/statuses/{sha}","languages_url":"https://api.github.com/repos/ViDA-NYU/ache/languages","stargazers_url":"https://api.github.com/repos/ViDA-NYU/ache/stargazers","contributors_url":"https://api.github.com/repos/ViDA-NYU/ache/contributors","subscribers_url":"https://api.github.com/repos/ViDA-NYU/ache/subscribers","subscription_url":"https://api.github.com/repos/ViDA-NYU/ache/subscription","commits_url":"https://api.github.com/repos/ViDA-NYU/ache/commits{/sha}","git_commits_url":"https://api.github.com/repos/ViDA-NYU/ache/git/commits{/sha}","comments_url":"https://api.github.com/repos/ViDA-NYU/ache/comments{/number}","issue_comment_url":"https://api.github.com/repos/ViDA-NYU/ache/issues/comments{/number}","contents_url":"https://api.github.com/repos/ViDA-NYU/ache/contents/{+path}","compare_url":"https://api.github.com/repos/ViDA-NYU/ache/compare/{base}...{head}","merges_url":"https://api.github.com/repos/ViDA-NYU/ache/merges","archive_url":"https://api.github.com/repos/ViDA-NYU/ache/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/ViDA-NYU/ache/downloads","issues_url":"https://api.github.com/repos/ViDA-NYU/ache/issues{/number}","pulls_url":"https://api.github.com/repos/ViDA-NYU/ache/pulls{/number}","milestones_url":"https://api.github.com/repos/ViDA-NYU/ache/milestones{/number}","notifications_url":"https://api.github.com/repos/ViDA-NYU/ache/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/ViDA-NYU/ache/labels{/name}","releases_url":"https://api.github.com/repos/ViDA-NYU/ache/releases{/id}","deployments_url":"https://api.github.com/repos/ViDA-NYU/ache/deployments","created_at":"2015-01-05T06:11:43Z","updated_at":"2016-04-05T08:28:00Z","pushed_at":"2016-04-08T23:49:54Z","git_url":"git://github.com/ViDA-NYU/ache.git","ssh_url":"git@github.com:ViDA-NYU/ache.git","clone_url":"https://github.com/ViDA-NYU/ache.git","svn_url":"https://github.com/ViDA-NYU/ache","homepage":"","size":64485,"stargazers_count":28,"watchers_count":28,"language":"HTML","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":16,"mirror_url":null,"open_issues_count":4,"forks":16,"open_issues":4,"watchers":28,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/ViDA-NYU/ache/pulls/34"},"html":{"href":"https://github.com/ViDA-NYU/ache/pull/34"},"issue":{"href":"https://api.github.com/repos/ViDA-NYU/ache/issues/34"},"comments":{"href":"https://api.github.com/repos/ViDA-NYU/ache/issues/34/comments"},"review_comments":{"href":"https://api.github.com/repos/ViDA-NYU/ache/pulls/34/comments"},"review_comment":{"href":"https://api.github.com/repos/ViDA-NYU/ache/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/ViDA-NYU/ache/pulls/34/commits"},"statuses":{"href":"https://api.github.com/repos/ViDA-NYU/ache/statuses/ff3efe6d158fdec85db90421963c7c79b5208bd5"}},"merged":true,"mergeable":null,"mergeable_state":"unknown","merged_by":{"login":"aecio","id":150570,"avatar_url":"https://avatars.githubusercontent.com/u/150570?v=3","gravatar_id":"","url":"https://api.github.com/users/aecio","html_url":"https://github.com/aecio","followers_url":"https://api.github.com/users/aecio/followers","following_url":"https://api.github.com/users/aecio/following{/other_user}","gists_url":"https://api.github.com/users/aecio/gists{/gist_id}","starred_url":"https://api.github.com/users/aecio/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/aecio/subscriptions","organizations_url":"https://api.github.com/users/aecio/orgs","repos_url":"https://api.github.com/users/aecio/repos","events_url":"https://api.github.com/users/aecio/events{/privacy}","received_events_url":"https://api.github.com/users/aecio/received_events","type":"User","site_admin":false},"comments":0,"review_comments":0,"commits":58,"additions":1100,"deletions":1455,"changed_files":25}}