{"action":"opened","number":764,"pull_request":{"url":"https://api.github.com/repos/DigitalPebble/storm-crawler/pulls/764","id":325764612,"node_id":"MDExOlB1bGxSZXF1ZXN0MzI1NzY0NjEy","html_url":"https://github.com/DigitalPebble/storm-crawler/pull/764","diff_url":"https://github.com/DigitalPebble/storm-crawler/pull/764.diff","patch_url":"https://github.com/DigitalPebble/storm-crawler/pull/764.patch","issue_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/issues/764","number":764,"state":"open","locked":false,"title":"urlbuffer code in a separate package + 2 new implementations","user":{"login":"jnioche","id":218319,"node_id":"MDQ6VXNlcjIxODMxOQ==","avatar_url":"https://avatars1.githubusercontent.com/u/218319?v=4","gravatar_id":"","url":"https://api.github.com/users/jnioche","html_url":"https://github.com/jnioche","followers_url":"https://api.github.com/users/jnioche/followers","following_url":"https://api.github.com/users/jnioche/following{/other_user}","gists_url":"https://api.github.com/users/jnioche/gists{/gist_id}","starred_url":"https://api.github.com/users/jnioche/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jnioche/subscriptions","organizations_url":"https://api.github.com/users/jnioche/orgs","repos_url":"https://api.github.com/users/jnioche/repos","events_url":"https://api.github.com/users/jnioche/events{/privacy}","received_events_url":"https://api.github.com/users/jnioche/received_events","type":"User","site_admin":false},"body":"The brand new  **PriorityURLBuffer** sorts the buckets by the number of acks they got since the last sort whereas the **SchedulingURLBuffer** tries to guess when a queue should release a URL based on how long it took its previous URLs to be acked on average.\r\n\r\nIn the context of Elasticsearch, these implementations work fine with the AggregationSpout but make more sense with the HybridSpout.","created_at":"2019-10-08T13:11:04Z","updated_at":"2019-10-08T13:11:04Z","closed_at":null,"merged_at":null,"merge_commit_sha":null,"assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[{"id":171289696,"node_id":"MDU6TGFiZWwxNzEyODk2OTY=","url":"https://api.github.com/repos/DigitalPebble/storm-crawler/labels/core","name":"core","color":"bfdadc","default":false}],"milestone":{"url":"https://api.github.com/repos/DigitalPebble/storm-crawler/milestones/26","html_url":"https://github.com/DigitalPebble/storm-crawler/milestone/26","labels_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/milestones/26/labels","id":4673619,"node_id":"MDk6TWlsZXN0b25lNDY3MzYxOQ==","number":26,"title":"1.16","description":"","creator":{"login":"jnioche","id":218319,"node_id":"MDQ6VXNlcjIxODMxOQ==","avatar_url":"https://avatars1.githubusercontent.com/u/218319?v=4","gravatar_id":"","url":"https://api.github.com/users/jnioche","html_url":"https://github.com/jnioche","followers_url":"https://api.github.com/users/jnioche/followers","following_url":"https://api.github.com/users/jnioche/following{/other_user}","gists_url":"https://api.github.com/users/jnioche/gists{/gist_id}","starred_url":"https://api.github.com/users/jnioche/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jnioche/subscriptions","organizations_url":"https://api.github.com/users/jnioche/orgs","repos_url":"https://api.github.com/users/jnioche/repos","events_url":"https://api.github.com/users/jnioche/events{/privacy}","received_events_url":"https://api.github.com/users/jnioche/received_events","type":"User","site_admin":false},"open_issues":2,"closed_issues":5,"state":"open","created_at":"2019-09-19T08:40:55Z","updated_at":"2019-10-08T13:11:04Z","due_on":"2019-12-31T08:00:00Z","closed_at":null},"commits_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/pulls/764/commits","review_comments_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/pulls/764/comments","review_comment_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/pulls/comments{/number}","comments_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/issues/764/comments","statuses_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/statuses/64909906ad95c1b6fc6ca9052a304f639902e3d3","head":{"label":"DigitalPebble:priorityBuffer","ref":"priorityBuffer","sha":"64909906ad95c1b6fc6ca9052a304f639902e3d3","user":{"login":"DigitalPebble","id":1726647,"node_id":"MDEyOk9yZ2FuaXphdGlvbjE3MjY2NDc=","avatar_url":"https://avatars3.githubusercontent.com/u/1726647?v=4","gravatar_id":"","url":"https://api.github.com/users/DigitalPebble","html_url":"https://github.com/DigitalPebble","followers_url":"https://api.github.com/users/DigitalPebble/followers","following_url":"https://api.github.com/users/DigitalPebble/following{/other_user}","gists_url":"https://api.github.com/users/DigitalPebble/gists{/gist_id}","starred_url":"https://api.github.com/users/DigitalPebble/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/DigitalPebble/subscriptions","organizations_url":"https://api.github.com/users/DigitalPebble/orgs","repos_url":"https://api.github.com/users/DigitalPebble/repos","events_url":"https://api.github.com/users/DigitalPebble/events{/privacy}","received_events_url":"https://api.github.com/users/DigitalPebble/received_events","type":"Organization","site_admin":false},"repo":{"id":9395761,"node_id":"MDEwOlJlcG9zaXRvcnk5Mzk1NzYx","name":"storm-crawler","full_name":"DigitalPebble/storm-crawler","private":false,"owner":{"login":"DigitalPebble","id":1726647,"node_id":"MDEyOk9yZ2FuaXphdGlvbjE3MjY2NDc=","avatar_url":"https://avatars3.githubusercontent.com/u/1726647?v=4","gravatar_id":"","url":"https://api.github.com/users/DigitalPebble","html_url":"https://github.com/DigitalPebble","followers_url":"https://api.github.com/users/DigitalPebble/followers","following_url":"https://api.github.com/users/DigitalPebble/following{/other_user}","gists_url":"https://api.github.com/users/DigitalPebble/gists{/gist_id}","starred_url":"https://api.github.com/users/DigitalPebble/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/DigitalPebble/subscriptions","organizations_url":"https://api.github.com/users/DigitalPebble/orgs","repos_url":"https://api.github.com/users/DigitalPebble/repos","events_url":"https://api.github.com/users/DigitalPebble/events{/privacy}","received_events_url":"https://api.github.com/users/DigitalPebble/received_events","type":"Organization","site_admin":false},"html_url":"https://github.com/DigitalPebble/storm-crawler","description":"Scalable web crawler based on Apache Storm ","fork":false,"url":"https://api.github.com/repos/DigitalPebble/storm-crawler","forks_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/forks","keys_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/keys{/key_id}","collaborators_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/teams","hooks_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/hooks","issue_events_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/issues/events{/number}","events_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/events","assignees_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/assignees{/user}","branches_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/branches{/branch}","tags_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/tags","blobs_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/git/refs{/sha}","trees_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/git/trees{/sha}","statuses_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/statuses/{sha}","languages_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/languages","stargazers_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/stargazers","contributors_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/contributors","subscribers_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/subscribers","subscription_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/subscription","commits_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/commits{/sha}","git_commits_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/git/commits{/sha}","comments_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/comments{/number}","issue_comment_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/issues/comments{/number}","contents_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/contents/{+path}","compare_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/compare/{base}...{head}","merges_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/merges","archive_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/downloads","issues_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/issues{/number}","pulls_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/pulls{/number}","milestones_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/milestones{/number}","notifications_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/labels{/name}","releases_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/releases{/id}","deployments_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/deployments","created_at":"2013-04-12T14:13:30Z","updated_at":"2019-10-08T01:14:32Z","pushed_at":"2019-10-08T08:01:52Z","git_url":"git://github.com/DigitalPebble/storm-crawler.git","ssh_url":"git@github.com:DigitalPebble/storm-crawler.git","clone_url":"https://github.com/DigitalPebble/storm-crawler.git","svn_url":"https://github.com/DigitalPebble/storm-crawler","homepage":"http://stormcrawler.net","size":6737,"stargazers_count":584,"watchers_count":584,"language":"Java","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":true,"forks_count":194,"mirror_url":null,"archived":false,"disabled":false,"open_issues_count":37,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0","node_id":"MDc6TGljZW5zZTI="},"forks":194,"open_issues":37,"watchers":584,"default_branch":"master"}},"base":{"label":"DigitalPebble:master","ref":"master","sha":"3ad8f62721d2e4660a34c43337ae969d5b1ed8a7","user":{"login":"DigitalPebble","id":1726647,"node_id":"MDEyOk9yZ2FuaXphdGlvbjE3MjY2NDc=","avatar_url":"https://avatars3.githubusercontent.com/u/1726647?v=4","gravatar_id":"","url":"https://api.github.com/users/DigitalPebble","html_url":"https://github.com/DigitalPebble","followers_url":"https://api.github.com/users/DigitalPebble/followers","following_url":"https://api.github.com/users/DigitalPebble/following{/other_user}","gists_url":"https://api.github.com/users/DigitalPebble/gists{/gist_id}","starred_url":"https://api.github.com/users/DigitalPebble/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/DigitalPebble/subscriptions","organizations_url":"https://api.github.com/users/DigitalPebble/orgs","repos_url":"https://api.github.com/users/DigitalPebble/repos","events_url":"https://api.github.com/users/DigitalPebble/events{/privacy}","received_events_url":"https://api.github.com/users/DigitalPebble/received_events","type":"Organization","site_admin":false},"repo":{"id":9395761,"node_id":"MDEwOlJlcG9zaXRvcnk5Mzk1NzYx","name":"storm-crawler","full_name":"DigitalPebble/storm-crawler","private":false,"owner":{"login":"DigitalPebble","id":1726647,"node_id":"MDEyOk9yZ2FuaXphdGlvbjE3MjY2NDc=","avatar_url":"https://avatars3.githubusercontent.com/u/1726647?v=4","gravatar_id":"","url":"https://api.github.com/users/DigitalPebble","html_url":"https://github.com/DigitalPebble","followers_url":"https://api.github.com/users/DigitalPebble/followers","following_url":"https://api.github.com/users/DigitalPebble/following{/other_user}","gists_url":"https://api.github.com/users/DigitalPebble/gists{/gist_id}","starred_url":"https://api.github.com/users/DigitalPebble/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/DigitalPebble/subscriptions","organizations_url":"https://api.github.com/users/DigitalPebble/orgs","repos_url":"https://api.github.com/users/DigitalPebble/repos","events_url":"https://api.github.com/users/DigitalPebble/events{/privacy}","received_events_url":"https://api.github.com/users/DigitalPebble/received_events","type":"Organization","site_admin":false},"html_url":"https://github.com/DigitalPebble/storm-crawler","description":"Scalable web crawler based on Apache Storm ","fork":false,"url":"https://api.github.com/repos/DigitalPebble/storm-crawler","forks_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/forks","keys_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/keys{/key_id}","collaborators_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/teams","hooks_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/hooks","issue_events_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/issues/events{/number}","events_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/events","assignees_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/assignees{/user}","branches_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/branches{/branch}","tags_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/tags","blobs_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/git/refs{/sha}","trees_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/git/trees{/sha}","statuses_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/statuses/{sha}","languages_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/languages","stargazers_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/stargazers","contributors_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/contributors","subscribers_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/subscribers","subscription_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/subscription","commits_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/commits{/sha}","git_commits_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/git/commits{/sha}","comments_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/comments{/number}","issue_comment_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/issues/comments{/number}","contents_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/contents/{+path}","compare_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/compare/{base}...{head}","merges_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/merges","archive_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/downloads","issues_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/issues{/number}","pulls_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/pulls{/number}","milestones_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/milestones{/number}","notifications_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/labels{/name}","releases_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/releases{/id}","deployments_url":"https://api.github.com/repos/DigitalPebble/storm-crawler/deployments","created_at":"2013-04-12T14:13:30Z","updated_at":"2019-10-08T01:14:32Z","pushed_at":"2019-10-08T08:01:52Z","git_url":"git://github.com/DigitalPebble/storm-crawler.git","ssh_url":"git@github.com:DigitalPebble/storm-crawler.git","clone_url":"https://github.com/DigitalPebble/storm-crawler.git","svn_url":"https://github.com/DigitalPebble/storm-crawler","homepage":"http://stormcrawler.net","size":6737,"stargazers_count":584,"watchers_count":584,"language":"Java","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":true,"forks_count":194,"mirror_url":null,"archived":false,"disabled":false,"open_issues_count":37,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0","node_id":"MDc6TGljZW5zZTI="},"forks":194,"open_issues":37,"watchers":584,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/DigitalPebble/storm-crawler/pulls/764"},"html":{"href":"https://github.com/DigitalPebble/storm-crawler/pull/764"},"issue":{"href":"https://api.github.com/repos/DigitalPebble/storm-crawler/issues/764"},"comments":{"href":"https://api.github.com/repos/DigitalPebble/storm-crawler/issues/764/comments"},"review_comments":{"href":"https://api.github.com/repos/DigitalPebble/storm-crawler/pulls/764/comments"},"review_comment":{"href":"https://api.github.com/repos/DigitalPebble/storm-crawler/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/DigitalPebble/storm-crawler/pulls/764/commits"},"statuses":{"href":"https://api.github.com/repos/DigitalPebble/storm-crawler/statuses/64909906ad95c1b6fc6ca9052a304f639902e3d3"}},"author_association":"MEMBER","merged":false,"mergeable":null,"rebaseable":null,"mergeable_state":"unknown","merged_by":null,"comments":0,"review_comments":0,"maintainer_can_modify":false,"commits":8,"additions":454,"deletions":66,"changed_files":8}}
{"actor":{"display_login":"jnioche"}}