{"action":"created","comment":{"url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/pulls/comments/169824868","pull_request_review_id":98414701,"id":169824868,"diff_hunk":"@@ -165,7 +165,7 @@ public void stop() throws Exception {\n         private BaseHttpFetcherBuilder _robotsFetcherBuilder = new SimpleHttpFetcherBuilder(INVALID_USER_AGENT);\n         private SimpleRobotRulesParser _robotsParser = new SimpleRobotRulesParser();\n \n-        private BaseUrlLengthener _urlLengthener = new SimpleUrlLengthener();\n+        private BaseUrlLengthener _urlLengthener = new SimpleUrlLengthener(new SimpleHttpFetcherBuilder(INVALID_USER_AGENT));","path":"src/main/java/com/scaleunlimited/flinkcrawler/tools/CrawlTopology.java","position":5,"original_position":5,"commit_id":"250bfcddf3c518c6fea0b313467adc6088d3dc51","original_commit_id":"250bfcddf3c518c6fea0b313467adc6088d3dc51","user":{"login":"kkrugler","id":66217,"avatar_url":"https://avatars3.githubusercontent.com/u/66217?v=4","gravatar_id":"","url":"https://api.github.com/users/kkrugler","html_url":"https://github.com/kkrugler","followers_url":"https://api.github.com/users/kkrugler/followers","following_url":"https://api.github.com/users/kkrugler/following{/other_user}","gists_url":"https://api.github.com/users/kkrugler/gists{/gist_id}","starred_url":"https://api.github.com/users/kkrugler/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/kkrugler/subscriptions","organizations_url":"https://api.github.com/users/kkrugler/orgs","repos_url":"https://api.github.com/users/kkrugler/repos","events_url":"https://api.github.com/users/kkrugler/events{/privacy}","received_events_url":"https://api.github.com/users/kkrugler/received_events","type":"User","site_admin":false},"body":"How does the user agent get changed if the user specifies it?","created_at":"2018-02-22T00:41:01Z","updated_at":"2018-02-22T00:41:01Z","html_url":"https://github.com/ScaleUnlimited/flink-crawler/pull/94#discussion_r169824868","pull_request_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/pulls/94","author_association":"OWNER","_links":{"self":{"href":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/pulls/comments/169824868"},"html":{"href":"https://github.com/ScaleUnlimited/flink-crawler/pull/94#discussion_r169824868"},"pull_request":{"href":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/pulls/94"}}},"pull_request":{"url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/pulls/94","id":170628000,"html_url":"https://github.com/ScaleUnlimited/flink-crawler/pull/94","diff_url":"https://github.com/ScaleUnlimited/flink-crawler/pull/94.diff","patch_url":"https://github.com/ScaleUnlimited/flink-crawler/pull/94.patch","issue_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/issues/94","number":94,"state":"open","locked":false,"title":"50 url lengthener","user":{"login":"Schmed","id":183408,"avatar_url":"https://avatars0.githubusercontent.com/u/183408?v=4","gravatar_id":"","url":"https://api.github.com/users/Schmed","html_url":"https://github.com/Schmed","followers_url":"https://api.github.com/users/Schmed/followers","following_url":"https://api.github.com/users/Schmed/following{/other_user}","gists_url":"https://api.github.com/users/Schmed/gists{/gist_id}","starred_url":"https://api.github.com/users/Schmed/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Schmed/subscriptions","organizations_url":"https://api.github.com/users/Schmed/orgs","repos_url":"https://api.github.com/users/Schmed/repos","events_url":"https://api.github.com/users/Schmed/events{/privacy}","received_events_url":"https://api.github.com/users/Schmed/received_events","type":"User","site_admin":false},"body":"I stole most of the core implementation from Bixo's UrlLengthener, but I wrote all the test code.","created_at":"2018-02-22T00:37:43Z","updated_at":"2018-02-22T00:41:01Z","closed_at":null,"merged_at":null,"merge_commit_sha":"5d451d25aee06fbe6a2803bfb955ef115f78e6c0","assignee":{"login":"Schmed","id":183408,"avatar_url":"https://avatars0.githubusercontent.com/u/183408?v=4","gravatar_id":"","url":"https://api.github.com/users/Schmed","html_url":"https://github.com/Schmed","followers_url":"https://api.github.com/users/Schmed/followers","following_url":"https://api.github.com/users/Schmed/following{/other_user}","gists_url":"https://api.github.com/users/Schmed/gists{/gist_id}","starred_url":"https://api.github.com/users/Schmed/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Schmed/subscriptions","organizations_url":"https://api.github.com/users/Schmed/orgs","repos_url":"https://api.github.com/users/Schmed/repos","events_url":"https://api.github.com/users/Schmed/events{/privacy}","received_events_url":"https://api.github.com/users/Schmed/received_events","type":"User","site_admin":false},"assignees":[{"login":"Schmed","id":183408,"avatar_url":"https://avatars0.githubusercontent.com/u/183408?v=4","gravatar_id":"","url":"https://api.github.com/users/Schmed","html_url":"https://github.com/Schmed","followers_url":"https://api.github.com/users/Schmed/followers","following_url":"https://api.github.com/users/Schmed/following{/other_user}","gists_url":"https://api.github.com/users/Schmed/gists{/gist_id}","starred_url":"https://api.github.com/users/Schmed/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Schmed/subscriptions","organizations_url":"https://api.github.com/users/Schmed/orgs","repos_url":"https://api.github.com/users/Schmed/repos","events_url":"https://api.github.com/users/Schmed/events{/privacy}","received_events_url":"https://api.github.com/users/Schmed/received_events","type":"User","site_admin":false}],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/pulls/94/commits","review_comments_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/pulls/94/comments","review_comment_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/pulls/comments{/number}","comments_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/issues/94/comments","statuses_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/statuses/250bfcddf3c518c6fea0b313467adc6088d3dc51","head":{"label":"ScaleUnlimited:50_url-lengthener","ref":"50_url-lengthener","sha":"250bfcddf3c518c6fea0b313467adc6088d3dc51","user":{"login":"ScaleUnlimited","id":1437539,"avatar_url":"https://avatars1.githubusercontent.com/u/1437539?v=4","gravatar_id":"","url":"https://api.github.com/users/ScaleUnlimited","html_url":"https://github.com/ScaleUnlimited","followers_url":"https://api.github.com/users/ScaleUnlimited/followers","following_url":"https://api.github.com/users/ScaleUnlimited/following{/other_user}","gists_url":"https://api.github.com/users/ScaleUnlimited/gists{/gist_id}","starred_url":"https://api.github.com/users/ScaleUnlimited/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ScaleUnlimited/subscriptions","organizations_url":"https://api.github.com/users/ScaleUnlimited/orgs","repos_url":"https://api.github.com/users/ScaleUnlimited/repos","events_url":"https://api.github.com/users/ScaleUnlimited/events{/privacy}","received_events_url":"https://api.github.com/users/ScaleUnlimited/received_events","type":"Organization","site_admin":false},"repo":{"id":66723242,"name":"flink-crawler","full_name":"ScaleUnlimited/flink-crawler","owner":{"login":"ScaleUnlimited","id":1437539,"avatar_url":"https://avatars1.githubusercontent.com/u/1437539?v=4","gravatar_id":"","url":"https://api.github.com/users/ScaleUnlimited","html_url":"https://github.com/ScaleUnlimited","followers_url":"https://api.github.com/users/ScaleUnlimited/followers","following_url":"https://api.github.com/users/ScaleUnlimited/following{/other_user}","gists_url":"https://api.github.com/users/ScaleUnlimited/gists{/gist_id}","starred_url":"https://api.github.com/users/ScaleUnlimited/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ScaleUnlimited/subscriptions","organizations_url":"https://api.github.com/users/ScaleUnlimited/orgs","repos_url":"https://api.github.com/users/ScaleUnlimited/repos","events_url":"https://api.github.com/users/ScaleUnlimited/events{/privacy}","received_events_url":"https://api.github.com/users/ScaleUnlimited/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/ScaleUnlimited/flink-crawler","description":"Continuous scalable web crawler built on top of Flink and crawler-commons","fork":false,"url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler","forks_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/forks","keys_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/keys{/key_id}","collaborators_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/teams","hooks_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/hooks","issue_events_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/issues/events{/number}","events_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/events","assignees_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/assignees{/user}","branches_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/branches{/branch}","tags_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/tags","blobs_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/git/refs{/sha}","trees_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/git/trees{/sha}","statuses_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/statuses/{sha}","languages_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/languages","stargazers_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/stargazers","contributors_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/contributors","subscribers_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/subscribers","subscription_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/subscription","commits_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/commits{/sha}","git_commits_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/git/commits{/sha}","comments_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/comments{/number}","issue_comment_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/issues/comments{/number}","contents_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/contents/{+path}","compare_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/compare/{base}...{head}","merges_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/merges","archive_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/downloads","issues_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/issues{/number}","pulls_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/pulls{/number}","milestones_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/milestones{/number}","notifications_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/labels{/name}","releases_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/releases{/id}","deployments_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/deployments","created_at":"2016-08-27T16:07:59Z","updated_at":"2018-02-19T10:04:03Z","pushed_at":"2018-02-22T00:37:43Z","git_url":"git://github.com/ScaleUnlimited/flink-crawler.git","ssh_url":"git@github.com:ScaleUnlimited/flink-crawler.git","clone_url":"https://github.com/ScaleUnlimited/flink-crawler.git","svn_url":"https://github.com/ScaleUnlimited/flink-crawler","homepage":null,"size":1123,"stargazers_count":1,"watchers_count":1,"language":"Java","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":30,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0"},"forks":0,"open_issues":30,"watchers":1,"default_branch":"master"}},"base":{"label":"ScaleUnlimited:master","ref":"master","sha":"3ca6e383576d7445be6b3681a01b7ae7d771510a","user":{"login":"ScaleUnlimited","id":1437539,"avatar_url":"https://avatars1.githubusercontent.com/u/1437539?v=4","gravatar_id":"","url":"https://api.github.com/users/ScaleUnlimited","html_url":"https://github.com/ScaleUnlimited","followers_url":"https://api.github.com/users/ScaleUnlimited/followers","following_url":"https://api.github.com/users/ScaleUnlimited/following{/other_user}","gists_url":"https://api.github.com/users/ScaleUnlimited/gists{/gist_id}","starred_url":"https://api.github.com/users/ScaleUnlimited/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ScaleUnlimited/subscriptions","organizations_url":"https://api.github.com/users/ScaleUnlimited/orgs","repos_url":"https://api.github.com/users/ScaleUnlimited/repos","events_url":"https://api.github.com/users/ScaleUnlimited/events{/privacy}","received_events_url":"https://api.github.com/users/ScaleUnlimited/received_events","type":"Organization","site_admin":false},"repo":{"id":66723242,"name":"flink-crawler","full_name":"ScaleUnlimited/flink-crawler","owner":{"login":"ScaleUnlimited","id":1437539,"avatar_url":"https://avatars1.githubusercontent.com/u/1437539?v=4","gravatar_id":"","url":"https://api.github.com/users/ScaleUnlimited","html_url":"https://github.com/ScaleUnlimited","followers_url":"https://api.github.com/users/ScaleUnlimited/followers","following_url":"https://api.github.com/users/ScaleUnlimited/following{/other_user}","gists_url":"https://api.github.com/users/ScaleUnlimited/gists{/gist_id}","starred_url":"https://api.github.com/users/ScaleUnlimited/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ScaleUnlimited/subscriptions","organizations_url":"https://api.github.com/users/ScaleUnlimited/orgs","repos_url":"https://api.github.com/users/ScaleUnlimited/repos","events_url":"https://api.github.com/users/ScaleUnlimited/events{/privacy}","received_events_url":"https://api.github.com/users/ScaleUnlimited/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/ScaleUnlimited/flink-crawler","description":"Continuous scalable web crawler built on top of Flink and crawler-commons","fork":false,"url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler","forks_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/forks","keys_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/keys{/key_id}","collaborators_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/teams","hooks_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/hooks","issue_events_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/issues/events{/number}","events_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/events","assignees_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/assignees{/user}","branches_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/branches{/branch}","tags_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/tags","blobs_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/git/refs{/sha}","trees_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/git/trees{/sha}","statuses_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/statuses/{sha}","languages_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/languages","stargazers_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/stargazers","contributors_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/contributors","subscribers_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/subscribers","subscription_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/subscription","commits_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/commits{/sha}","git_commits_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/git/commits{/sha}","comments_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/comments{/number}","issue_comment_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/issues/comments{/number}","contents_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/contents/{+path}","compare_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/compare/{base}...{head}","merges_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/merges","archive_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/downloads","issues_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/issues{/number}","pulls_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/pulls{/number}","milestones_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/milestones{/number}","notifications_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/labels{/name}","releases_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/releases{/id}","deployments_url":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/deployments","created_at":"2016-08-27T16:07:59Z","updated_at":"2018-02-19T10:04:03Z","pushed_at":"2018-02-22T00:37:43Z","git_url":"git://github.com/ScaleUnlimited/flink-crawler.git","ssh_url":"git@github.com:ScaleUnlimited/flink-crawler.git","clone_url":"https://github.com/ScaleUnlimited/flink-crawler.git","svn_url":"https://github.com/ScaleUnlimited/flink-crawler","homepage":null,"size":1123,"stargazers_count":1,"watchers_count":1,"language":"Java","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":30,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0"},"forks":0,"open_issues":30,"watchers":1,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/pulls/94"},"html":{"href":"https://github.com/ScaleUnlimited/flink-crawler/pull/94"},"issue":{"href":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/issues/94"},"comments":{"href":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/issues/94/comments"},"review_comments":{"href":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/pulls/94/comments"},"review_comment":{"href":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/pulls/94/commits"},"statuses":{"href":"https://api.github.com/repos/ScaleUnlimited/flink-crawler/statuses/250bfcddf3c518c6fea0b313467adc6088d3dc51"}},"author_association":"MEMBER"}}
{"actor":{"display_login":"kkrugler"}}