{"action":"created","comment":{"url":"https://api.github.com/repos/alphagov/service-manual-publisher/pulls/comments/56647528","id":56647528,"diff_hunk":"@@ -4,44 +4,63 @@ def initialize govspeak\n   end\n \n   def find_broken_urls\n-    extract_all_urls_from_govspeak.select do |url|\n-      checked_url = CheckedUrl.find_or_initialize_by(url: url)\n-      if checked_url.new_record? || checked_url.expired?\n-        checked_url.ok = check_url(url)\n-        checked_url.save!\n-      end\n+    urls_to_check = extract_all_urls_from_govspeak.select do |url|\n+      url = CheckedUrl.find_or_initialize_by(url: url)\n+      url.new_record? || url.expired?\n+    end\n \n-      !checked_url.ok?\n+    if urls_to_check.any?\n+      broken_urls = check_urls(urls_to_check)\n+      urls_to_check.each do |url|\n+        CheckedUrl.find_or_initialize_by(url: url) do |checked_url|\n+          checked_url.ok = !broken_urls.include?(checked_url.url)\n+        end.save!\n+      end\n+      broken_urls\n+    else\n+      []\n     end\n   end\n \n   private\n \n-    def check_url url\n+  def check_urls urls\n+    broken_urls = []\n+    hydra = Typhoeus::Hydra.new(max_concurrency: 5)\n+\n+    urls.each do |url|\n       options = {\n-        follow_redirects: true,\n-        timeout: 5,\n+        followlocation: true,\n+        forbid_reuse: true,\n+        timeout: 10,\n+        connecttimeout: 10,\n       }\n \n       if url.starts_with? \"/\"\n         url = \"#{Plek.new.website_root}#{url}\"\n         if ENV[\"HTTP_USERNAME\"] && ENV[\"HTTP_PASSWORD\"]\n-          options[:basic_auth] = {\n-            username: ENV[\"HTTP_USERNAME\"],\n-            password: ENV[\"HTTP_PASSWORD\"],\n-          }\n+          options[:userpwd] = \"#{ENV[\"HTTP_USERNAME\"]}:#{ENV[\"HTTP_PASSWORD\"]}\"\n         end\n       end\n \n       begin\n-        HTTParty.get(\n-          url,\n-          options,\n-        ).ok?\n-      rescue\n-        false\n+        Addressable::URI.parse(url)","path":"app/models/govspeak_url_checker.rb","position":64,"original_position":64,"commit_id":"20a62bc2010cc617164843bcaeb638c52a3b74e8","original_commit_id":"20a62bc2010cc617164843bcaeb638c52a3b74e8","user":{"login":"thehenster","id":78311,"avatar_url":"https://avatars.githubusercontent.com/u/78311?v=3","gravatar_id":"","url":"https://api.github.com/users/thehenster","html_url":"https://github.com/thehenster","followers_url":"https://api.github.com/users/thehenster/followers","following_url":"https://api.github.com/users/thehenster/following{/other_user}","gists_url":"https://api.github.com/users/thehenster/gists{/gist_id}","starred_url":"https://api.github.com/users/thehenster/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/thehenster/subscriptions","organizations_url":"https://api.github.com/users/thehenster/orgs","repos_url":"https://api.github.com/users/thehenster/repos","events_url":"https://api.github.com/users/thehenster/events{/privacy}","received_events_url":"https://api.github.com/users/thehenster/received_events","type":"User","site_admin":false},"body":"Could we add addressable to the gemfile if we're gonna use it over normal URI? It looks like it's just a fortunate dependency of other gems atm.","created_at":"2016-03-18T12:17:17Z","updated_at":"2016-03-18T12:17:17Z","html_url":"https://github.com/alphagov/service-manual-publisher/pull/192#discussion_r56647528","pull_request_url":"https://api.github.com/repos/alphagov/service-manual-publisher/pulls/192","_links":{"self":{"href":"https://api.github.com/repos/alphagov/service-manual-publisher/pulls/comments/56647528"},"html":{"href":"https://github.com/alphagov/service-manual-publisher/pull/192#discussion_r56647528"},"pull_request":{"href":"https://api.github.com/repos/alphagov/service-manual-publisher/pulls/192"}}},"pull_request":{"url":"https://api.github.com/repos/alphagov/service-manual-publisher/pulls/192","id":63239406,"html_url":"https://github.com/alphagov/service-manual-publisher/pull/192","diff_url":"https://github.com/alphagov/service-manual-publisher/pull/192.diff","patch_url":"https://github.com/alphagov/service-manual-publisher/pull/192.patch","issue_url":"https://api.github.com/repos/alphagov/service-manual-publisher/issues/192","number":192,"state":"open","locked":false,"title":"Validate urls in parallel","user":{"login":"AndrewVos","id":363618,"avatar_url":"https://avatars.githubusercontent.com/u/363618?v=3","gravatar_id":"","url":"https://api.github.com/users/AndrewVos","html_url":"https://github.com/AndrewVos","followers_url":"https://api.github.com/users/AndrewVos/followers","following_url":"https://api.github.com/users/AndrewVos/following{/other_user}","gists_url":"https://api.github.com/users/AndrewVos/gists{/gist_id}","starred_url":"https://api.github.com/users/AndrewVos/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/AndrewVos/subscriptions","organizations_url":"https://api.github.com/users/AndrewVos/orgs","repos_url":"https://api.github.com/users/AndrewVos/repos","events_url":"https://api.github.com/users/AndrewVos/events{/privacy}","received_events_url":"https://api.github.com/users/AndrewVos/received_events","type":"User","site_admin":false},"body":"We are getting timeouts in integration if all the urls are requested\nsynchronously.","created_at":"2016-03-17T15:17:48Z","updated_at":"2016-03-18T12:17:17Z","closed_at":null,"merged_at":null,"merge_commit_sha":"ca5270386c56f90c021eec1dcb8efc9f47f32ba9","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/alphagov/service-manual-publisher/pulls/192/commits","review_comments_url":"https://api.github.com/repos/alphagov/service-manual-publisher/pulls/192/comments","review_comment_url":"https://api.github.com/repos/alphagov/service-manual-publisher/pulls/comments{/number}","comments_url":"https://api.github.com/repos/alphagov/service-manual-publisher/issues/192/comments","statuses_url":"https://api.github.com/repos/alphagov/service-manual-publisher/statuses/20a62bc2010cc617164843bcaeb638c52a3b74e8","head":{"label":"alphagov:parallel-url-validation","ref":"parallel-url-validation","sha":"20a62bc2010cc617164843bcaeb638c52a3b74e8","user":{"login":"alphagov","id":596977,"avatar_url":"https://avatars.githubusercontent.com/u/596977?v=3","gravatar_id":"","url":"https://api.github.com/users/alphagov","html_url":"https://github.com/alphagov","followers_url":"https://api.github.com/users/alphagov/followers","following_url":"https://api.github.com/users/alphagov/following{/other_user}","gists_url":"https://api.github.com/users/alphagov/gists{/gist_id}","starred_url":"https://api.github.com/users/alphagov/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/alphagov/subscriptions","organizations_url":"https://api.github.com/users/alphagov/orgs","repos_url":"https://api.github.com/users/alphagov/repos","events_url":"https://api.github.com/users/alphagov/events{/privacy}","received_events_url":"https://api.github.com/users/alphagov/received_events","type":"Organization","site_admin":false},"repo":{"id":43757023,"name":"service-manual-publisher","full_name":"alphagov/service-manual-publisher","owner":{"login":"alphagov","id":596977,"avatar_url":"https://avatars.githubusercontent.com/u/596977?v=3","gravatar_id":"","url":"https://api.github.com/users/alphagov","html_url":"https://github.com/alphagov","followers_url":"https://api.github.com/users/alphagov/followers","following_url":"https://api.github.com/users/alphagov/following{/other_user}","gists_url":"https://api.github.com/users/alphagov/gists{/gist_id}","starred_url":"https://api.github.com/users/alphagov/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/alphagov/subscriptions","organizations_url":"https://api.github.com/users/alphagov/orgs","repos_url":"https://api.github.com/users/alphagov/repos","events_url":"https://api.github.com/users/alphagov/events{/privacy}","received_events_url":"https://api.github.com/users/alphagov/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/alphagov/service-manual-publisher","description":"Create and manage content for the service manual","fork":false,"url":"https://api.github.com/repos/alphagov/service-manual-publisher","forks_url":"https://api.github.com/repos/alphagov/service-manual-publisher/forks","keys_url":"https://api.github.com/repos/alphagov/service-manual-publisher/keys{/key_id}","collaborators_url":"https://api.github.com/repos/alphagov/service-manual-publisher/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/alphagov/service-manual-publisher/teams","hooks_url":"https://api.github.com/repos/alphagov/service-manual-publisher/hooks","issue_events_url":"https://api.github.com/repos/alphagov/service-manual-publisher/issues/events{/number}","events_url":"https://api.github.com/repos/alphagov/service-manual-publisher/events","assignees_url":"https://api.github.com/repos/alphagov/service-manual-publisher/assignees{/user}","branches_url":"https://api.github.com/repos/alphagov/service-manual-publisher/branches{/branch}","tags_url":"https://api.github.com/repos/alphagov/service-manual-publisher/tags","blobs_url":"https://api.github.com/repos/alphagov/service-manual-publisher/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/alphagov/service-manual-publisher/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/alphagov/service-manual-publisher/git/refs{/sha}","trees_url":"https://api.github.com/repos/alphagov/service-manual-publisher/git/trees{/sha}","statuses_url":"https://api.github.com/repos/alphagov/service-manual-publisher/statuses/{sha}","languages_url":"https://api.github.com/repos/alphagov/service-manual-publisher/languages","stargazers_url":"https://api.github.com/repos/alphagov/service-manual-publisher/stargazers","contributors_url":"https://api.github.com/repos/alphagov/service-manual-publisher/contributors","subscribers_url":"https://api.github.com/repos/alphagov/service-manual-publisher/subscribers","subscription_url":"https://api.github.com/repos/alphagov/service-manual-publisher/subscription","commits_url":"https://api.github.com/repos/alphagov/service-manual-publisher/commits{/sha}","git_commits_url":"https://api.github.com/repos/alphagov/service-manual-publisher/git/commits{/sha}","comments_url":"https://api.github.com/repos/alphagov/service-manual-publisher/comments{/number}","issue_comment_url":"https://api.github.com/repos/alphagov/service-manual-publisher/issues/comments{/number}","contents_url":"https://api.github.com/repos/alphagov/service-manual-publisher/contents/{+path}","compare_url":"https://api.github.com/repos/alphagov/service-manual-publisher/compare/{base}...{head}","merges_url":"https://api.github.com/repos/alphagov/service-manual-publisher/merges","archive_url":"https://api.github.com/repos/alphagov/service-manual-publisher/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/alphagov/service-manual-publisher/downloads","issues_url":"https://api.github.com/repos/alphagov/service-manual-publisher/issues{/number}","pulls_url":"https://api.github.com/repos/alphagov/service-manual-publisher/pulls{/number}","milestones_url":"https://api.github.com/repos/alphagov/service-manual-publisher/milestones{/number}","notifications_url":"https://api.github.com/repos/alphagov/service-manual-publisher/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/alphagov/service-manual-publisher/labels{/name}","releases_url":"https://api.github.com/repos/alphagov/service-manual-publisher/releases{/id}","deployments_url":"https://api.github.com/repos/alphagov/service-manual-publisher/deployments","created_at":"2015-10-06T14:45:26Z","updated_at":"2016-01-22T00:56:21Z","pushed_at":"2016-03-18T12:07:01Z","git_url":"git://github.com/alphagov/service-manual-publisher.git","ssh_url":"git@github.com:alphagov/service-manual-publisher.git","clone_url":"https://github.com/alphagov/service-manual-publisher.git","svn_url":"https://github.com/alphagov/service-manual-publisher","homepage":null,"size":803,"stargazers_count":1,"watchers_count":1,"language":"Ruby","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":2,"mirror_url":null,"open_issues_count":1,"forks":2,"open_issues":1,"watchers":1,"default_branch":"master"}},"base":{"label":"alphagov:master","ref":"master","sha":"e71bbaeec32c1d1df4a16b8f03877e723b70bb2b","user":{"login":"alphagov","id":596977,"avatar_url":"https://avatars.githubusercontent.com/u/596977?v=3","gravatar_id":"","url":"https://api.github.com/users/alphagov","html_url":"https://github.com/alphagov","followers_url":"https://api.github.com/users/alphagov/followers","following_url":"https://api.github.com/users/alphagov/following{/other_user}","gists_url":"https://api.github.com/users/alphagov/gists{/gist_id}","starred_url":"https://api.github.com/users/alphagov/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/alphagov/subscriptions","organizations_url":"https://api.github.com/users/alphagov/orgs","repos_url":"https://api.github.com/users/alphagov/repos","events_url":"https://api.github.com/users/alphagov/events{/privacy}","received_events_url":"https://api.github.com/users/alphagov/received_events","type":"Organization","site_admin":false},"repo":{"id":43757023,"name":"service-manual-publisher","full_name":"alphagov/service-manual-publisher","owner":{"login":"alphagov","id":596977,"avatar_url":"https://avatars.githubusercontent.com/u/596977?v=3","gravatar_id":"","url":"https://api.github.com/users/alphagov","html_url":"https://github.com/alphagov","followers_url":"https://api.github.com/users/alphagov/followers","following_url":"https://api.github.com/users/alphagov/following{/other_user}","gists_url":"https://api.github.com/users/alphagov/gists{/gist_id}","starred_url":"https://api.github.com/users/alphagov/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/alphagov/subscriptions","organizations_url":"https://api.github.com/users/alphagov/orgs","repos_url":"https://api.github.com/users/alphagov/repos","events_url":"https://api.github.com/users/alphagov/events{/privacy}","received_events_url":"https://api.github.com/users/alphagov/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/alphagov/service-manual-publisher","description":"Create and manage content for the service manual","fork":false,"url":"https://api.github.com/repos/alphagov/service-manual-publisher","forks_url":"https://api.github.com/repos/alphagov/service-manual-publisher/forks","keys_url":"https://api.github.com/repos/alphagov/service-manual-publisher/keys{/key_id}","collaborators_url":"https://api.github.com/repos/alphagov/service-manual-publisher/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/alphagov/service-manual-publisher/teams","hooks_url":"https://api.github.com/repos/alphagov/service-manual-publisher/hooks","issue_events_url":"https://api.github.com/repos/alphagov/service-manual-publisher/issues/events{/number}","events_url":"https://api.github.com/repos/alphagov/service-manual-publisher/events","assignees_url":"https://api.github.com/repos/alphagov/service-manual-publisher/assignees{/user}","branches_url":"https://api.github.com/repos/alphagov/service-manual-publisher/branches{/branch}","tags_url":"https://api.github.com/repos/alphagov/service-manual-publisher/tags","blobs_url":"https://api.github.com/repos/alphagov/service-manual-publisher/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/alphagov/service-manual-publisher/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/alphagov/service-manual-publisher/git/refs{/sha}","trees_url":"https://api.github.com/repos/alphagov/service-manual-publisher/git/trees{/sha}","statuses_url":"https://api.github.com/repos/alphagov/service-manual-publisher/statuses/{sha}","languages_url":"https://api.github.com/repos/alphagov/service-manual-publisher/languages","stargazers_url":"https://api.github.com/repos/alphagov/service-manual-publisher/stargazers","contributors_url":"https://api.github.com/repos/alphagov/service-manual-publisher/contributors","subscribers_url":"https://api.github.com/repos/alphagov/service-manual-publisher/subscribers","subscription_url":"https://api.github.com/repos/alphagov/service-manual-publisher/subscription","commits_url":"https://api.github.com/repos/alphagov/service-manual-publisher/commits{/sha}","git_commits_url":"https://api.github.com/repos/alphagov/service-manual-publisher/git/commits{/sha}","comments_url":"https://api.github.com/repos/alphagov/service-manual-publisher/comments{/number}","issue_comment_url":"https://api.github.com/repos/alphagov/service-manual-publisher/issues/comments{/number}","contents_url":"https://api.github.com/repos/alphagov/service-manual-publisher/contents/{+path}","compare_url":"https://api.github.com/repos/alphagov/service-manual-publisher/compare/{base}...{head}","merges_url":"https://api.github.com/repos/alphagov/service-manual-publisher/merges","archive_url":"https://api.github.com/repos/alphagov/service-manual-publisher/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/alphagov/service-manual-publisher/downloads","issues_url":"https://api.github.com/repos/alphagov/service-manual-publisher/issues{/number}","pulls_url":"https://api.github.com/repos/alphagov/service-manual-publisher/pulls{/number}","milestones_url":"https://api.github.com/repos/alphagov/service-manual-publisher/milestones{/number}","notifications_url":"https://api.github.com/repos/alphagov/service-manual-publisher/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/alphagov/service-manual-publisher/labels{/name}","releases_url":"https://api.github.com/repos/alphagov/service-manual-publisher/releases{/id}","deployments_url":"https://api.github.com/repos/alphagov/service-manual-publisher/deployments","created_at":"2015-10-06T14:45:26Z","updated_at":"2016-01-22T00:56:21Z","pushed_at":"2016-03-18T12:07:01Z","git_url":"git://github.com/alphagov/service-manual-publisher.git","ssh_url":"git@github.com:alphagov/service-manual-publisher.git","clone_url":"https://github.com/alphagov/service-manual-publisher.git","svn_url":"https://github.com/alphagov/service-manual-publisher","homepage":null,"size":803,"stargazers_count":1,"watchers_count":1,"language":"Ruby","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":2,"mirror_url":null,"open_issues_count":1,"forks":2,"open_issues":1,"watchers":1,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/alphagov/service-manual-publisher/pulls/192"},"html":{"href":"https://github.com/alphagov/service-manual-publisher/pull/192"},"issue":{"href":"https://api.github.com/repos/alphagov/service-manual-publisher/issues/192"},"comments":{"href":"https://api.github.com/repos/alphagov/service-manual-publisher/issues/192/comments"},"review_comments":{"href":"https://api.github.com/repos/alphagov/service-manual-publisher/pulls/192/comments"},"review_comment":{"href":"https://api.github.com/repos/alphagov/service-manual-publisher/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/alphagov/service-manual-publisher/pulls/192/commits"},"statuses":{"href":"https://api.github.com/repos/alphagov/service-manual-publisher/statuses/20a62bc2010cc617164843bcaeb638c52a3b74e8"}}}}