{"action":"created","comment":{"url":"https://api.github.com/repos/mozilla/prowac/pulls/comments/53733297","id":53733297,"diff_hunk":"@@ -1,36 +1,106 @@\n import { default as fetch } from 'node-fetch';\n+import { default as cheerio } from 'cheerio';\n+import { default as urlModule } from 'url';\n \n const probes = [];\n \n-function configure(/* opts */) {\n-  // TODO: Set up probes. They'll be specified somehow in opts and we\n-  // should be able to load each probe individually. Maybe something\n-  // like `require('./probe-' + probeName + '.js');`\n+function configure(opts) {\n+  if (!opts.probes || !opts.probes.length) {\n+    return Promise.reject(new Error('At least one urlJobProcessor probe must be specified'));\n+  }\n+\n+  const ret = [Promise.resolve()];\n+\n+  opts.probes.forEach((probeOpts) => {\n+    if (!probeOpts.name) {\n+      return Promise.reject(new Error('Probe without name encountered'));\n+    }\n+\n+    // We allow the probe to be specified by name or by path. If specified by name,\n+    // we expect it to be a known subdir.\n+    const path = probeOpts.path || `./urlJobProcessorProbes/${probeOpts.name}.js`;\n+    const probeModule = require(path).default;\n+\n+    if (!probeModule.runProbe) {\n+      return Promise.reject(new Error(`Probe ${probeOpts.name} does not implement runProbe`));\n+    }\n+\n+    let p = Promise.resolve();\n+    if (probeModule.configure) {\n+      p = probeModule.configure(probeOpts.opts || {});\n+    }\n+\n+    ret.push(p.then(() => {\n+      probes.push({\n+        name: probeOpts.name,\n+        probeFn: probeModule.runProbe,\n+      });\n+    }));\n+  });\n+\n+  return Promise.all(ret);","path":"crawler/urlJobProcessor.js","position":45,"original_position":45,"commit_id":"eb0e0a7ce923dff0afb3d4bb6be74561f5ae0cb9","original_commit_id":"158bba732c1599f13e6af540d09245082bc7a829","user":{"login":"TimAbraldes","id":955084,"avatar_url":"https://avatars.githubusercontent.com/u/955084?v=3","gravatar_id":"","url":"https://api.github.com/users/TimAbraldes","html_url":"https://github.com/TimAbraldes","followers_url":"https://api.github.com/users/TimAbraldes/followers","following_url":"https://api.github.com/users/TimAbraldes/following{/other_user}","gists_url":"https://api.github.com/users/TimAbraldes/gists{/gist_id}","starred_url":"https://api.github.com/users/TimAbraldes/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/TimAbraldes/subscriptions","organizations_url":"https://api.github.com/users/TimAbraldes/orgs","repos_url":"https://api.github.com/users/TimAbraldes/repos","events_url":"https://api.github.com/users/TimAbraldes/events{/privacy}","received_events_url":"https://api.github.com/users/TimAbraldes/received_events","type":"User","site_admin":false},"body":"@digitarald I removed your phone number since it didn't seem like you intended to include it. It's part of your email signature so it showed up in your github comment\r\n\r\n> If we get concurrency with kue, do we still need it on for the probes?\r\n\r\nI don't think we can really know the answer without taking some perf measurements. It probably depends on the types of probes we end up implementing. I don't imagine that it will make too much of a difference and it's a super simple change to make so I'm happy to make them run serially for now and see how things work out.","created_at":"2016-02-23T03:46:34Z","updated_at":"2016-02-23T03:46:34Z","html_url":"https://github.com/mozilla/prowac/pull/30#discussion_r53733297","pull_request_url":"https://api.github.com/repos/mozilla/prowac/pulls/30","_links":{"self":{"href":"https://api.github.com/repos/mozilla/prowac/pulls/comments/53733297"},"html":{"href":"https://github.com/mozilla/prowac/pull/30#discussion_r53733297"},"pull_request":{"href":"https://api.github.com/repos/mozilla/prowac/pulls/30"}}},"pull_request":{"url":"https://api.github.com/repos/mozilla/prowac/pulls/30","id":59238922,"html_url":"https://github.com/mozilla/prowac/pull/30","diff_url":"https://github.com/mozilla/prowac/pull/30.diff","patch_url":"https://github.com/mozilla/prowac/pull/30.patch","issue_url":"https://api.github.com/repos/mozilla/prowac/issues/30","number":30,"state":"open","locked":false,"title":"Fetch resources and pass them to probes","user":{"login":"TimAbraldes","id":955084,"avatar_url":"https://avatars.githubusercontent.com/u/955084?v=3","gravatar_id":"","url":"https://api.github.com/users/TimAbraldes","html_url":"https://github.com/TimAbraldes","followers_url":"https://api.github.com/users/TimAbraldes/followers","following_url":"https://api.github.com/users/TimAbraldes/following{/other_user}","gists_url":"https://api.github.com/users/TimAbraldes/gists{/gist_id}","starred_url":"https://api.github.com/users/TimAbraldes/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/TimAbraldes/subscriptions","organizations_url":"https://api.github.com/users/TimAbraldes/orgs","repos_url":"https://api.github.com/users/TimAbraldes/repos","events_url":"https://api.github.com/users/TimAbraldes/events{/privacy}","received_events_url":"https://api.github.com/users/TimAbraldes/received_events","type":"User","site_admin":false},"body":"Also implements the 'totalRecords' probe","created_at":"2016-02-12T22:14:35Z","updated_at":"2016-02-23T03:46:34Z","closed_at":null,"merged_at":null,"merge_commit_sha":"aec1eb421fe4b4f4dfb7a733d397571470a5ca74","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/mozilla/prowac/pulls/30/commits","review_comments_url":"https://api.github.com/repos/mozilla/prowac/pulls/30/comments","review_comment_url":"https://api.github.com/repos/mozilla/prowac/pulls/comments{/number}","comments_url":"https://api.github.com/repos/mozilla/prowac/issues/30/comments","statuses_url":"https://api.github.com/repos/mozilla/prowac/statuses/eb0e0a7ce923dff0afb3d4bb6be74561f5ae0cb9","head":{"label":"mozilla:process","ref":"process","sha":"eb0e0a7ce923dff0afb3d4bb6be74561f5ae0cb9","user":{"login":"mozilla","id":131524,"avatar_url":"https://avatars.githubusercontent.com/u/131524?v=3","gravatar_id":"","url":"https://api.github.com/users/mozilla","html_url":"https://github.com/mozilla","followers_url":"https://api.github.com/users/mozilla/followers","following_url":"https://api.github.com/users/mozilla/following{/other_user}","gists_url":"https://api.github.com/users/mozilla/gists{/gist_id}","starred_url":"https://api.github.com/users/mozilla/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mozilla/subscriptions","organizations_url":"https://api.github.com/users/mozilla/orgs","repos_url":"https://api.github.com/users/mozilla/repos","events_url":"https://api.github.com/users/mozilla/events{/privacy}","received_events_url":"https://api.github.com/users/mozilla/received_events","type":"Organization","site_admin":false},"repo":{"id":49994092,"name":"prowac","full_name":"mozilla/prowac","owner":{"login":"mozilla","id":131524,"avatar_url":"https://avatars.githubusercontent.com/u/131524?v=3","gravatar_id":"","url":"https://api.github.com/users/mozilla","html_url":"https://github.com/mozilla","followers_url":"https://api.github.com/users/mozilla/followers","following_url":"https://api.github.com/users/mozilla/following{/other_user}","gists_url":"https://api.github.com/users/mozilla/gists{/gist_id}","starred_url":"https://api.github.com/users/mozilla/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mozilla/subscriptions","organizations_url":"https://api.github.com/users/mozilla/orgs","repos_url":"https://api.github.com/users/mozilla/repos","events_url":"https://api.github.com/users/mozilla/events{/privacy}","received_events_url":"https://api.github.com/users/mozilla/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/mozilla/prowac","description":"Progressive Web App Crawler","fork":false,"url":"https://api.github.com/repos/mozilla/prowac","forks_url":"https://api.github.com/repos/mozilla/prowac/forks","keys_url":"https://api.github.com/repos/mozilla/prowac/keys{/key_id}","collaborators_url":"https://api.github.com/repos/mozilla/prowac/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/mozilla/prowac/teams","hooks_url":"https://api.github.com/repos/mozilla/prowac/hooks","issue_events_url":"https://api.github.com/repos/mozilla/prowac/issues/events{/number}","events_url":"https://api.github.com/repos/mozilla/prowac/events","assignees_url":"https://api.github.com/repos/mozilla/prowac/assignees{/user}","branches_url":"https://api.github.com/repos/mozilla/prowac/branches{/branch}","tags_url":"https://api.github.com/repos/mozilla/prowac/tags","blobs_url":"https://api.github.com/repos/mozilla/prowac/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/mozilla/prowac/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/mozilla/prowac/git/refs{/sha}","trees_url":"https://api.github.com/repos/mozilla/prowac/git/trees{/sha}","statuses_url":"https://api.github.com/repos/mozilla/prowac/statuses/{sha}","languages_url":"https://api.github.com/repos/mozilla/prowac/languages","stargazers_url":"https://api.github.com/repos/mozilla/prowac/stargazers","contributors_url":"https://api.github.com/repos/mozilla/prowac/contributors","subscribers_url":"https://api.github.com/repos/mozilla/prowac/subscribers","subscription_url":"https://api.github.com/repos/mozilla/prowac/subscription","commits_url":"https://api.github.com/repos/mozilla/prowac/commits{/sha}","git_commits_url":"https://api.github.com/repos/mozilla/prowac/git/commits{/sha}","comments_url":"https://api.github.com/repos/mozilla/prowac/comments{/number}","issue_comment_url":"https://api.github.com/repos/mozilla/prowac/issues/comments{/number}","contents_url":"https://api.github.com/repos/mozilla/prowac/contents/{+path}","compare_url":"https://api.github.com/repos/mozilla/prowac/compare/{base}...{head}","merges_url":"https://api.github.com/repos/mozilla/prowac/merges","archive_url":"https://api.github.com/repos/mozilla/prowac/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/mozilla/prowac/downloads","issues_url":"https://api.github.com/repos/mozilla/prowac/issues{/number}","pulls_url":"https://api.github.com/repos/mozilla/prowac/pulls{/number}","milestones_url":"https://api.github.com/repos/mozilla/prowac/milestones{/number}","notifications_url":"https://api.github.com/repos/mozilla/prowac/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/mozilla/prowac/labels{/name}","releases_url":"https://api.github.com/repos/mozilla/prowac/releases{/id}","deployments_url":"https://api.github.com/repos/mozilla/prowac/deployments","created_at":"2016-01-20T00:56:12Z","updated_at":"2016-02-03T19:31:53Z","pushed_at":"2016-02-23T01:50:47Z","git_url":"git://github.com/mozilla/prowac.git","ssh_url":"git@github.com:mozilla/prowac.git","clone_url":"https://github.com/mozilla/prowac.git","svn_url":"https://github.com/mozilla/prowac","homepage":"","size":61,"stargazers_count":1,"watchers_count":1,"language":"JavaScript","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":1,"mirror_url":null,"open_issues_count":21,"forks":1,"open_issues":21,"watchers":1,"default_branch":"master"}},"base":{"label":"mozilla:master","ref":"master","sha":"5e674e73dd57edd2880079fc840e5b14b89f8ffa","user":{"login":"mozilla","id":131524,"avatar_url":"https://avatars.githubusercontent.com/u/131524?v=3","gravatar_id":"","url":"https://api.github.com/users/mozilla","html_url":"https://github.com/mozilla","followers_url":"https://api.github.com/users/mozilla/followers","following_url":"https://api.github.com/users/mozilla/following{/other_user}","gists_url":"https://api.github.com/users/mozilla/gists{/gist_id}","starred_url":"https://api.github.com/users/mozilla/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mozilla/subscriptions","organizations_url":"https://api.github.com/users/mozilla/orgs","repos_url":"https://api.github.com/users/mozilla/repos","events_url":"https://api.github.com/users/mozilla/events{/privacy}","received_events_url":"https://api.github.com/users/mozilla/received_events","type":"Organization","site_admin":false},"repo":{"id":49994092,"name":"prowac","full_name":"mozilla/prowac","owner":{"login":"mozilla","id":131524,"avatar_url":"https://avatars.githubusercontent.com/u/131524?v=3","gravatar_id":"","url":"https://api.github.com/users/mozilla","html_url":"https://github.com/mozilla","followers_url":"https://api.github.com/users/mozilla/followers","following_url":"https://api.github.com/users/mozilla/following{/other_user}","gists_url":"https://api.github.com/users/mozilla/gists{/gist_id}","starred_url":"https://api.github.com/users/mozilla/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mozilla/subscriptions","organizations_url":"https://api.github.com/users/mozilla/orgs","repos_url":"https://api.github.com/users/mozilla/repos","events_url":"https://api.github.com/users/mozilla/events{/privacy}","received_events_url":"https://api.github.com/users/mozilla/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/mozilla/prowac","description":"Progressive Web App Crawler","fork":false,"url":"https://api.github.com/repos/mozilla/prowac","forks_url":"https://api.github.com/repos/mozilla/prowac/forks","keys_url":"https://api.github.com/repos/mozilla/prowac/keys{/key_id}","collaborators_url":"https://api.github.com/repos/mozilla/prowac/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/mozilla/prowac/teams","hooks_url":"https://api.github.com/repos/mozilla/prowac/hooks","issue_events_url":"https://api.github.com/repos/mozilla/prowac/issues/events{/number}","events_url":"https://api.github.com/repos/mozilla/prowac/events","assignees_url":"https://api.github.com/repos/mozilla/prowac/assignees{/user}","branches_url":"https://api.github.com/repos/mozilla/prowac/branches{/branch}","tags_url":"https://api.github.com/repos/mozilla/prowac/tags","blobs_url":"https://api.github.com/repos/mozilla/prowac/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/mozilla/prowac/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/mozilla/prowac/git/refs{/sha}","trees_url":"https://api.github.com/repos/mozilla/prowac/git/trees{/sha}","statuses_url":"https://api.github.com/repos/mozilla/prowac/statuses/{sha}","languages_url":"https://api.github.com/repos/mozilla/prowac/languages","stargazers_url":"https://api.github.com/repos/mozilla/prowac/stargazers","contributors_url":"https://api.github.com/repos/mozilla/prowac/contributors","subscribers_url":"https://api.github.com/repos/mozilla/prowac/subscribers","subscription_url":"https://api.github.com/repos/mozilla/prowac/subscription","commits_url":"https://api.github.com/repos/mozilla/prowac/commits{/sha}","git_commits_url":"https://api.github.com/repos/mozilla/prowac/git/commits{/sha}","comments_url":"https://api.github.com/repos/mozilla/prowac/comments{/number}","issue_comment_url":"https://api.github.com/repos/mozilla/prowac/issues/comments{/number}","contents_url":"https://api.github.com/repos/mozilla/prowac/contents/{+path}","compare_url":"https://api.github.com/repos/mozilla/prowac/compare/{base}...{head}","merges_url":"https://api.github.com/repos/mozilla/prowac/merges","archive_url":"https://api.github.com/repos/mozilla/prowac/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/mozilla/prowac/downloads","issues_url":"https://api.github.com/repos/mozilla/prowac/issues{/number}","pulls_url":"https://api.github.com/repos/mozilla/prowac/pulls{/number}","milestones_url":"https://api.github.com/repos/mozilla/prowac/milestones{/number}","notifications_url":"https://api.github.com/repos/mozilla/prowac/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/mozilla/prowac/labels{/name}","releases_url":"https://api.github.com/repos/mozilla/prowac/releases{/id}","deployments_url":"https://api.github.com/repos/mozilla/prowac/deployments","created_at":"2016-01-20T00:56:12Z","updated_at":"2016-02-03T19:31:53Z","pushed_at":"2016-02-23T01:50:47Z","git_url":"git://github.com/mozilla/prowac.git","ssh_url":"git@github.com:mozilla/prowac.git","clone_url":"https://github.com/mozilla/prowac.git","svn_url":"https://github.com/mozilla/prowac","homepage":"","size":61,"stargazers_count":1,"watchers_count":1,"language":"JavaScript","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":1,"mirror_url":null,"open_issues_count":21,"forks":1,"open_issues":21,"watchers":1,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/mozilla/prowac/pulls/30"},"html":{"href":"https://github.com/mozilla/prowac/pull/30"},"issue":{"href":"https://api.github.com/repos/mozilla/prowac/issues/30"},"comments":{"href":"https://api.github.com/repos/mozilla/prowac/issues/30/comments"},"review_comments":{"href":"https://api.github.com/repos/mozilla/prowac/pulls/30/comments"},"review_comment":{"href":"https://api.github.com/repos/mozilla/prowac/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/mozilla/prowac/pulls/30/commits"},"statuses":{"href":"https://api.github.com/repos/mozilla/prowac/statuses/eb0e0a7ce923dff0afb3d4bb6be74561f5ae0cb9"}}}}