{"action":"created","comment":{"url":"https://api.github.com/repos/s0ph1e/node-website-scraper/pulls/comments/68311522","id":68311522,"diff_hunk":"@@ -26,24 +26,49 @@ function loadHtmlAndCss (context, po) {\n \n function Scraper (options) {\n \tthis.originalResources = [];\n+\tthis.occupiedFileNames = [];\n \tthis.loadedResources = [];\n+\tthis.respondedResourcePromises = {};\n+\tthis.loadedResourcePromises = {};\n \n \tthis.options = _.extend({}, defaults, options);\n \tthis.options.request = _.extend({}, defaults.request, options.request);\n \tthis.options.directory = path.resolve(process.cwd(), this.options.directory || '');\n \tthis.options.filenameGenerator = getFilenameGenerator(this.options.filenameGenerator);\n }\n \n-Scraper.prototype.getLoadedResource = function getLoadedResource (resource) {","path":"lib/scraper.js","position":24,"original_position":24,"commit_id":"2580836d05ec8a0d6a4b9382c8eb716847965167","original_commit_id":"2580836d05ec8a0d6a4b9382c8eb716847965167","user":{"login":"s0ph1e","id":2333969,"avatar_url":"https://avatars.githubusercontent.com/u/2333969?v=3","gravatar_id":"","url":"https://api.github.com/users/s0ph1e","html_url":"https://github.com/s0ph1e","followers_url":"https://api.github.com/users/s0ph1e/followers","following_url":"https://api.github.com/users/s0ph1e/following{/other_user}","gists_url":"https://api.github.com/users/s0ph1e/gists{/gist_id}","starred_url":"https://api.github.com/users/s0ph1e/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/s0ph1e/subscriptions","organizations_url":"https://api.github.com/users/s0ph1e/orgs","repos_url":"https://api.github.com/users/s0ph1e/repos","events_url":"https://api.github.com/users/s0ph1e/events{/privacy}","received_events_url":"https://api.github.com/users/s0ph1e/received_events","type":"User","site_admin":false},"body":"Maybe it makes sense to get rid of `addLoadedResource` too?","created_at":"2016-06-23T20:44:47Z","updated_at":"2016-06-23T20:44:47Z","html_url":"https://github.com/s0ph1e/node-website-scraper/pull/78#discussion_r68311522","pull_request_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/pulls/78","_links":{"self":{"href":"https://api.github.com/repos/s0ph1e/node-website-scraper/pulls/comments/68311522"},"html":{"href":"https://github.com/s0ph1e/node-website-scraper/pull/78#discussion_r68311522"},"pull_request":{"href":"https://api.github.com/repos/s0ph1e/node-website-scraper/pulls/78"}}},"pull_request":{"url":"https://api.github.com/repos/s0ph1e/node-website-scraper/pulls/78","id":73728608,"html_url":"https://github.com/s0ph1e/node-website-scraper/pull/78","diff_url":"https://github.com/s0ph1e/node-website-scraper/pull/78.diff","patch_url":"https://github.com/s0ph1e/node-website-scraper/pull/78.patch","issue_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/issues/78","number":78,"state":"open","locked":false,"title":"Improved mechanism for re-using already loaded / loading resources.","user":{"login":"marklagendijk","id":1188039,"avatar_url":"https://avatars.githubusercontent.com/u/1188039?v=3","gravatar_id":"","url":"https://api.github.com/users/marklagendijk","html_url":"https://github.com/marklagendijk","followers_url":"https://api.github.com/users/marklagendijk/followers","following_url":"https://api.github.com/users/marklagendijk/following{/other_user}","gists_url":"https://api.github.com/users/marklagendijk/gists{/gist_id}","starred_url":"https://api.github.com/users/marklagendijk/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/marklagendijk/subscriptions","organizations_url":"https://api.github.com/users/marklagendijk/orgs","repos_url":"https://api.github.com/users/marklagendijk/repos","events_url":"https://api.github.com/users/marklagendijk/events{/privacy}","received_events_url":"https://api.github.com/users/marklagendijk/received_events","type":"User","site_admin":false},"body":"1. `requestResource` executes the request for the resource, and saves the response. Processing the same resource multiple times is prevented by maintaining `respondedResourcePromises`.\r\n2. `loadResource` calls `requestResource` and saves the resource to disk. Processing the same resource multiple times is prevented by maintaining `loadedResourcePromises`.\r\n3. To prevent deadlocks `file-handlers/html.js` only waits for `requestResource`, not for `loadResource`.\r\n4. To determine when the scraper is finished `loadedResourcePromises` is used.","created_at":"2016-06-14T12:13:29Z","updated_at":"2016-06-23T20:44:47Z","closed_at":null,"merged_at":null,"merge_commit_sha":"2547db26844c9c95bc26ae79d01ea0c4b983b2e2","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/pulls/78/commits","review_comments_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/pulls/78/comments","review_comment_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/pulls/comments{/number}","comments_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/issues/78/comments","statuses_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/statuses/2580836d05ec8a0d6a4b9382c8eb716847965167","head":{"label":"s0ph1e:resource-loading-concurrency-fix","ref":"resource-loading-concurrency-fix","sha":"2580836d05ec8a0d6a4b9382c8eb716847965167","user":{"login":"s0ph1e","id":2333969,"avatar_url":"https://avatars.githubusercontent.com/u/2333969?v=3","gravatar_id":"","url":"https://api.github.com/users/s0ph1e","html_url":"https://github.com/s0ph1e","followers_url":"https://api.github.com/users/s0ph1e/followers","following_url":"https://api.github.com/users/s0ph1e/following{/other_user}","gists_url":"https://api.github.com/users/s0ph1e/gists{/gist_id}","starred_url":"https://api.github.com/users/s0ph1e/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/s0ph1e/subscriptions","organizations_url":"https://api.github.com/users/s0ph1e/orgs","repos_url":"https://api.github.com/users/s0ph1e/repos","events_url":"https://api.github.com/users/s0ph1e/events{/privacy}","received_events_url":"https://api.github.com/users/s0ph1e/received_events","type":"User","site_admin":false},"repo":{"id":23670920,"name":"node-website-scraper","full_name":"s0ph1e/node-website-scraper","owner":{"login":"s0ph1e","id":2333969,"avatar_url":"https://avatars.githubusercontent.com/u/2333969?v=3","gravatar_id":"","url":"https://api.github.com/users/s0ph1e","html_url":"https://github.com/s0ph1e","followers_url":"https://api.github.com/users/s0ph1e/followers","following_url":"https://api.github.com/users/s0ph1e/following{/other_user}","gists_url":"https://api.github.com/users/s0ph1e/gists{/gist_id}","starred_url":"https://api.github.com/users/s0ph1e/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/s0ph1e/subscriptions","organizations_url":"https://api.github.com/users/s0ph1e/orgs","repos_url":"https://api.github.com/users/s0ph1e/repos","events_url":"https://api.github.com/users/s0ph1e/events{/privacy}","received_events_url":"https://api.github.com/users/s0ph1e/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/s0ph1e/node-website-scraper","description":"Download website to a local directory (including all css, images, js, etc.)","fork":false,"url":"https://api.github.com/repos/s0ph1e/node-website-scraper","forks_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/forks","keys_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/keys{/key_id}","collaborators_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/teams","hooks_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/hooks","issue_events_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/issues/events{/number}","events_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/events","assignees_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/assignees{/user}","branches_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/branches{/branch}","tags_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/tags","blobs_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/git/refs{/sha}","trees_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/git/trees{/sha}","statuses_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/statuses/{sha}","languages_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/languages","stargazers_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/stargazers","contributors_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/contributors","subscribers_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/subscribers","subscription_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/subscription","commits_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/commits{/sha}","git_commits_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/git/commits{/sha}","comments_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/comments{/number}","issue_comment_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/issues/comments{/number}","contents_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/contents/{+path}","compare_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/compare/{base}...{head}","merges_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/merges","archive_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/downloads","issues_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/issues{/number}","pulls_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/pulls{/number}","milestones_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/milestones{/number}","notifications_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/labels{/name}","releases_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/releases{/id}","deployments_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/deployments","created_at":"2014-09-04T16:48:20Z","updated_at":"2016-06-23T19:04:18Z","pushed_at":"2016-06-20T18:16:42Z","git_url":"git://github.com/s0ph1e/node-website-scraper.git","ssh_url":"git@github.com:s0ph1e/node-website-scraper.git","clone_url":"https://github.com/s0ph1e/node-website-scraper.git","svn_url":"https://github.com/s0ph1e/node-website-scraper","homepage":"https://www.npmjs.org/package/website-scraper","size":348,"stargazers_count":79,"watchers_count":79,"language":"JavaScript","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":17,"mirror_url":null,"open_issues_count":4,"forks":17,"open_issues":4,"watchers":79,"default_branch":"master"}},"base":{"label":"s0ph1e:master","ref":"master","sha":"b6c64a9b7fc046a858a9eb143404aadaad9b315a","user":{"login":"s0ph1e","id":2333969,"avatar_url":"https://avatars.githubusercontent.com/u/2333969?v=3","gravatar_id":"","url":"https://api.github.com/users/s0ph1e","html_url":"https://github.com/s0ph1e","followers_url":"https://api.github.com/users/s0ph1e/followers","following_url":"https://api.github.com/users/s0ph1e/following{/other_user}","gists_url":"https://api.github.com/users/s0ph1e/gists{/gist_id}","starred_url":"https://api.github.com/users/s0ph1e/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/s0ph1e/subscriptions","organizations_url":"https://api.github.com/users/s0ph1e/orgs","repos_url":"https://api.github.com/users/s0ph1e/repos","events_url":"https://api.github.com/users/s0ph1e/events{/privacy}","received_events_url":"https://api.github.com/users/s0ph1e/received_events","type":"User","site_admin":false},"repo":{"id":23670920,"name":"node-website-scraper","full_name":"s0ph1e/node-website-scraper","owner":{"login":"s0ph1e","id":2333969,"avatar_url":"https://avatars.githubusercontent.com/u/2333969?v=3","gravatar_id":"","url":"https://api.github.com/users/s0ph1e","html_url":"https://github.com/s0ph1e","followers_url":"https://api.github.com/users/s0ph1e/followers","following_url":"https://api.github.com/users/s0ph1e/following{/other_user}","gists_url":"https://api.github.com/users/s0ph1e/gists{/gist_id}","starred_url":"https://api.github.com/users/s0ph1e/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/s0ph1e/subscriptions","organizations_url":"https://api.github.com/users/s0ph1e/orgs","repos_url":"https://api.github.com/users/s0ph1e/repos","events_url":"https://api.github.com/users/s0ph1e/events{/privacy}","received_events_url":"https://api.github.com/users/s0ph1e/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/s0ph1e/node-website-scraper","description":"Download website to a local directory (including all css, images, js, etc.)","fork":false,"url":"https://api.github.com/repos/s0ph1e/node-website-scraper","forks_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/forks","keys_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/keys{/key_id}","collaborators_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/teams","hooks_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/hooks","issue_events_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/issues/events{/number}","events_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/events","assignees_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/assignees{/user}","branches_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/branches{/branch}","tags_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/tags","blobs_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/git/refs{/sha}","trees_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/git/trees{/sha}","statuses_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/statuses/{sha}","languages_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/languages","stargazers_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/stargazers","contributors_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/contributors","subscribers_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/subscribers","subscription_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/subscription","commits_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/commits{/sha}","git_commits_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/git/commits{/sha}","comments_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/comments{/number}","issue_comment_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/issues/comments{/number}","contents_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/contents/{+path}","compare_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/compare/{base}...{head}","merges_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/merges","archive_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/downloads","issues_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/issues{/number}","pulls_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/pulls{/number}","milestones_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/milestones{/number}","notifications_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/labels{/name}","releases_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/releases{/id}","deployments_url":"https://api.github.com/repos/s0ph1e/node-website-scraper/deployments","created_at":"2014-09-04T16:48:20Z","updated_at":"2016-06-23T19:04:18Z","pushed_at":"2016-06-20T18:16:42Z","git_url":"git://github.com/s0ph1e/node-website-scraper.git","ssh_url":"git@github.com:s0ph1e/node-website-scraper.git","clone_url":"https://github.com/s0ph1e/node-website-scraper.git","svn_url":"https://github.com/s0ph1e/node-website-scraper","homepage":"https://www.npmjs.org/package/website-scraper","size":348,"stargazers_count":79,"watchers_count":79,"language":"JavaScript","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":17,"mirror_url":null,"open_issues_count":4,"forks":17,"open_issues":4,"watchers":79,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/s0ph1e/node-website-scraper/pulls/78"},"html":{"href":"https://github.com/s0ph1e/node-website-scraper/pull/78"},"issue":{"href":"https://api.github.com/repos/s0ph1e/node-website-scraper/issues/78"},"comments":{"href":"https://api.github.com/repos/s0ph1e/node-website-scraper/issues/78/comments"},"review_comments":{"href":"https://api.github.com/repos/s0ph1e/node-website-scraper/pulls/78/comments"},"review_comment":{"href":"https://api.github.com/repos/s0ph1e/node-website-scraper/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/s0ph1e/node-website-scraper/pulls/78/commits"},"statuses":{"href":"https://api.github.com/repos/s0ph1e/node-website-scraper/statuses/2580836d05ec8a0d6a4b9382c8eb716847965167"}}}}
{"actor":{"display_login":"s0ph1e"}}