{"action":"created","comment":{"url":"https://api.github.com/repos/jez500/content-crawler/pulls/comments/350472817","pull_request_review_id":322641571,"id":350472817,"node_id":"MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDM1MDQ3MjgxNw==","diff_hunk":"@@ -0,0 +1,37 @@\n+<?php\n+\n+$fp = fopen('urls.lock', 'r+');\n+$lock = flock($fp, LOCK_EX);\n+\n+if (!file_exists('urls.json')) {\n+  file_put_contents('urls.json', json_encode(['token' => 'url']));\n+}\n+\n+$all = json_decode(file_get_contents('urls.json'));\n+\n+$token = isset($_GET['token'])?$_GET['token']:'';\n+\n+if ($token) {\n+  if (isset($all->$token)) {\n+    $headers = get_headers($all->$token);\n+    foreach ($headers as $header) {\n+      header($header);\n+    }\n+    readfile($all->$token);\n+  } else {\n+    http_response_code(404);\n+  }\n+} else {\n+  $url = $_POST['url'];","path":"public/resource.php","position":25,"original_position":25,"commit_id":"52a2e7f78566f96becb31756d206777ffd9e753a","original_commit_id":"52a2e7f78566f96becb31756d206777ffd9e753a","user":{"login":"jez500","id":782823,"node_id":"MDQ6VXNlcjc4MjgyMw==","avatar_url":"https://avatars3.githubusercontent.com/u/782823?v=4","gravatar_id":"","url":"https://api.github.com/users/jez500","html_url":"https://github.com/jez500","followers_url":"https://api.github.com/users/jez500/followers","following_url":"https://api.github.com/users/jez500/following{/other_user}","gists_url":"https://api.github.com/users/jez500/gists{/gist_id}","starred_url":"https://api.github.com/users/jez500/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jez500/subscriptions","organizations_url":"https://api.github.com/users/jez500/orgs","repos_url":"https://api.github.com/users/jez500/repos","events_url":"https://api.github.com/users/jez500/events{/privacy}","received_events_url":"https://api.github.com/users/jez500/received_events","type":"User","site_admin":false},"body":"prevent notice with `$url = isset($_POST['url']) ? $_POST['url'] : ''` ?","created_at":"2019-11-25T23:18:28Z","updated_at":"2019-11-25T23:24:20Z","html_url":"https://github.com/jez500/content-crawler/pull/3#discussion_r350472817","pull_request_url":"https://api.github.com/repos/jez500/content-crawler/pulls/3","author_association":"OWNER","_links":{"self":{"href":"https://api.github.com/repos/jez500/content-crawler/pulls/comments/350472817"},"html":{"href":"https://github.com/jez500/content-crawler/pull/3#discussion_r350472817"},"pull_request":{"href":"https://api.github.com/repos/jez500/content-crawler/pulls/3"}}},"pull_request":{"url":"https://api.github.com/repos/jez500/content-crawler/pulls/3","id":344977237,"node_id":"MDExOlB1bGxSZXF1ZXN0MzQ0OTc3MjM3","html_url":"https://github.com/jez500/content-crawler/pull/3","diff_url":"https://github.com/jez500/content-crawler/pull/3.diff","patch_url":"https://github.com/jez500/content-crawler/pull/3.patch","issue_url":"https://api.github.com/repos/jez500/content-crawler/issues/3","number":3,"state":"open","locked":false,"title":"Large site validation","user":{"login":"damyon","id":758959,"node_id":"MDQ6VXNlcjc1ODk1OQ==","avatar_url":"https://avatars1.githubusercontent.com/u/758959?v=4","gravatar_id":"","url":"https://api.github.com/users/damyon","html_url":"https://github.com/damyon","followers_url":"https://api.github.com/users/damyon/followers","following_url":"https://api.github.com/users/damyon/following{/other_user}","gists_url":"https://api.github.com/users/damyon/gists{/gist_id}","starred_url":"https://api.github.com/users/damyon/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/damyon/subscriptions","organizations_url":"https://api.github.com/users/damyon/orgs","repos_url":"https://api.github.com/users/damyon/repos","events_url":"https://api.github.com/users/damyon/events{/privacy}","received_events_url":"https://api.github.com/users/damyon/received_events","type":"User","site_admin":false},"body":"Structure and UI improvements based on experience crawling a large site.\r\n\r\nThe main changes are some additional settings for crawling the site, some changes to settings to allow many values instead of a single field. Changes to the cleaning so it makes few changes to the content. Improvements to the copy / paste which struggled with large documents. Detect extremely long URLS. \r\n","created_at":"2019-11-25T01:29:06Z","updated_at":"2019-11-25T23:24:20Z","closed_at":null,"merged_at":null,"merge_commit_sha":"7ed18ad575b1c6e6d306f1da5a834df2ff3018cd","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/jez500/content-crawler/pulls/3/commits","review_comments_url":"https://api.github.com/repos/jez500/content-crawler/pulls/3/comments","review_comment_url":"https://api.github.com/repos/jez500/content-crawler/pulls/comments{/number}","comments_url":"https://api.github.com/repos/jez500/content-crawler/issues/3/comments","statuses_url":"https://api.github.com/repos/jez500/content-crawler/statuses/52a2e7f78566f96becb31756d206777ffd9e753a","head":{"label":"damyon:master","ref":"master","sha":"52a2e7f78566f96becb31756d206777ffd9e753a","user":{"login":"damyon","id":758959,"node_id":"MDQ6VXNlcjc1ODk1OQ==","avatar_url":"https://avatars1.githubusercontent.com/u/758959?v=4","gravatar_id":"","url":"https://api.github.com/users/damyon","html_url":"https://github.com/damyon","followers_url":"https://api.github.com/users/damyon/followers","following_url":"https://api.github.com/users/damyon/following{/other_user}","gists_url":"https://api.github.com/users/damyon/gists{/gist_id}","starred_url":"https://api.github.com/users/damyon/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/damyon/subscriptions","organizations_url":"https://api.github.com/users/damyon/orgs","repos_url":"https://api.github.com/users/damyon/repos","events_url":"https://api.github.com/users/damyon/events{/privacy}","received_events_url":"https://api.github.com/users/damyon/received_events","type":"User","site_admin":false},"repo":{"id":203546973,"node_id":"MDEwOlJlcG9zaXRvcnkyMDM1NDY5NzM=","name":"content-crawler","full_name":"damyon/content-crawler","private":false,"owner":{"login":"damyon","id":758959,"node_id":"MDQ6VXNlcjc1ODk1OQ==","avatar_url":"https://avatars1.githubusercontent.com/u/758959?v=4","gravatar_id":"","url":"https://api.github.com/users/damyon","html_url":"https://github.com/damyon","followers_url":"https://api.github.com/users/damyon/followers","following_url":"https://api.github.com/users/damyon/following{/other_user}","gists_url":"https://api.github.com/users/damyon/gists{/gist_id}","starred_url":"https://api.github.com/users/damyon/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/damyon/subscriptions","organizations_url":"https://api.github.com/users/damyon/orgs","repos_url":"https://api.github.com/users/damyon/repos","events_url":"https://api.github.com/users/damyon/events{/privacy}","received_events_url":"https://api.github.com/users/damyon/received_events","type":"User","site_admin":false},"html_url":"https://github.com/damyon/content-crawler","description":"Crawl a site to get a count of the pages, images, documents and forms it contains","fork":true,"url":"https://api.github.com/repos/damyon/content-crawler","forks_url":"https://api.github.com/repos/damyon/content-crawler/forks","keys_url":"https://api.github.com/repos/damyon/content-crawler/keys{/key_id}","collaborators_url":"https://api.github.com/repos/damyon/content-crawler/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/damyon/content-crawler/teams","hooks_url":"https://api.github.com/repos/damyon/content-crawler/hooks","issue_events_url":"https://api.github.com/repos/damyon/content-crawler/issues/events{/number}","events_url":"https://api.github.com/repos/damyon/content-crawler/events","assignees_url":"https://api.github.com/repos/damyon/content-crawler/assignees{/user}","branches_url":"https://api.github.com/repos/damyon/content-crawler/branches{/branch}","tags_url":"https://api.github.com/repos/damyon/content-crawler/tags","blobs_url":"https://api.github.com/repos/damyon/content-crawler/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/damyon/content-crawler/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/damyon/content-crawler/git/refs{/sha}","trees_url":"https://api.github.com/repos/damyon/content-crawler/git/trees{/sha}","statuses_url":"https://api.github.com/repos/damyon/content-crawler/statuses/{sha}","languages_url":"https://api.github.com/repos/damyon/content-crawler/languages","stargazers_url":"https://api.github.com/repos/damyon/content-crawler/stargazers","contributors_url":"https://api.github.com/repos/damyon/content-crawler/contributors","subscribers_url":"https://api.github.com/repos/damyon/content-crawler/subscribers","subscription_url":"https://api.github.com/repos/damyon/content-crawler/subscription","commits_url":"https://api.github.com/repos/damyon/content-crawler/commits{/sha}","git_commits_url":"https://api.github.com/repos/damyon/content-crawler/git/commits{/sha}","comments_url":"https://api.github.com/repos/damyon/content-crawler/comments{/number}","issue_comment_url":"https://api.github.com/repos/damyon/content-crawler/issues/comments{/number}","contents_url":"https://api.github.com/repos/damyon/content-crawler/contents/{+path}","compare_url":"https://api.github.com/repos/damyon/content-crawler/compare/{base}...{head}","merges_url":"https://api.github.com/repos/damyon/content-crawler/merges","archive_url":"https://api.github.com/repos/damyon/content-crawler/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/damyon/content-crawler/downloads","issues_url":"https://api.github.com/repos/damyon/content-crawler/issues{/number}","pulls_url":"https://api.github.com/repos/damyon/content-crawler/pulls{/number}","milestones_url":"https://api.github.com/repos/damyon/content-crawler/milestones{/number}","notifications_url":"https://api.github.com/repos/damyon/content-crawler/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/damyon/content-crawler/labels{/name}","releases_url":"https://api.github.com/repos/damyon/content-crawler/releases{/id}","deployments_url":"https://api.github.com/repos/damyon/content-crawler/deployments","created_at":"2019-08-21T09:03:32Z","updated_at":"2019-11-25T01:24:16Z","pushed_at":"2019-11-25T01:24:11Z","git_url":"git://github.com/damyon/content-crawler.git","ssh_url":"git@github.com:damyon/content-crawler.git","clone_url":"https://github.com/damyon/content-crawler.git","svn_url":"https://github.com/damyon/content-crawler","homepage":null,"size":100,"stargazers_count":0,"watchers_count":0,"language":"HTML","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"disabled":false,"open_issues_count":0,"license":null,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"jez500:master","ref":"master","sha":"fdbe407473d92a2d6c264815724bb7815951da9d","user":{"login":"jez500","id":782823,"node_id":"MDQ6VXNlcjc4MjgyMw==","avatar_url":"https://avatars3.githubusercontent.com/u/782823?v=4","gravatar_id":"","url":"https://api.github.com/users/jez500","html_url":"https://github.com/jez500","followers_url":"https://api.github.com/users/jez500/followers","following_url":"https://api.github.com/users/jez500/following{/other_user}","gists_url":"https://api.github.com/users/jez500/gists{/gist_id}","starred_url":"https://api.github.com/users/jez500/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jez500/subscriptions","organizations_url":"https://api.github.com/users/jez500/orgs","repos_url":"https://api.github.com/users/jez500/repos","events_url":"https://api.github.com/users/jez500/events{/privacy}","received_events_url":"https://api.github.com/users/jez500/received_events","type":"User","site_admin":false},"repo":{"id":182496786,"node_id":"MDEwOlJlcG9zaXRvcnkxODI0OTY3ODY=","name":"content-crawler","full_name":"jez500/content-crawler","private":false,"owner":{"login":"jez500","id":782823,"node_id":"MDQ6VXNlcjc4MjgyMw==","avatar_url":"https://avatars3.githubusercontent.com/u/782823?v=4","gravatar_id":"","url":"https://api.github.com/users/jez500","html_url":"https://github.com/jez500","followers_url":"https://api.github.com/users/jez500/followers","following_url":"https://api.github.com/users/jez500/following{/other_user}","gists_url":"https://api.github.com/users/jez500/gists{/gist_id}","starred_url":"https://api.github.com/users/jez500/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jez500/subscriptions","organizations_url":"https://api.github.com/users/jez500/orgs","repos_url":"https://api.github.com/users/jez500/repos","events_url":"https://api.github.com/users/jez500/events{/privacy}","received_events_url":"https://api.github.com/users/jez500/received_events","type":"User","site_admin":false},"html_url":"https://github.com/jez500/content-crawler","description":"Crawl a site to get a count of the pages, images, documents and forms it contains","fork":false,"url":"https://api.github.com/repos/jez500/content-crawler","forks_url":"https://api.github.com/repos/jez500/content-crawler/forks","keys_url":"https://api.github.com/repos/jez500/content-crawler/keys{/key_id}","collaborators_url":"https://api.github.com/repos/jez500/content-crawler/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/jez500/content-crawler/teams","hooks_url":"https://api.github.com/repos/jez500/content-crawler/hooks","issue_events_url":"https://api.github.com/repos/jez500/content-crawler/issues/events{/number}","events_url":"https://api.github.com/repos/jez500/content-crawler/events","assignees_url":"https://api.github.com/repos/jez500/content-crawler/assignees{/user}","branches_url":"https://api.github.com/repos/jez500/content-crawler/branches{/branch}","tags_url":"https://api.github.com/repos/jez500/content-crawler/tags","blobs_url":"https://api.github.com/repos/jez500/content-crawler/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/jez500/content-crawler/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/jez500/content-crawler/git/refs{/sha}","trees_url":"https://api.github.com/repos/jez500/content-crawler/git/trees{/sha}","statuses_url":"https://api.github.com/repos/jez500/content-crawler/statuses/{sha}","languages_url":"https://api.github.com/repos/jez500/content-crawler/languages","stargazers_url":"https://api.github.com/repos/jez500/content-crawler/stargazers","contributors_url":"https://api.github.com/repos/jez500/content-crawler/contributors","subscribers_url":"https://api.github.com/repos/jez500/content-crawler/subscribers","subscription_url":"https://api.github.com/repos/jez500/content-crawler/subscription","commits_url":"https://api.github.com/repos/jez500/content-crawler/commits{/sha}","git_commits_url":"https://api.github.com/repos/jez500/content-crawler/git/commits{/sha}","comments_url":"https://api.github.com/repos/jez500/content-crawler/comments{/number}","issue_comment_url":"https://api.github.com/repos/jez500/content-crawler/issues/comments{/number}","contents_url":"https://api.github.com/repos/jez500/content-crawler/contents/{+path}","compare_url":"https://api.github.com/repos/jez500/content-crawler/compare/{base}...{head}","merges_url":"https://api.github.com/repos/jez500/content-crawler/merges","archive_url":"https://api.github.com/repos/jez500/content-crawler/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/jez500/content-crawler/downloads","issues_url":"https://api.github.com/repos/jez500/content-crawler/issues{/number}","pulls_url":"https://api.github.com/repos/jez500/content-crawler/pulls{/number}","milestones_url":"https://api.github.com/repos/jez500/content-crawler/milestones{/number}","notifications_url":"https://api.github.com/repos/jez500/content-crawler/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/jez500/content-crawler/labels{/name}","releases_url":"https://api.github.com/repos/jez500/content-crawler/releases{/id}","deployments_url":"https://api.github.com/repos/jez500/content-crawler/deployments","created_at":"2019-04-21T06:09:28Z","updated_at":"2019-11-21T22:33:55Z","pushed_at":"2019-11-25T01:29:07Z","git_url":"git://github.com/jez500/content-crawler.git","ssh_url":"git@github.com:jez500/content-crawler.git","clone_url":"https://github.com/jez500/content-crawler.git","svn_url":"https://github.com/jez500/content-crawler","homepage":null,"size":36,"stargazers_count":1,"watchers_count":1,"language":"JavaScript","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":1,"mirror_url":null,"archived":false,"disabled":false,"open_issues_count":1,"license":null,"forks":1,"open_issues":1,"watchers":1,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/jez500/content-crawler/pulls/3"},"html":{"href":"https://github.com/jez500/content-crawler/pull/3"},"issue":{"href":"https://api.github.com/repos/jez500/content-crawler/issues/3"},"comments":{"href":"https://api.github.com/repos/jez500/content-crawler/issues/3/comments"},"review_comments":{"href":"https://api.github.com/repos/jez500/content-crawler/pulls/3/comments"},"review_comment":{"href":"https://api.github.com/repos/jez500/content-crawler/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/jez500/content-crawler/pulls/3/commits"},"statuses":{"href":"https://api.github.com/repos/jez500/content-crawler/statuses/52a2e7f78566f96becb31756d206777ffd9e753a"}},"author_association":"CONTRIBUTOR"}}
{"actor":{"display_login":"jez500"}}