{"action":"created","comment":{"url":"https://api.github.com/repos/sidvishnoi/specxref/pulls/comments/235627404","pull_request_review_id":177534099,"id":235627404,"node_id":"MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDIzNTYyNzQwNA==","diff_hunk":"@@ -0,0 +1,104 @@\n+// Reads and parses anchor data files from bikeshed-data repository\n+// and writes data.json containing parsed and formatted data\n+\n+const { readFileSync, readdirSync, writeFileSync } = require(\"fs\");\n+const path = require(\"path\");\n+const fixURI = require(\"./fix-uri\");\n+\n+const INPUT_DIR = path.resolve(\"./bikeshed-data/data/anchors/\");\n+const OUT_FILE = path.resolve(\"./data.json\");\n+\n+const IDL_TYPES = new Set([\n+  \"attribute\",\n+  \"dict-member\",\n+  \"dictionary\",\n+  \"enum\",\n+  \"enum-value\",\n+  \"interface\",\n+  \"method\",\n+]);\n+const DFN_TYPES = new Set([\"dfn\", \"event\", \"element\"]);\n+\n+console.log(`Reading files from ${INPUT_DIR}`);\n+const fileNames = readdirSync(INPUT_DIR);\n+\n+const data = fileNames.reduce((data, fileName, i) => {\n+  process.stdout.write(`\\rProcessing: ${i + 1}/${fileNames.length}`);\n+  const file = path.join(INPUT_DIR, fileName);\n+  const content = readFileSync(file, \"utf8\");\n+  try {\n+    const terms = parseData(content);\n+    addTermsToData(terms, data);\n+  } catch (error) {\n+    console.error(`Error while processing ${fileName}`);\n+    throw error;\n+  }\n+  return data;\n+}, Object.create(null));","path":"scraper/scraper.js","position":37,"original_position":37,"commit_id":"57d92a550a07bcfe2164bd9cc278d2b3e1bb94e4","original_commit_id":"57d92a550a07bcfe2164bd9cc278d2b3e1bb94e4","user":{"login":"marcoscaceres","id":870154,"node_id":"MDQ6VXNlcjg3MDE1NA==","avatar_url":"https://avatars1.githubusercontent.com/u/870154?v=4","gravatar_id":"","url":"https://api.github.com/users/marcoscaceres","html_url":"https://github.com/marcoscaceres","followers_url":"https://api.github.com/users/marcoscaceres/followers","following_url":"https://api.github.com/users/marcoscaceres/following{/other_user}","gists_url":"https://api.github.com/users/marcoscaceres/gists{/gist_id}","starred_url":"https://api.github.com/users/marcoscaceres/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/marcoscaceres/subscriptions","organizations_url":"https://api.github.com/users/marcoscaceres/orgs","repos_url":"https://api.github.com/users/marcoscaceres/repos","events_url":"https://api.github.com/users/marcoscaceres/events{/privacy}","received_events_url":"https://api.github.com/users/marcoscaceres/received_events","type":"User","site_admin":false},"body":"Ok, cool :D Good reason. ","created_at":"2018-11-22T07:51:32Z","updated_at":"2018-11-22T07:51:32Z","html_url":"https://github.com/sidvishnoi/specxref/pull/1#discussion_r235627404","pull_request_url":"https://api.github.com/repos/sidvishnoi/specxref/pulls/1","author_association":"COLLABORATOR","_links":{"self":{"href":"https://api.github.com/repos/sidvishnoi/specxref/pulls/comments/235627404"},"html":{"href":"https://github.com/sidvishnoi/specxref/pull/1#discussion_r235627404"},"pull_request":{"href":"https://api.github.com/repos/sidvishnoi/specxref/pulls/1"}},"in_reply_to_id":235601307},"pull_request":{"url":"https://api.github.com/repos/sidvishnoi/specxref/pulls/1","id":232643786,"node_id":"MDExOlB1bGxSZXF1ZXN0MjMyNjQzNzg2","html_url":"https://github.com/sidvishnoi/specxref/pull/1","diff_url":"https://github.com/sidvishnoi/specxref/pull/1.diff","patch_url":"https://github.com/sidvishnoi/specxref/pull/1.patch","issue_url":"https://api.github.com/repos/sidvishnoi/specxref/issues/1","number":1,"state":"open","locked":false,"title":"Add scraper","user":{"login":"sidvishnoi","id":8426945,"node_id":"MDQ6VXNlcjg0MjY5NDU=","avatar_url":"https://avatars0.githubusercontent.com/u/8426945?v=4","gravatar_id":"","url":"https://api.github.com/users/sidvishnoi","html_url":"https://github.com/sidvishnoi","followers_url":"https://api.github.com/users/sidvishnoi/followers","following_url":"https://api.github.com/users/sidvishnoi/following{/other_user}","gists_url":"https://api.github.com/users/sidvishnoi/gists{/gist_id}","starred_url":"https://api.github.com/users/sidvishnoi/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/sidvishnoi/subscriptions","organizations_url":"https://api.github.com/users/sidvishnoi/orgs","repos_url":"https://api.github.com/users/sidvishnoi/repos","events_url":"https://api.github.com/users/sidvishnoi/events{/privacy}","received_events_url":"https://api.github.com/users/sidvishnoi/received_events","type":"User","site_admin":false},"body":"This adds the bikeshed-data scraper.\r\n\r\n``` bash\r\n# in project root:\r\ngit clone https://github.com/tabatkins/bikeshed-data.git\r\nnode scraper/scraper.js # writes data.json in project root\r\n```\r\n\r\nTODO:\r\n- ~Fix URI (remove host part, convert to short URI)~\r\n- Add automation script\r\n\r\nUpdate: Oh. And it's sync. Hopefully fine for a cron job script.","created_at":"2018-11-21T12:20:38Z","updated_at":"2018-11-22T07:51:32Z","closed_at":null,"merged_at":null,"merge_commit_sha":"ab882f0c763c20265f94715b30f9e3f3993ba72c","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/sidvishnoi/specxref/pulls/1/commits","review_comments_url":"https://api.github.com/repos/sidvishnoi/specxref/pulls/1/comments","review_comment_url":"https://api.github.com/repos/sidvishnoi/specxref/pulls/comments{/number}","comments_url":"https://api.github.com/repos/sidvishnoi/specxref/issues/1/comments","statuses_url":"https://api.github.com/repos/sidvishnoi/specxref/statuses/57d92a550a07bcfe2164bd9cc278d2b3e1bb94e4","head":{"label":"sidvishnoi:scraper","ref":"scraper","sha":"57d92a550a07bcfe2164bd9cc278d2b3e1bb94e4","user":{"login":"sidvishnoi","id":8426945,"node_id":"MDQ6VXNlcjg0MjY5NDU=","avatar_url":"https://avatars0.githubusercontent.com/u/8426945?v=4","gravatar_id":"","url":"https://api.github.com/users/sidvishnoi","html_url":"https://github.com/sidvishnoi","followers_url":"https://api.github.com/users/sidvishnoi/followers","following_url":"https://api.github.com/users/sidvishnoi/following{/other_user}","gists_url":"https://api.github.com/users/sidvishnoi/gists{/gist_id}","starred_url":"https://api.github.com/users/sidvishnoi/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/sidvishnoi/subscriptions","organizations_url":"https://api.github.com/users/sidvishnoi/orgs","repos_url":"https://api.github.com/users/sidvishnoi/repos","events_url":"https://api.github.com/users/sidvishnoi/events{/privacy}","received_events_url":"https://api.github.com/users/sidvishnoi/received_events","type":"User","site_admin":false},"repo":{"id":158532058,"node_id":"MDEwOlJlcG9zaXRvcnkxNTg1MzIwNTg=","name":"specxref","full_name":"sidvishnoi/specxref","private":false,"owner":{"login":"sidvishnoi","id":8426945,"node_id":"MDQ6VXNlcjg0MjY5NDU=","avatar_url":"https://avatars0.githubusercontent.com/u/8426945?v=4","gravatar_id":"","url":"https://api.github.com/users/sidvishnoi","html_url":"https://github.com/sidvishnoi","followers_url":"https://api.github.com/users/sidvishnoi/followers","following_url":"https://api.github.com/users/sidvishnoi/following{/other_user}","gists_url":"https://api.github.com/users/sidvishnoi/gists{/gist_id}","starred_url":"https://api.github.com/users/sidvishnoi/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/sidvishnoi/subscriptions","organizations_url":"https://api.github.com/users/sidvishnoi/orgs","repos_url":"https://api.github.com/users/sidvishnoi/repos","events_url":"https://api.github.com/users/sidvishnoi/events{/privacy}","received_events_url":"https://api.github.com/users/sidvishnoi/received_events","type":"User","site_admin":false},"html_url":"https://github.com/sidvishnoi/specxref","description":"Keyword based search API for CSSWG's Shepherd data, used in ReSpec.","fork":false,"url":"https://api.github.com/repos/sidvishnoi/specxref","forks_url":"https://api.github.com/repos/sidvishnoi/specxref/forks","keys_url":"https://api.github.com/repos/sidvishnoi/specxref/keys{/key_id}","collaborators_url":"https://api.github.com/repos/sidvishnoi/specxref/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/sidvishnoi/specxref/teams","hooks_url":"https://api.github.com/repos/sidvishnoi/specxref/hooks","issue_events_url":"https://api.github.com/repos/sidvishnoi/specxref/issues/events{/number}","events_url":"https://api.github.com/repos/sidvishnoi/specxref/events","assignees_url":"https://api.github.com/repos/sidvishnoi/specxref/assignees{/user}","branches_url":"https://api.github.com/repos/sidvishnoi/specxref/branches{/branch}","tags_url":"https://api.github.com/repos/sidvishnoi/specxref/tags","blobs_url":"https://api.github.com/repos/sidvishnoi/specxref/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/sidvishnoi/specxref/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/sidvishnoi/specxref/git/refs{/sha}","trees_url":"https://api.github.com/repos/sidvishnoi/specxref/git/trees{/sha}","statuses_url":"https://api.github.com/repos/sidvishnoi/specxref/statuses/{sha}","languages_url":"https://api.github.com/repos/sidvishnoi/specxref/languages","stargazers_url":"https://api.github.com/repos/sidvishnoi/specxref/stargazers","contributors_url":"https://api.github.com/repos/sidvishnoi/specxref/contributors","subscribers_url":"https://api.github.com/repos/sidvishnoi/specxref/subscribers","subscription_url":"https://api.github.com/repos/sidvishnoi/specxref/subscription","commits_url":"https://api.github.com/repos/sidvishnoi/specxref/commits{/sha}","git_commits_url":"https://api.github.com/repos/sidvishnoi/specxref/git/commits{/sha}","comments_url":"https://api.github.com/repos/sidvishnoi/specxref/comments{/number}","issue_comment_url":"https://api.github.com/repos/sidvishnoi/specxref/issues/comments{/number}","contents_url":"https://api.github.com/repos/sidvishnoi/specxref/contents/{+path}","compare_url":"https://api.github.com/repos/sidvishnoi/specxref/compare/{base}...{head}","merges_url":"https://api.github.com/repos/sidvishnoi/specxref/merges","archive_url":"https://api.github.com/repos/sidvishnoi/specxref/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/sidvishnoi/specxref/downloads","issues_url":"https://api.github.com/repos/sidvishnoi/specxref/issues{/number}","pulls_url":"https://api.github.com/repos/sidvishnoi/specxref/pulls{/number}","milestones_url":"https://api.github.com/repos/sidvishnoi/specxref/milestones{/number}","notifications_url":"https://api.github.com/repos/sidvishnoi/specxref/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/sidvishnoi/specxref/labels{/name}","releases_url":"https://api.github.com/repos/sidvishnoi/specxref/releases{/id}","deployments_url":"https://api.github.com/repos/sidvishnoi/specxref/deployments","created_at":"2018-11-21T10:38:22Z","updated_at":"2018-11-21T10:55:33Z","pushed_at":"2018-11-21T16:37:27Z","git_url":"git://github.com/sidvishnoi/specxref.git","ssh_url":"git@github.com:sidvishnoi/specxref.git","clone_url":"https://github.com/sidvishnoi/specxref.git","svn_url":"https://github.com/sidvishnoi/specxref","homepage":"","size":6,"stargazers_count":0,"watchers_count":0,"language":null,"has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":1,"license":null,"forks":0,"open_issues":1,"watchers":0,"default_branch":"master"}},"base":{"label":"sidvishnoi:master","ref":"master","sha":"d80643d6762627278d476d3e65e0e47a45725dad","user":{"login":"sidvishnoi","id":8426945,"node_id":"MDQ6VXNlcjg0MjY5NDU=","avatar_url":"https://avatars0.githubusercontent.com/u/8426945?v=4","gravatar_id":"","url":"https://api.github.com/users/sidvishnoi","html_url":"https://github.com/sidvishnoi","followers_url":"https://api.github.com/users/sidvishnoi/followers","following_url":"https://api.github.com/users/sidvishnoi/following{/other_user}","gists_url":"https://api.github.com/users/sidvishnoi/gists{/gist_id}","starred_url":"https://api.github.com/users/sidvishnoi/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/sidvishnoi/subscriptions","organizations_url":"https://api.github.com/users/sidvishnoi/orgs","repos_url":"https://api.github.com/users/sidvishnoi/repos","events_url":"https://api.github.com/users/sidvishnoi/events{/privacy}","received_events_url":"https://api.github.com/users/sidvishnoi/received_events","type":"User","site_admin":false},"repo":{"id":158532058,"node_id":"MDEwOlJlcG9zaXRvcnkxNTg1MzIwNTg=","name":"specxref","full_name":"sidvishnoi/specxref","private":false,"owner":{"login":"sidvishnoi","id":8426945,"node_id":"MDQ6VXNlcjg0MjY5NDU=","avatar_url":"https://avatars0.githubusercontent.com/u/8426945?v=4","gravatar_id":"","url":"https://api.github.com/users/sidvishnoi","html_url":"https://github.com/sidvishnoi","followers_url":"https://api.github.com/users/sidvishnoi/followers","following_url":"https://api.github.com/users/sidvishnoi/following{/other_user}","gists_url":"https://api.github.com/users/sidvishnoi/gists{/gist_id}","starred_url":"https://api.github.com/users/sidvishnoi/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/sidvishnoi/subscriptions","organizations_url":"https://api.github.com/users/sidvishnoi/orgs","repos_url":"https://api.github.com/users/sidvishnoi/repos","events_url":"https://api.github.com/users/sidvishnoi/events{/privacy}","received_events_url":"https://api.github.com/users/sidvishnoi/received_events","type":"User","site_admin":false},"html_url":"https://github.com/sidvishnoi/specxref","description":"Keyword based search API for CSSWG's Shepherd data, used in ReSpec.","fork":false,"url":"https://api.github.com/repos/sidvishnoi/specxref","forks_url":"https://api.github.com/repos/sidvishnoi/specxref/forks","keys_url":"https://api.github.com/repos/sidvishnoi/specxref/keys{/key_id}","collaborators_url":"https://api.github.com/repos/sidvishnoi/specxref/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/sidvishnoi/specxref/teams","hooks_url":"https://api.github.com/repos/sidvishnoi/specxref/hooks","issue_events_url":"https://api.github.com/repos/sidvishnoi/specxref/issues/events{/number}","events_url":"https://api.github.com/repos/sidvishnoi/specxref/events","assignees_url":"https://api.github.com/repos/sidvishnoi/specxref/assignees{/user}","branches_url":"https://api.github.com/repos/sidvishnoi/specxref/branches{/branch}","tags_url":"https://api.github.com/repos/sidvishnoi/specxref/tags","blobs_url":"https://api.github.com/repos/sidvishnoi/specxref/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/sidvishnoi/specxref/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/sidvishnoi/specxref/git/refs{/sha}","trees_url":"https://api.github.com/repos/sidvishnoi/specxref/git/trees{/sha}","statuses_url":"https://api.github.com/repos/sidvishnoi/specxref/statuses/{sha}","languages_url":"https://api.github.com/repos/sidvishnoi/specxref/languages","stargazers_url":"https://api.github.com/repos/sidvishnoi/specxref/stargazers","contributors_url":"https://api.github.com/repos/sidvishnoi/specxref/contributors","subscribers_url":"https://api.github.com/repos/sidvishnoi/specxref/subscribers","subscription_url":"https://api.github.com/repos/sidvishnoi/specxref/subscription","commits_url":"https://api.github.com/repos/sidvishnoi/specxref/commits{/sha}","git_commits_url":"https://api.github.com/repos/sidvishnoi/specxref/git/commits{/sha}","comments_url":"https://api.github.com/repos/sidvishnoi/specxref/comments{/number}","issue_comment_url":"https://api.github.com/repos/sidvishnoi/specxref/issues/comments{/number}","contents_url":"https://api.github.com/repos/sidvishnoi/specxref/contents/{+path}","compare_url":"https://api.github.com/repos/sidvishnoi/specxref/compare/{base}...{head}","merges_url":"https://api.github.com/repos/sidvishnoi/specxref/merges","archive_url":"https://api.github.com/repos/sidvishnoi/specxref/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/sidvishnoi/specxref/downloads","issues_url":"https://api.github.com/repos/sidvishnoi/specxref/issues{/number}","pulls_url":"https://api.github.com/repos/sidvishnoi/specxref/pulls{/number}","milestones_url":"https://api.github.com/repos/sidvishnoi/specxref/milestones{/number}","notifications_url":"https://api.github.com/repos/sidvishnoi/specxref/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/sidvishnoi/specxref/labels{/name}","releases_url":"https://api.github.com/repos/sidvishnoi/specxref/releases{/id}","deployments_url":"https://api.github.com/repos/sidvishnoi/specxref/deployments","created_at":"2018-11-21T10:38:22Z","updated_at":"2018-11-21T10:55:33Z","pushed_at":"2018-11-21T16:37:27Z","git_url":"git://github.com/sidvishnoi/specxref.git","ssh_url":"git@github.com:sidvishnoi/specxref.git","clone_url":"https://github.com/sidvishnoi/specxref.git","svn_url":"https://github.com/sidvishnoi/specxref","homepage":"","size":6,"stargazers_count":0,"watchers_count":0,"language":null,"has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":1,"license":null,"forks":0,"open_issues":1,"watchers":0,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/sidvishnoi/specxref/pulls/1"},"html":{"href":"https://github.com/sidvishnoi/specxref/pull/1"},"issue":{"href":"https://api.github.com/repos/sidvishnoi/specxref/issues/1"},"comments":{"href":"https://api.github.com/repos/sidvishnoi/specxref/issues/1/comments"},"review_comments":{"href":"https://api.github.com/repos/sidvishnoi/specxref/pulls/1/comments"},"review_comment":{"href":"https://api.github.com/repos/sidvishnoi/specxref/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/sidvishnoi/specxref/pulls/1/commits"},"statuses":{"href":"https://api.github.com/repos/sidvishnoi/specxref/statuses/57d92a550a07bcfe2164bd9cc278d2b3e1bb94e4"}},"author_association":"OWNER"}}
{"actor":{"display_login":"marcoscaceres"}}